diff --git a/.github/workflows/check-junit-test.yml b/.github/workflows/check-junit-test.yml index 58bbc6fe538..4078e6508fc 100644 --- a/.github/workflows/check-junit-test.yml +++ b/.github/workflows/check-junit-test.yml @@ -1,8 +1,36 @@ name: Check junits +run-name: "Check Junit. Hadoop ${{ inputs.hadoop }} ${{ inputs.module == 'all' && ' ' || format(':{0}. ', inputs.module ) }}${{ inputs.short_tests && 'Short ' || '' }}${{ inputs.medium_tests && 'Medium ' || '' }}${{ inputs.long_tests && 'Long ' || '' }}${{ ! ( inputs.short_tests || inputs.medium_tests || inputs.long_tests ) && 'Short Medium Long ' || '' }}tests." on: workflow_dispatch: inputs: + hadoop: + type: choice + description: 'Hadoop flavour.' + required: false + default: "hdp3.1" + options: + - "all" + - "hdp3.1" + - "hdi5.1" + - "emr6.1" + - "emr6.13" + module: + type: choice + description: 'OpenCGA module to test.' + required: false + default: "all" + options: + - "all" + - "opencga-analysis" + - "opencga-app" + - "opencga-catalog" + - "opencga-client" + - "opencga-core" + - "opencga-master" + - "opencga-server" + - "opencga-storage" + - "opencga-test" short_tests: type: boolean required: false @@ -25,31 +53,49 @@ jobs: name: Test JUnit runs-on: ubuntu-22.04 outputs: - profiles: ${{ steps.getter.outputs.profiles }} + profiles: ${{ steps.validate.outputs.profiles }} + modules: ${{ steps.validate.outputs.modules }} + hadoop: ${{ steps.validate.outputs.hadoop }} steps: - uses: actions/checkout@v4 with: fetch-depth: '10' - name: Building string profiles to run - id: getter + id: validate run: | if [ -f "./.github/workflows/scripts/get_profiles.sh" ]; then chmod +x ./.github/workflows/scripts/get_profiles.sh - echo "profiles=$(./.github/workflows/scripts/get_profiles.sh ${{ inputs.short_tests }} ${{ inputs.medium_tests }} ${{ inputs.long_tests }})" >> $GITHUB_OUTPUT + profiles=$(./.github/workflows/scripts/get_profiles.sh ${{ inputs.short_tests }} ${{ inputs.medium_tests }} ${{ inputs.long_tests }}) + modules='["${{ inputs.module }}"]' + hadoop='["${{ inputs.hadoop }}"]' + + if [ "${{ inputs.hadoop }}" == "all" ]; then + hadoop='["hdp3.1", "hdi5.1", "emr6.1", "emr6.13"]' + elif [ "${{ inputs.module }}" == "all" ]; then + # Only execute modules with matrix strategy if we are testing one single hadoop profile + modules='["opencga-analysis", "opencga-app", "opencga-catalog", "opencga-client", "opencga-core", "opencga-master", "opencga-server", "opencga-storage", "opencga-test"]' + fi + echo "profiles=[\"$profiles\"]" >> $GITHUB_OUTPUT + echo "modules=$modules" >> $GITHUB_OUTPUT + echo "hadoop=$hadoop" >> $GITHUB_OUTPUT + echo "Executing testing profiles -> $profiles" >> $GITHUB_STEP_SUMMARY + echo "Modules -> $modules" >> $GITHUB_STEP_SUMMARY + echo "Hadoop -> $hadoop" >> $GITHUB_STEP_SUMMARY + #echo "matrix={\"hadoop\": $hadoop, \"modules\": $modules}" >> $GITHUB_OUTPUT fi - - echo_log: - needs: get_profiles - runs-on: ubuntu-22.04 - steps: - - name: Echo profiles - id: log - run: echo "Executing testing profiles -> ${{ needs.get_profiles.outputs.profiles }}" - test: - needs: [ get_profiles, echo_log ] + strategy: + fail-fast: false + matrix: + hadoop: ${{ fromJson(needs.get_profiles.outputs.hadoop) }} + module: ${{ fromJson(needs.get_profiles.outputs.modules) }} + profile: ${{ fromJson(needs.get_profiles.outputs.profiles) }} + needs: [ get_profiles ] uses: ./.github/workflows/test-analysis.yml with: - test_profile: ${{ needs.get_profiles.outputs.profiles }} + test_profile: ${{ matrix.profile }} + hadoop: ${{ matrix.hadoop }} + module: ${{ matrix.module }} mvn_opts: ${{ inputs.mvn_opts }} + sonar: false secrets: inherit \ No newline at end of file diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml index 50c396dd7cc..00f6aa9fa53 100644 --- a/.github/workflows/develop.yml +++ b/.github/workflows/develop.yml @@ -11,25 +11,26 @@ jobs: build: uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop with: - maven_opts: -P storage-hadoop,hdp3.1,RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-storage-hadoop-deps-emr6.1,!:opencga-storage-hadoop-deps-hdp2.6' + maven_opts: -Phdp3.1,RClient -Dopencga.war.name=opencga -Dcheckstyle.skip - test: - uses: ./.github/workflows/test-analysis.yml - needs: build - with: - test_profile: runShortTests,runMediumTests - secrets: inherit +## TASK-4970: remove the test job from the develop.yml workflow because it is not needed anymore. +# test: +# uses: ./.github/workflows/test-analysis.yml +# needs: build +# with: +# test_profile: runShortTests,runMediumTests +# secrets: inherit deploy-maven: uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@develop - needs: test + needs: build with: - maven_opts: -P storage-hadoop,hdp3.1 -Dopencga.war.name=opencga + maven_opts: -P hdp3.1 -Dopencga.war.name=opencga secrets: inherit deploy-docker: uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop - needs: test + needs: build with: cli: python3 ./build/cloud/docker/docker-build.py push --images base,init secrets: inherit diff --git a/.github/workflows/docker-aws-emr.yml b/.github/workflows/docker-aws-emr.yml deleted file mode 100644 index b44a777134d..00000000000 --- a/.github/workflows/docker-aws-emr.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Release new AWS EMR OpenCGA version - -on: - push: - tags: - - '*' - workflow_dispatch: - -jobs: - build: - uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop - with: - maven_opts: -P storage-hadoop,emr6.1 -Dopencga.war.name=opencga - - deploy-docker: - uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop - needs: build - with: - cli: python3 ./build/cloud/docker/docker-build.py push --images base,init - secrets: inherit diff --git a/.github/workflows/long-test-analysis.yml b/.github/workflows/long-test-analysis.yml index c898c4dfa0b..10fdee5a0c3 100644 --- a/.github/workflows/long-test-analysis.yml +++ b/.github/workflows/long-test-analysis.yml @@ -9,9 +9,13 @@ on: jobs: test: + strategy: + fail-fast: false + matrix: + hadoop: [ "hdp3.1", "hdi5.1", "emr6.1", "emr6.13" ] uses: ./.github/workflows/test-analysis.yml - secrets: inherit with: - test_profile: runShortTests,runMediumTests,runLongTests, - + test_profile: runShortTests,runMediumTests,runLongTests + hadoop: ${{ matrix.hadoop }} + secrets: inherit diff --git a/.github/workflows/manual-deploy-docker.yml b/.github/workflows/manual-deploy-docker.yml index f4c79f4c241..3ee65923466 100644 --- a/.github/workflows/manual-deploy-docker.yml +++ b/.github/workflows/manual-deploy-docker.yml @@ -10,6 +10,16 @@ on: description: "The tag for the new docker." type: string required: true + hadoop: + type: choice + description: 'Hadoop flavour. Any of: [hdp3.1, hdi5.1, emr6.1, emr6.13]' + required: false + default: hdp3.1 + options: + - hdp3.1 + - hdi5.1 + - emr6.1 + - emr6.13 jobs: build: @@ -32,19 +42,18 @@ jobs: run: | if [ -f "./.github/workflows/scripts/get_same_branch.sh" ]; then chmod +x ./.github/workflows/scripts/get_same_branch.sh - ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }} + ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }} ${{ inputs.hadoop }} fi - name: Maven Build (skip tests) - run: mvn -T 2 clean install -DskipTests -P storage-hadoop,hdp3.1,RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-storage-hadoop-deps-emr6.1,!:opencga-storage-hadoop-deps-hdp2.6' + run: mvn -T 2 clean install -DskipTests -P${{ inputs.hadoop }} -Dopencga.war.name=opencga -Dcheckstyle.skip - uses: actions/upload-artifact@v4 with: name: build-folder path: build - deploy-docker: uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop needs: build with: cli: python3 ./build/cloud/docker/docker-build.py push --images base,init --tag ${{ inputs.tag }} - secrets: inherit \ No newline at end of file + secrets: inherit diff --git a/.github/workflows/manual-deploy-ext-tools.yml b/.github/workflows/manual-deploy-ext-tools.yml index f64e58cbb10..b614a520caf 100644 --- a/.github/workflows/manual-deploy-ext-tools.yml +++ b/.github/workflows/manual-deploy-ext-tools.yml @@ -10,6 +10,11 @@ on: description: "The tag for the new docker." type: string required: true + hadoop: + type: string + description: 'Hadoop flavour. Any of: [hdp3.1, hdi5.1, emr6.1, emr6.13]' + required: false + default: "hdp3.1" jobs: build: @@ -32,7 +37,7 @@ jobs: run: | if [ -f "./.github/workflows/scripts/get_same_branch.sh" ]; then chmod +x ./.github/workflows/scripts/get_same_branch.sh - ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }} + ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }} ${{ inputs.hadoop }} fi - name: Maven Build (skip tests) run: mvn -T 2 clean install -DskipTests @@ -41,11 +46,9 @@ jobs: name: build-folder path: build - deploy-docker-ext-tools: uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop needs: build with: cli: python3 ./build/cloud/docker/docker-build.py push --images ext-tools --tag ${{ inputs.tag }} secrets: inherit - diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml new file mode 100644 index 00000000000..0ff62b00d48 --- /dev/null +++ b/.github/workflows/pull-request-approved.yml @@ -0,0 +1,35 @@ +name: Pull request approve workflow +run-name: 'Pull request approve workflow ${{ github.event.pull_request.head.ref }} -> ${{ github.event.pull_request.base.ref }} by @${{ github.actor }}' + +on: + pull_request_review: + types: [ submitted ] + +jobs: + calculate-xetabase-branch: + name: Calculate Xetabase branch + runs-on: ubuntu-22.04 + outputs: + xetabase_branch: ${{ steps.get_xetabase_branch.outputs.xetabase_branch }} + steps: + - name: Clone project + uses: actions/checkout@v4 + with: + fetch-depth: '10' + - id: get_xetabase_branch + name: "Get current branch for Xetabase from target branch" + run: | + chmod +x ./.github/workflows/scripts/get-xetabase-branch.sh + xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh ${{ github.event.pull_request.base.ref }}) + echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY} + echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT + + test: + name: "Run all tests before merging" + needs: calculate-xetabase-branch + uses: opencb/java-common-libs/.github/workflows/test-xetabase-workflow.yml@TASK-6399 + with: + branch: ${{ needs.calculate-xetabase-branch.outputs.xetabase_branch }} + task: ${{ github.event.pull_request.head.ref }} + secrets: inherit + diff --git a/.github/workflows/pull-request-merge.yml b/.github/workflows/pull-request-merged.yml similarity index 72% rename from .github/workflows/pull-request-merge.yml rename to .github/workflows/pull-request-merged.yml index faedd27bebc..24906414101 100644 --- a/.github/workflows/pull-request-merge.yml +++ b/.github/workflows/pull-request-merged.yml @@ -13,7 +13,7 @@ jobs: build: uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop with: - maven_opts: -P storage-hadoop,hdp3.1,RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-storage-hadoop-deps-emr6.1,!:opencga-storage-hadoop-deps-hdp2.6' + maven_opts: -P hdp3.1,RClient -Dopencga.war.name=opencga -Dcheckstyle.skip delete-docker: uses: opencb/java-common-libs/.github/workflows/delete-docker-hub-workflow.yml@develop diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0481b386dca..b59c92125e0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,39 +7,90 @@ on: workflow_dispatch: jobs: - build: + # Add the build-hdp job + build-hdp: uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop with: - maven_opts: -P storage-hadoop,hdp3.1,RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-storage-hadoop-deps-emr6.1,!:opencga-storage-hadoop-deps-hdp2.6' + maven_opts: -P hdp3.1,RClient -Dopencga.war.name=opencga -Dcheckstyle.skip + build_folder: build-folder - deploy-maven: - uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@develop - needs: build + # Add the deploy-docker-hdp job that depends on the build-hdp job + deploy-docker-hdp: + uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop + needs: build-hdp with: - maven_opts: -P storage-hadoop,hdp3.1 -Dopencga.war.name=opencga + cli: python3 ./build/cloud/docker/docker-build.py push --images base,init --tag "${{ needs.build-hdp.outputs.version }}-hdp3.1" + build_folder: build-folder secrets: inherit - deploy-docker: - uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop - needs: build + # Add the deploy-maven and deploy-python jobs they depend on the build-hdp job + deploy-maven: + uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@develop + needs: build-hdp with: - cli: python3 ./build/cloud/docker/docker-build.py push --images base,init --tag ${{ needs.build.outputs.version }} + maven_opts: -P hdp3.1 -Dopencga.war.name=opencga secrets: inherit deploy-python: uses: opencb/java-common-libs/.github/workflows/deploy-python-workflow.yml@develop - needs: build + needs: build-hdp with: cli: ./clients/python/python-build.sh push artifact: build-folder secrets: inherit + # Add the release job that depends on all the previous jobs release: uses: opencb/java-common-libs/.github/workflows/release-github-workflow.yml@develop - needs: [ build, deploy-maven, deploy-docker, deploy-python ] + needs: [ build-hdp, deploy-docker-hdp, deploy-maven, deploy-python] with: artifact: build-folder file: | - opencga-client-${{ needs.build.outputs.version }}.tar.gz - clients/R/opencgaR_${{ needs.build.outputs.version }}.tar.gz + opencga-client-${{ needs.build-hdp.outputs.version }}.tar.gz + clients/R/opencgaR_${{ needs.build-hdp.outputs.version }}.tar.gz + + # Compile and deploy other hadoop flavours + # Add the build-hdi and deploy-docker-hdi jobs + build-hdi: + uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop + with: + maven_opts: -P hdi5.1,RClient -Dopencga.war.name=opencga -Dcheckstyle.skip + build_folder: build-folder-hdi + + deploy-docker-hdi: + uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop + needs: build-hdi + with: + cli: python3 ./build/cloud/docker/docker-build.py push --images base,init --tag "${{ needs.build-hdi.outputs.version }}-hdi5.1" + build_folder: build-folder-hdi + secrets: inherit + # Add the build-emr and deploy-docker-emr jobs + build-emr: + uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop + with: + maven_opts: -P emr6.1,RClient -Dopencga.war.name=opencga -Dcheckstyle.skip + build_folder: build-folder-emr + + deploy-docker-emr: + uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop + needs: build-emr + with: + cli: python3 ./build/cloud/docker/docker-build.py push --images base,init --tag "${{ needs.build-emr.outputs.version }}-emr6.1" + build_folder: build-folder-emr + secrets: inherit + + # Add the build-emr613 and deploy-docker-emr613 jobs + build-emr613: + uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop + with: + maven_opts: -P emr6.13,RClient -Dopencga.war.name=opencga -Dcheckstyle.skip + build_folder: build-folder-emr613 + + deploy-docker-emr613: + uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop + needs: build-emr613 + with: + cli: python3 ./build/cloud/docker/docker-build.py push --images base,init --tag "${{ needs.build-emr613.outputs.version }}-emr6.13" + build_folder: build-folder-emr613 + secrets: inherit diff --git a/.github/workflows/scripts/get-xetabase-branch.sh b/.github/workflows/scripts/get-xetabase-branch.sh new file mode 100644 index 00000000000..e971f990c2e --- /dev/null +++ b/.github/workflows/scripts/get-xetabase-branch.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +# Function to calculate the corresponding branch of Xetabase project +get_xetabase_branch() { + # Input parameter (branch name) + input_branch="$1" + + # If the branch begins with 'TASK' and exists in the opencga-enterprise repository, I return it + if [[ $input_branch == TASK* ]]; then + if [ "$(git ls-remote https://github.com/zetta-genomics/opencga-enterprise.git "$input_branch" )" ] ; then + echo "$GIT_BRANCH"; + exit 0; + fi + fi + + # Check if the branch name is "develop" in that case return the same branch name + if [[ "$input_branch" == "develop" ]]; then + echo "develop" + return 0 + fi + + # Check if the branch name starts with "release-" and follows the patterns "release-a.b.x" or "release-a.b.c.x" + if [[ "$input_branch" =~ ^release-([0-9]+)\.([0-9]+)\.x$ ]] || [[ "$input_branch" =~ ^release-([0-9]+)\.([0-9]+)\.([0-9]+)\.x$ ]]; then + # Extract the MAJOR part of the branch name + MAJOR=${BASH_REMATCH[1]} + # Calculate the XETABASE_MAJOR by subtracting 3 from MAJOR + XETABASE_MAJOR=$((MAJOR - 1)) + # Check if the XETABASE_MAJOR is negative + if (( XETABASE_MAJOR < 0 )); then + echo "Error: 'MAJOR' digit after subtraction results in a negative number." + return 1 + fi + # Construct and echo the new branch name + echo "release-$XETABASE_MAJOR.${input_branch#release-$MAJOR.}" + return 0 + fi + + # If the branch name does not match any of the expected patterns + echo "Error: The branch name is not correct." + return 1 +} + +# Check if the script receives exactly one argument +if [ "$#" -ne 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +# Call the function with the input branch name +get_xetabase_branch "$1" diff --git a/.github/workflows/scripts/get_same_branch.sh b/.github/workflows/scripts/get_same_branch.sh index 750271b0c6c..5e9d9062fa1 100644 --- a/.github/workflows/scripts/get_same_branch.sh +++ b/.github/workflows/scripts/get_same_branch.sh @@ -1,13 +1,13 @@ #!/bin/bash BRANCH_NAME=$1 +HADOOP=${2:-hdp3.1} -if [[ -z $BRANCH_NAME ]]; then +if [[ -z "$BRANCH_NAME" ]]; then echo "The first parameter is mandatory and must be a valid branch name." exit 1 fi - function install(){ local REPO=$1 cd /home/runner/work/ || exit 2 @@ -15,7 +15,11 @@ function install(){ if [ -d "./$REPO" ]; then cd "$REPO" || exit 2 echo "Branch name $BRANCH_NAME already exists." - mvn clean install -DskipTests + if [[ "$REPO" == "opencga-hadoop-thirdparty" ]]; then + ./dev/build.sh "$HADOOP" + else + mvn clean install -DskipTests + fi else echo "$CURRENT Branch is NOT EQUALS $BRANCH_NAME " fi @@ -24,3 +28,4 @@ function install(){ install "java-common-libs" install "biodata" install "cellbase" +install "opencga-hadoop-thirdparty" diff --git a/.github/workflows/task.yml b/.github/workflows/task.yml index 51888a9179d..cc8470b747a 100644 --- a/.github/workflows/task.yml +++ b/.github/workflows/task.yml @@ -12,14 +12,14 @@ jobs: build: uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop with: - maven_opts: -P storage-hadoop,hdp3.1,RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-storage-hadoop-deps-emr6.1,!:opencga-storage-hadoop-deps-hdp2.6' + maven_opts: -Phdp3.1,RClient -Dopencga.war.name=opencga -Dcheckstyle.skip test: uses: ./.github/workflows/test-analysis.yml needs: build - secrets: inherit with: test_profile: runShortTests + secrets: inherit deploy-docker: uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop diff --git a/.github/workflows/test-analysis.yml b/.github/workflows/test-analysis.yml index ccb848d11ec..554e1f72520 100644 --- a/.github/workflows/test-analysis.yml +++ b/.github/workflows/test-analysis.yml @@ -1,14 +1,29 @@ name: Build and test the project +run-name: "Build and test the project. Hadoop flavour: ${{ inputs.hadoop }}. Test profile: ${{ inputs.test_profile }}. Test module: ${{ inputs.module }}" on: workflow_call: inputs: test_profile: + description: 'Maven test profile. Any combination of : [runShortTests, runMediumTests, runLongTests]' type: string required: true + hadoop: + type: string + description: 'Hadoop flavour. Any of: [hdp3.1, hdi5.1, emr6.1, emr6.13]' + required: false + default: "hdp3.1" mvn_opts: type: string - + required: false + default: "" + sonar: + type: boolean + required: false + default: true + module: + type: string + description: "Maven modules to test. Empty means all. Only top-level modules. Example: 'opencga-storage'" required: false default: "" secrets: @@ -19,10 +34,11 @@ jobs: analysis: name: Execute Sonar Analysis runs-on: ubuntu-22.04 + if: ${{ inputs.sonar }} steps: - uses: actions/checkout@v4 with: - fetch-depth: '0' + fetch-depth: '10' - name: Set up JDK 11 uses: actions/setup-java@v4 with: @@ -33,7 +49,7 @@ jobs: run: | if [ -f "./.github/workflows/scripts/get_same_branch.sh" ]; then chmod +x ./.github/workflows/scripts/get_same_branch.sh - ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }} + ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }} ${{ inputs.hadoop }} else echo "./.github/workflows/scripts/get_same_branch.sh does not exist." fi @@ -41,7 +57,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - run: mvn -T 2 clean install -P storage-hadoop,hdp3.1,${{ inputs.test_profile }} -DskipTests -Dcheckstyle.skip org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=opencb_opencga + run: mvn -T 2 clean install -P ${{ inputs.hadoop }},${{ inputs.test_profile }} -DskipTests -Dcheckstyle.skip org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=opencb_opencga test: name: Execute JUnit and Jacoco tests @@ -49,7 +65,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - fetch-depth: '0' + fetch-depth: '10' - name: Set up JDK 8 uses: actions/setup-java@v4 with: @@ -60,7 +76,7 @@ jobs: run: | if [ -f "./.github/workflows/scripts/get_same_branch.sh" ]; then chmod +x ./.github/workflows/scripts/get_same_branch.sh - ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }} + ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }} ${{ inputs.hadoop }} else echo "./.github/workflows/scripts/get_same_branch.sh does not exist." fi @@ -71,9 +87,11 @@ jobs: with: mongodb-version: 6.0 mongodb-replica-set: rs-test + - name: Maven build + run: mvn -B clean install -DskipTests -P ${{ inputs.hadoop }} -Dcheckstyle.skip ${{ inputs.mvn_opts }} - name: Run Junit tests - run: mvn -B verify surefire-report:report --fail-never -P storage-hadoop,hdp3.1,${{ inputs.test_profile }} -Dcheckstyle.skip -Popencga-storage-hadoop-deps -pl '!:opencga-storage-hadoop-deps-emr6.1,!:opencga-storage-hadoop-deps-hdp2.6' ${{ inputs.mvn_opts }} - - name: Publish Test Report + run: mvn -B verify surefire-report:report --fail-never -Dsurefire.testFailureIgnore=true -f ${{ (inputs.module == '' || inputs.module == 'all') && '.' || inputs.module }} -P ${{ inputs.hadoop }},${{ inputs.test_profile }} -Dcheckstyle.skip ${{ inputs.mvn_opts }} + - name: Publish Test Report on GitHub uses: scacap/action-surefire-report@v1 env: NODE_OPTIONS: '--max_old_space_size=4096' @@ -81,9 +99,10 @@ jobs: ## https://docs.github.com/en/actions/learn-github-actions/expressions#cancelled if: success() || failure() with: - check_name: "Surefire tests report" + check_name: "Surefire tests report ${{ inputs.hadoop }} ${{ inputs.module }} ${{ inputs.test_profile }}" report_paths: './**/surefire-reports/TEST-*.xml' commit: '${{ github.sha }}' fail_on_test_failures: true + fail_if_no_tests: false diff --git a/checkstyle.xml b/checkstyle.xml index 4e8b71e32da..640ded5df1b 100644 --- a/checkstyle.xml +++ b/checkstyle.xml @@ -21,7 +21,7 @@ - + diff --git a/opencga-analysis/pom.xml b/opencga-analysis/pom.xml index 7c32e987cdc..1d127746157 100644 --- a/opencga-analysis/pom.xml +++ b/opencga-analysis/pom.xml @@ -66,31 +66,50 @@ org.opencb.opencga opencga-storage-hadoop-core - test-jar test + test-jar org.opencb.opencga - ${opencga-storage-hadoop-deps.artifactId} + opencga-storage-hadoop-compat-${opencga-storage-hadoop-compat.id} ${project.parent.version} - shaded + test - org.opencb.opencga - ${opencga-storage-hadoop-deps.artifactId} - ${project.parent.version} - tests - test-jar + org.opencb.opencga.hadoop.thirdparty + ${opencga-hadoop-shaded.artifactId} + ${opencga.hadoop.thirdparty.version} test + + + org.yaml + snakeyaml + + + + + org.opencb.opencga.hadoop.thirdparty + ${opencga-hadoop-shaded.artifactId} + ${opencga.hadoop.thirdparty.version} + test + test-jar + + + org.yaml + snakeyaml + + org.mockito mockito-core + test junit junit + test org.opencb.opencga @@ -105,7 +124,6 @@ org.opencb.commons commons-datastore-mongodb - test org.opencb.commons @@ -159,10 +177,6 @@ org.apache.commons commons-lang3 - - commons-lang - commons-lang - commons-io commons-io @@ -232,16 +246,116 @@ org.apache.solr solr-test-framework + test - + javax.servlet javax.servlet-api test + + + + + org.eclipse.jetty + jetty-server + ${jetty-for-hadoop-test.version} + test + + + org.eclipse.jetty + jetty-webapp + ${jetty-for-hadoop-test.version} + test + + + org.eclipse.jetty + jetty-servlet + ${jetty-for-hadoop-test.version} + test + + + org.eclipse.jetty + jetty-http + ${jetty-for-hadoop-test.version} + test + + + org.eclipse.jetty.http2 + http2-hpack + ${jetty-for-hadoop-test.version} + test + + + org.eclipse.jetty + jetty-io + ${jetty-for-hadoop-test.version} + test + + + org.eclipse.jetty + jetty-util + ${jetty-for-hadoop-test.version} + test + + + org.eclipse.jetty + jetty-util-ajax + ${jetty-for-hadoop-test.version} + test + + + org.eclipse.jetty + jetty-xml + ${jetty-for-hadoop-test.version} + test + + + org.eclipse.jetty + jetty-rewrite + ${jetty-for-hadoop-test.version} + test + + + org.eclipse.jetty + jetty-security + ${jetty-for-hadoop-test.version} + test + + + org.eclipse.jetty + jetty-client + ${jetty-for-hadoop-test.version} + test + + + org.eclipse.jetty + jetty-project + ${jetty-for-hadoop-test.version} + test + + + org.eclipse.jetty + jetty-alpn-client + ${jetty-for-hadoop-test.version} + test + + + org.eclipse.jetty + jetty-alpn-java-client + ${jetty-for-hadoop-test.version} + test + + + + @@ -282,10 +396,17 @@ - com.google.code.findbugs:jsr305:jar:3.0.2 + com.google.code.findbugs:jsr305 + + + org.opencb.opencga.hadoop.thirdparty:* - * + + + org.opencb.commons:commons-datastore-mongodb + + * diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/ConfigurationUtils.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/ConfigurationUtils.java index bc6e1dc674b..c494f8db7ca 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/ConfigurationUtils.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/ConfigurationUtils.java @@ -16,9 +16,13 @@ package org.opencb.opencga.analysis; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.opencb.commons.utils.FileUtils; +import org.opencb.opencga.core.config.AnalysisTool; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.config.storage.StorageConfiguration; +import org.opencb.opencga.core.exceptions.ToolException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,10 +32,15 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; public class ConfigurationUtils { private static Logger logger = LoggerFactory.getLogger(ConfigurationUtils.class); + private ConfigurationUtils() { + throw new IllegalStateException("Utility class"); + } /** * This method attempts to load general configuration from OpenCGA installation folder, if not exists then loads JAR configuration.yml. * @@ -83,4 +92,33 @@ public static StorageConfiguration loadStorageConfiguration(String opencgaHome) .load(StorageConfiguration.class.getClassLoader().getResourceAsStream("storage-configuration.yml")); } } + + public static String getToolDefaultVersion(String toolId, Configuration configuration) throws ToolException { + List tools = new ArrayList<>(); + for (AnalysisTool tool : configuration.getAnalysis().getTools()) { + if (tool.getId().equals(toolId)) { + tools.add(tool); + } + } + if (CollectionUtils.isEmpty(tools)) { + throw new ToolException("Tool ID '" + toolId + "' missing in the configuration file"); + } + if (tools.size() == 1) { + return tools.get(0).getVersion(); + } + String defaultVersion = null; + for (AnalysisTool tool : tools) { + if (tool.isDefaultVersion()) { + if (!StringUtils.isEmpty(defaultVersion)) { + throw new ToolException("More than one default version found for tool ID '" + toolId + "'"); + } else { + defaultVersion = tool.getVersion(); + } + } + } + if (StringUtils.isEmpty(defaultVersion)) { + throw new ToolException("Multiple tools '" + toolId + "' were found, but none have the default version set to true"); + } + return defaultVersion; + } } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/ResourceUtils.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/ResourceUtils.java index 11c9741cde2..161c07890df 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/ResourceUtils.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/ResourceUtils.java @@ -24,6 +24,8 @@ import org.opencb.opencga.catalog.db.api.ProjectDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.CatalogManager; +import org.opencb.opencga.catalog.utils.CatalogFqn; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.project.Project; import org.opencb.opencga.core.response.OpenCGAResult; @@ -116,7 +118,12 @@ public static DownloadedRefGenome downloadRefGenome(String assembly, Path outDir public static String getAssembly(CatalogManager catalogManager, String studyId, String sessionId) throws CatalogException { String assembly = ""; OpenCGAResult projectQueryResult; - projectQueryResult = catalogManager.getProjectManager().search(new Query(ProjectDBAdaptor.QueryParams.STUDY.key(), studyId), + + JwtPayload jwtPayload = catalogManager.getUserManager().validateToken(sessionId); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, jwtPayload); + String organizationId = studyFqn.getOrganizationId(); + + projectQueryResult = catalogManager.getProjectManager().search(organizationId, new Query(ProjectDBAdaptor.QueryParams.STUDY.key(), studyId), new QueryOptions(QueryOptions.INCLUDE, ProjectDBAdaptor.QueryParams.ORGANISM.key()), sessionId); if (CollectionUtils.isNotEmpty(projectQueryResult.getResults()) && projectQueryResult.first().getOrganism() != null diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/StorageManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/StorageManager.java index 6fd2ab85719..484e6a5131a 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/StorageManager.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/StorageManager.java @@ -24,8 +24,10 @@ import org.opencb.opencga.catalog.db.api.ProjectDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.CatalogManager; +import org.opencb.opencga.catalog.utils.CatalogFqn; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.config.storage.StorageConfiguration; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.file.File; import org.opencb.opencga.core.models.project.DataStore; import org.opencb.opencga.core.models.project.Project; @@ -73,19 +75,6 @@ protected StorageManager(CatalogManager catalogManager, CacheManager cacheManage logger = LoggerFactory.getLogger(getClass()); } - - public void clearCache(String sessionId) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(sessionId); - - } - - - public void clearCache(String studyId, String sessionId) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(sessionId); - - } - - public abstract void testConnection() throws StorageEngineException; @Deprecated @@ -95,18 +84,20 @@ protected StudyInfo getStudyInfo(@Nullable String studyIdStr, String fileIdStr, } @Deprecated - protected StudyInfo getStudyInfo(@Nullable String studyIdStr, List fileIdStrs, String sessionId) - throws CatalogException { - StudyInfo studyInfo = new StudyInfo().setSessionId(sessionId); + protected StudyInfo getStudyInfo(@Nullable String studyIdStr, List fileIdStrs, String token) throws CatalogException { + StudyInfo studyInfo = new StudyInfo().setSessionId(token); + + JwtPayload jwtPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn catalogFqn = CatalogFqn.extractFqnFromStudy(studyIdStr, jwtPayload); + String organizationId = catalogFqn.getOrganizationId(); - String userId = catalogManager.getUserManager().getUserId(sessionId); - Study study = catalogManager.getStudyManager().get(studyIdStr, QueryOptions.empty(), sessionId).first(); + Study study = catalogManager.getStudyManager().get(studyIdStr, QueryOptions.empty(), token).first(); List files; if (fileIdStrs.isEmpty()) { files = Collections.emptyList(); } else { - DataResult queryResult = catalogManager.getFileManager().get(studyIdStr, fileIdStrs, null, sessionId); + DataResult queryResult = catalogManager.getFileManager().get(studyIdStr, fileIdStrs, null, token); files = queryResult.getResults(); } List fileInfos = new ArrayList<>(fileIdStrs.size()); @@ -130,17 +121,15 @@ protected StudyInfo getStudyInfo(@Nullable String studyIdStr, List fileI studyInfo.setStudy(study); String projectFqn = catalogManager.getStudyManager().getProjectFqn(study.getFqn()); - Project project = catalogManager.getProjectManager().search(new Query(ProjectDBAdaptor.QueryParams.FQN.key(), projectFqn), - new QueryOptions(), sessionId).first(); + Project project = catalogManager.getProjectManager().search(organizationId, new Query(ProjectDBAdaptor.QueryParams.FQN.key(), projectFqn), + new QueryOptions(), token).first(); studyInfo.setProjectUid(project.getUid()); studyInfo.setProjectId(project.getId()); studyInfo.setOrganism(project.getOrganism()); - String user = catalogManager.getProjectManager().getOwner(project.getUid()); - studyInfo.setUserId(user); Map dataStores = new HashMap<>(); - dataStores.put(File.Bioformat.VARIANT, getDataStore(catalogManager, study.getFqn(), File.Bioformat.VARIANT, sessionId)); - dataStores.put(File.Bioformat.ALIGNMENT, getDataStore(catalogManager, study.getFqn(), File.Bioformat.ALIGNMENT, sessionId)); + dataStores.put(File.Bioformat.VARIANT, getDataStore(catalogManager, study.getFqn(), File.Bioformat.VARIANT, token)); + dataStores.put(File.Bioformat.ALIGNMENT, getDataStore(catalogManager, study.getFqn(), File.Bioformat.ALIGNMENT, token)); studyInfo.setDataStores(dataStores); return studyInfo; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentIndexOperation.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentIndexOperation.java index 0c597a6a8f4..fa537f5a973 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentIndexOperation.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentIndexOperation.java @@ -128,7 +128,7 @@ protected void run() throws Exception { // Update BAM file internal in order to set the alignment index (BAI) FileInternalAlignmentIndex fileAlignmentIndex = new FileInternalAlignmentIndex(new InternalStatus(InternalStatus.READY), baiCatalogFile.getId(), "HTSJDK library"); - catalogManager.getFileManager().updateFileInternalAlignmentIndex(inputCatalogFile, fileAlignmentIndex, token); + catalogManager.getFileManager().updateFileInternalAlignmentIndex(study, inputCatalogFile, fileAlignmentIndex, token); }); } } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentStorageManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentStorageManager.java index 5c267add689..9dbf2d9223a 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentStorageManager.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentStorageManager.java @@ -27,23 +27,18 @@ import org.opencb.biodata.tools.alignment.BamUtils; import org.opencb.cellbase.client.rest.CellBaseClient; import org.opencb.cellbase.client.rest.GeneClient; -import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.commons.utils.FileUtils; import org.opencb.opencga.analysis.StorageManager; import org.opencb.opencga.analysis.models.FileInfo; import org.opencb.opencga.analysis.models.StudyInfo; -import org.opencb.opencga.analysis.tools.ToolRunner; import org.opencb.opencga.catalog.db.api.FileDBAdaptor; import org.opencb.opencga.catalog.db.api.ProjectDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.CatalogManager; +import org.opencb.opencga.catalog.utils.CatalogFqn; import org.opencb.opencga.catalog.utils.ParamUtils; -import org.opencb.opencga.core.api.ParamConstants; -import org.opencb.opencga.core.exceptions.ToolException; -import org.opencb.opencga.core.models.alignment.AlignmentIndexParams; -import org.opencb.opencga.core.models.alignment.CoverageIndexParams; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.file.File; import org.opencb.opencga.core.models.project.Project; import org.opencb.opencga.core.models.study.Study; @@ -69,6 +64,7 @@ public class AlignmentStorageManager extends StorageManager { private AlignmentStorageEngine alignmentStorageEngine; private String jobId; + private boolean dryRun; private static final Map statsMap = new HashMap<>(); @@ -81,12 +77,13 @@ public AlignmentStorageManager(CatalogManager catalogManager, StorageEngineFacto initStatsMap(); } - public AlignmentStorageManager(CatalogManager catalogManager, StorageEngineFactory storageEngineFactory, String jobId) { + public AlignmentStorageManager(CatalogManager catalogManager, StorageEngineFactory storageEngineFactory, String jobId, boolean dryRun) { super(catalogManager, storageEngineFactory); // TODO: Create this alignmentStorageEngine by reflection this.alignmentStorageEngine = new LocalAlignmentStorageEngine(); this.jobId = jobId; + this.dryRun = dryRun; initStatsMap(); } @@ -219,9 +216,13 @@ public OpenCGAResult coverageStats(String studyIdStr, String File file = extractAlignmentOrCoverageFile(studyIdStr, fileIdStr, token); // System.out.println("file = " + file.getUri()); + JwtPayload jwtPayload = new JwtPayload(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyIdStr, jwtPayload); + String organizationId = studyFqn.getOrganizationId(); + // Get species and assembly from catalog OpenCGAResult projectQueryResult = catalogManager.getProjectManager().search( - new Query(ProjectDBAdaptor.QueryParams.STUDY.key(), studyIdStr), + organizationId, new Query(ProjectDBAdaptor.QueryParams.STUDY.key(), studyIdStr), new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(ProjectDBAdaptor.QueryParams.ORGANISM.key(), ProjectDBAdaptor.QueryParams.CELLBASE.key())), token); if (projectQueryResult.getNumResults() != 1) { @@ -437,9 +438,12 @@ public List mergeRegions(List regions, List genes, boole } } + JwtPayload jwtPayload = new JwtPayload(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(study, jwtPayload); + String organizationId = studyFqn.getOrganizationId(); // Get species and assembly from catalog OpenCGAResult projectQueryResult = catalogManager.getProjectManager().search( - new Query(ProjectDBAdaptor.QueryParams.STUDY.key(), study), + organizationId, new Query(ProjectDBAdaptor.QueryParams.STUDY.key(), study), new QueryOptions(QueryOptions.INCLUDE, Arrays.asList( ProjectDBAdaptor.QueryParams.ORGANISM.key(), ProjectDBAdaptor.QueryParams.CELLBASE.key())), token); @@ -562,23 +566,6 @@ private void checkAlignmentBioformat(List fileInfo) throws CatalogExce public void testConnection() throws StorageEngineException { } - @Deprecated - private Path getFilePath(long fileId, String sessionId) throws CatalogException, IOException { - QueryOptions fileOptions = new QueryOptions(QueryOptions.INCLUDE, - Arrays.asList(FileDBAdaptor.QueryParams.URI.key(), FileDBAdaptor.QueryParams.NAME.key())); - OpenCGAResult fileResult = catalogManager.getFileManager().get(fileId, fileOptions, sessionId); - - if (fileResult.getNumResults() != 1) { - logger.error("Critical error: File {} not found in catalog.", fileId); - throw new CatalogException("Critical error: File " + fileId + " not found in catalog"); - } - - Path path = Paths.get(fileResult.first().getUri().getRawPath()); - FileUtils.checkFile(path); - - return path; - } - @Deprecated private Path getWorkspace(long studyId, String sessionId) throws CatalogException, IOException { // Obtain the study uri diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/qc/AlignmentFastQcMetricsAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/qc/AlignmentFastQcMetricsAnalysis.java index c2541d1560d..1dd71a315af 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/qc/AlignmentFastQcMetricsAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/qc/AlignmentFastQcMetricsAnalysis.java @@ -17,7 +17,6 @@ package org.opencb.opencga.analysis.alignment.qc; import org.apache.commons.lang3.StringUtils; -import org.opencb.biodata.formats.alignment.samtools.SamtoolsFlagstats; import org.opencb.biodata.formats.sequence.fastqc.FastQcMetrics; import org.opencb.biodata.formats.sequence.fastqc.io.FastQcParser; import org.opencb.commons.datastore.core.Query; @@ -29,8 +28,6 @@ import org.opencb.opencga.core.models.alignment.AlignmentFastQcMetricsParams; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.file.File; -import org.opencb.opencga.core.models.file.FileQualityControl; -import org.opencb.opencga.core.models.file.FileUpdateParams; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.tools.annotations.Tool; import org.opencb.opencga.core.tools.annotations.ToolParams; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/qc/AlignmentFlagStatsAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/qc/AlignmentFlagStatsAnalysis.java index e78cab5a9d9..8e18085e69d 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/qc/AlignmentFlagStatsAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/qc/AlignmentFlagStatsAnalysis.java @@ -30,8 +30,6 @@ import org.opencb.opencga.core.models.alignment.AlignmentFlagStatsParams; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.file.File; -import org.opencb.opencga.core.models.file.FileQualityControl; -import org.opencb.opencga.core.models.file.FileUpdateParams; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.tools.annotations.Tool; import org.opencb.opencga.core.tools.annotations.ToolParams; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/qc/AlignmentQcAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/qc/AlignmentQcAnalysis.java index d4d08bba37f..67ac836c36a 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/qc/AlignmentQcAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/qc/AlignmentQcAnalysis.java @@ -17,7 +17,6 @@ package org.opencb.opencga.analysis.alignment.qc; import org.apache.commons.lang3.StringUtils; -import org.opencb.biodata.formats.alignment.picard.HsMetrics; import org.opencb.biodata.formats.alignment.samtools.SamtoolsFlagstats; import org.opencb.biodata.formats.alignment.samtools.SamtoolsStats; import org.opencb.biodata.formats.sequence.fastqc.FastQcMetrics; @@ -121,7 +120,8 @@ protected void run() throws ToolException { OpenCGAResult flagStatsJobResult = catalogManager.getJobManager() .submit(study, AlignmentFlagStatsAnalysis.ID, Enums.Priority.MEDIUM, params, null, "Job generated by " - + getId() + " - " + getJobId(), Collections.emptyList(), Collections.emptyList(), token); + + getId() + " - " + getJobId(), Collections.emptyList(), Collections.emptyList(), getJobId(), null, + false, token); flagStatsJobId = flagStatsJobResult.first().getId(); addEvent(Event.Type.INFO, "Submit job " + flagStatsJobId + " to compute stats (" + AlignmentFlagStatsAnalysis.ID + ")"); @@ -138,7 +138,8 @@ protected void run() throws ToolException { OpenCGAResult statsJobResult = catalogManager.getJobManager() .submit(study, AlignmentStatsAnalysis.ID, Enums.Priority.MEDIUM, params, null, "Job generated by " - + getId() + " - " + getJobId(), Collections.emptyList(), Collections.emptyList(), token); + + getId() + " - " + getJobId(), Collections.emptyList(), Collections.emptyList(), getJobId(), null, + false, token); statsJobId = statsJobResult.first().getId(); addEvent(Event.Type.INFO, "Submit job " + statsJobId + " to compute stats (" + AlignmentStatsAnalysis.ID + ")"); } @@ -155,7 +156,7 @@ protected void run() throws ToolException { OpenCGAResult fastQcMetricsJobResult = catalogManager.getJobManager() .submit(study, AlignmentFastQcMetricsAnalysis.ID, Enums.Priority.MEDIUM, params, null, "Job generated by " + getId() + " - " + getJobId(), Collections.emptyList(), Collections.emptyList(), - token); + getJobId(), null, false, token); fastQcMetricsJobId = fastQcMetricsJobResult.first().getId(); addEvent(Event.Type.INFO, "Submit job " + fastQcMetricsJobId + " to compute FastQC metrics (" + AlignmentFastQcMetricsAnalysis.ID + ")"); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/qc/AlignmentStatsAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/qc/AlignmentStatsAnalysis.java index d53da2f5299..c39a5a6f80e 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/qc/AlignmentStatsAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/qc/AlignmentStatsAnalysis.java @@ -18,9 +18,7 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; -import org.opencb.biodata.formats.alignment.samtools.SamtoolsFlagstats; import org.opencb.biodata.formats.alignment.samtools.SamtoolsStats; -import org.opencb.biodata.formats.alignment.samtools.io.SamtoolsFlagstatsParser; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.analysis.tools.OpenCgaToolScopeStudy; @@ -32,8 +30,6 @@ import org.opencb.opencga.core.models.alignment.AlignmentStatsParams; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.file.File; -import org.opencb.opencga.core.models.file.FileQualityControl; -import org.opencb.opencga.core.models.file.FileUpdateParams; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.tools.annotations.Tool; import org.opencb.opencga.core.tools.annotations.ToolParams; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/annotations/TsvAnnotationLoader.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/annotations/TsvAnnotationLoader.java index 8fba9b92654..8e912fdaaf9 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/annotations/TsvAnnotationLoader.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/annotations/TsvAnnotationLoader.java @@ -26,9 +26,11 @@ import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.FileManager; import org.opencb.opencga.catalog.utils.AnnotationUtils; +import org.opencb.opencga.catalog.utils.CatalogFqn; import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.exceptions.ToolException; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.common.AnnotationSet; import org.opencb.opencga.core.models.file.File; import org.opencb.opencga.core.models.sample.SamplePermissions; @@ -71,7 +73,9 @@ public void setStudy(String study) { @Override protected void check() throws Exception { - String userId = catalogManager.getUserManager().getUserId(token); + JwtPayload jwtPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(study, jwtPayload); + String userId = jwtPayload.getUserId(studyFqn.getOrganizationId()); OpenCGAResult fileResult = catalogManager.getFileManager().get(study, path, FileManager.INCLUDE_FILE_URI_PATH, token); if (fileResult.getNumResults() == 0) { diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/ClinicalAnalysisLoadTask.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/ClinicalAnalysisLoadTask.java new file mode 100644 index 00000000000..d099fc2f3a7 --- /dev/null +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/ClinicalAnalysisLoadTask.java @@ -0,0 +1,63 @@ +package org.opencb.opencga.analysis.clinical; + +import org.apache.commons.lang3.StringUtils; +import org.opencb.opencga.analysis.tools.OpenCgaToolScopeStudy; +import org.opencb.opencga.catalog.managers.FileManager; +import org.opencb.opencga.catalog.models.ClinicalAnalysisLoadResult; +import org.opencb.opencga.core.exceptions.ToolException; +import org.opencb.opencga.core.models.clinical.ClinicalAnalysisLoadParams; +import org.opencb.opencga.core.models.common.Enums; +import org.opencb.opencga.core.models.file.File; +import org.opencb.opencga.core.tools.annotations.Tool; +import org.opencb.opencga.core.tools.annotations.ToolParams; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Map; + +@Tool(id = ClinicalAnalysisLoadTask.ID, resource = Enums.Resource.CLINICAL_ANALYSIS, description = ClinicalAnalysisLoadTask.DESCRIPTION) +public class ClinicalAnalysisLoadTask extends OpenCgaToolScopeStudy { + public final static String ID = "load"; + public static final String DESCRIPTION = "Load clinical analyses from a file"; + + private Path filePath; + + @ToolParams + protected ClinicalAnalysisLoadParams params = new ClinicalAnalysisLoadParams(); + + @Override + protected void check() throws Exception { + super.check(); + + String fileStr = params.getFile(); + if (StringUtils.isEmpty(fileStr)) { + throw new ToolException("Missing input file when loading clinical analyses."); + } + + File file = catalogManager.getFileManager().get(getStudy(), fileStr, FileManager.INCLUDE_FILE_URI_PATH, token).first(); + filePath = Paths.get(file.getUri()); + if (!filePath.toFile().exists()) { + throw new ToolException("Input file '" + fileStr + "' does not exist: " + filePath); + } + } + + @Override + protected void run() throws Exception { + step(() -> { + ClinicalAnalysisLoadResult loadResult = catalogManager.getClinicalAnalysisManager().load(getStudy(), filePath, token); + + // Add results as attributes + addAttribute("Num. clinical analyses loaded", loadResult.getNumLoaded()); + addAttribute("Num. clinical analyses not loaded", loadResult.getFailures().size()); + addAttribute("Loading time (in sec.)", loadResult.getTime()); + addAttribute("Clinical analyses file name", loadResult.getFilename()); + + // Add warnings with the not loaded clinical analysis + if (loadResult.getFailures().size() > 0) { + for (Map.Entry entry : loadResult.getFailures().entrySet()) { + addWarning("Clinical analysis " + entry.getKey() + " could not be loaded due to error: " + entry.getValue()); + } + } + }); + } +} diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/ClinicalInterpretationManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/ClinicalInterpretationManager.java index f240d877fc5..d9c2e7c5734 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/ClinicalInterpretationManager.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/ClinicalInterpretationManager.java @@ -22,7 +22,10 @@ import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.StopWatch; -import org.opencb.biodata.models.clinical.*; +import org.opencb.biodata.models.clinical.ClinicalAcmg; +import org.opencb.biodata.models.clinical.ClinicalAnalyst; +import org.opencb.biodata.models.clinical.ClinicalProperty; +import org.opencb.biodata.models.clinical.Disorder; import org.opencb.biodata.models.clinical.interpretation.*; import org.opencb.biodata.models.clinical.interpretation.exceptions.InterpretationAnalysisException; import org.opencb.biodata.models.clinical.pedigree.Pedigree; @@ -35,8 +38,6 @@ import org.opencb.biodata.tools.clinical.ClinicalVariantCreator; import org.opencb.biodata.tools.clinical.DefaultClinicalVariantCreator; import org.opencb.biodata.tools.pedigree.ModeOfInheritance; -import org.opencb.commons.datastore.core.DataResult; -import org.opencb.commons.datastore.core.FacetField; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.utils.ListUtils; @@ -44,28 +45,27 @@ import org.opencb.opencga.analysis.alignment.AlignmentStorageManager; import org.opencb.opencga.analysis.variant.manager.VariantCatalogQueryUtils; import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; -import org.opencb.opencga.catalog.db.api.*; +import org.opencb.opencga.catalog.db.api.ProjectDBAdaptor; +import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; +import org.opencb.opencga.catalog.db.api.UserDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.CatalogManager; import org.opencb.opencga.catalog.managers.ClinicalAnalysisManager; import org.opencb.opencga.catalog.managers.FamilyManager; -import org.opencb.opencga.catalog.managers.StudyManager; +import org.opencb.opencga.catalog.utils.CatalogFqn; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.exceptions.ToolException; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.clinical.ClinicalAnalysis; import org.opencb.opencga.core.models.clinical.Interpretation; import org.opencb.opencga.core.models.individual.Individual; import org.opencb.opencga.core.models.project.Project; -import org.opencb.opencga.core.models.study.Group; import org.opencb.opencga.core.models.study.Study; import org.opencb.opencga.core.models.user.User; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.StorageEngineFactory; -import org.opencb.opencga.storage.core.clinical.ClinicalVariantEngine; -import org.opencb.opencga.storage.core.clinical.ClinicalVariantException; -import org.opencb.opencga.storage.core.clinical.ClinicalVariantIterator; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.metadata.models.SampleMetadata; import org.opencb.opencga.storage.core.metadata.models.TaskMetadata; @@ -78,7 +78,6 @@ import java.nio.file.Path; import java.util.*; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; import static org.opencb.biodata.models.clinical.interpretation.VariantClassification.calculateAcmgClassification; import static org.opencb.biodata.models.clinical.interpretation.VariantClassification.computeClinicalSignificance; @@ -94,7 +93,7 @@ public class ClinicalInterpretationManager extends StorageManager { private InterpretationAnalysisConfiguration config; private ClinicalAnalysisManager clinicalAnalysisManager; - private ClinicalVariantEngine clinicalVariantEngine; +// private ClinicalVariantEngine clinicalVariantEngine; private VariantStorageManager variantStorageManager; protected AlignmentStorageManager alignmentStorageManager; @@ -158,35 +157,35 @@ public ClinicalInterpretationManager(CatalogManager catalogManager, StorageEngin @Override public void testConnection() throws StorageEngineException { } - - public DataResult index(String token) { - return null; - } - - public DataResult index(String study, String token) throws IOException, ClinicalVariantException, CatalogException { - DBIterator clinicalAnalysisDBIterator = - clinicalAnalysisManager.iterator(study, new Query(), QueryOptions.empty(), token); - - while (clinicalAnalysisDBIterator.hasNext()) { - ClinicalAnalysis clinicalAnalysis = clinicalAnalysisDBIterator.next(); - if (clinicalAnalysis.getInterpretation() != null) { - clinicalAnalysis.getInterpretation().getAttributes().put("OPENCGA_CLINICAL_ANALYSIS", clinicalAnalysis); - - this.clinicalVariantEngine.insert(clinicalAnalysis.getInterpretation(), database); - - } - } - return null; - } - - public DataResult query(Query query, QueryOptions options, String token) - throws IOException, ClinicalVariantException, CatalogException { - // Check permissions - query = checkQueryPermissions(query, token); - - return clinicalVariantEngine.query(query, options, ""); - } - +// +// public DataResult index(String token) { +// return null; +// } +// +// public DataResult index(String study, String token) throws IOException, ClinicalVariantException, CatalogException { +// DBIterator clinicalAnalysisDBIterator = +// clinicalAnalysisManager.iterator(study, new Query(), QueryOptions.empty(), token); +// +// while (clinicalAnalysisDBIterator.hasNext()) { +// ClinicalAnalysis clinicalAnalysis = clinicalAnalysisDBIterator.next(); +// if (clinicalAnalysis.getInterpretation() != null) { +// clinicalAnalysis.getInterpretation().getAttributes().put("OPENCGA_CLINICAL_ANALYSIS", clinicalAnalysis); +// +// this.clinicalVariantEngine.insert(clinicalAnalysis.getInterpretation(), database); +// +// } +// } +// return null; +// } +// +// public DataResult query(Query query, QueryOptions options, String token) +// throws IOException, ClinicalVariantException, CatalogException { +// // Check permissions +// query = checkQueryPermissions(query, token); +// +// return clinicalVariantEngine.query(query, options, ""); +// } +// // public DataResult interpretationQuery(Query query, QueryOptions options, String token) // throws IOException, ClinicalVariantException, CatalogException { // // Check permissions @@ -194,38 +193,38 @@ public DataResult query(Query query, QueryOptions options, Stri // // return clinicalVariantEngine.interpretationQuery(query, options, ""); // } - - public DataResult facet(Query query, QueryOptions queryOptions, String token) - throws IOException, ClinicalVariantException, CatalogException { - // Check permissions - query = checkQueryPermissions(query, token); - - return clinicalVariantEngine.facet(query, queryOptions, ""); - } - - public ClinicalVariantIterator iterator(Query query, QueryOptions options, String token) - throws IOException, ClinicalVariantException, CatalogException { - // Check permissions - query = checkQueryPermissions(query, token); - - return clinicalVariantEngine.iterator(query, options, ""); - } - - public void addInterpretationComment(String study, long interpretationId, ClinicalComment comment, String token) - throws IOException, ClinicalVariantException, CatalogException { - // Check permissions - checkInterpretationPermissions(study, interpretationId, token); - - clinicalVariantEngine.addInterpretationComment(interpretationId, comment, ""); - } - - public void addClinicalVariantComment(String study, long interpretationId, String variantId, ClinicalComment comment, String token) - throws IOException, ClinicalVariantException, CatalogException { - // Check permissions - checkInterpretationPermissions(study, interpretationId, token); - - clinicalVariantEngine.addClinicalVariantComment(interpretationId, variantId, comment, ""); - } +// +// public DataResult facet(Query query, QueryOptions queryOptions, String token) +// throws IOException, ClinicalVariantException, CatalogException { +// // Check permissions +// query = checkQueryPermissions(query, token); +// +// return clinicalVariantEngine.facet(query, queryOptions, ""); +// } +// +// public ClinicalVariantIterator iterator(Query query, QueryOptions options, String token) +// throws IOException, ClinicalVariantException, CatalogException { +// // Check permissions +// query = checkQueryPermissions(query, token); +// +// return clinicalVariantEngine.iterator(query, options, ""); +// } +// +// public void addInterpretationComment(String study, long interpretationId, ClinicalComment comment, String token) +// throws IOException, ClinicalVariantException, CatalogException { +// // Check permissions +// checkInterpretationPermissions(study, interpretationId, token); +// +// clinicalVariantEngine.addInterpretationComment(interpretationId, comment, ""); +// } +// +// public void addClinicalVariantComment(String study, long interpretationId, String variantId, ClinicalComment comment, String token) +// throws IOException, ClinicalVariantException, CatalogException { +// // Check permissions +// checkInterpretationPermissions(study, interpretationId, token); +// +// clinicalVariantEngine.addClinicalVariantComment(interpretationId, variantId, comment, ""); +// } /*--------------------------------------------------------------------------*/ /* Get clinical variants */ @@ -818,10 +817,14 @@ public List getPrimaryFindings(String clinicalAnalysisId, Query return clinicalVariants; } - public ClinicalAnalyst getAnalyst(String token) throws ToolException { + public ClinicalAnalyst getAnalyst(String studyId, String token) throws ToolException { try { - String userId = catalogManager.getUserManager().getUserId(token); - OpenCGAResult userQueryResult = catalogManager.getUserManager().get(userId, new QueryOptions(QueryOptions.INCLUDE, + JwtPayload jwtPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, jwtPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = jwtPayload.getUserId(studyFqn.getOrganizationId()); + + OpenCGAResult userQueryResult = catalogManager.getUserManager().get(organizationId, userId, new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(UserDBAdaptor.QueryParams.EMAIL.key(), UserDBAdaptor.QueryParams.ORGANIZATION.key())), token); User user = userQueryResult.first(); return new ClinicalAnalyst(userId, user.getName(), user.getEmail(), "", Collections.emptyMap()); @@ -831,9 +834,13 @@ public ClinicalAnalyst getAnalyst(String token) throws ToolException { } public String getAssembly(String studyId, String sessionId) throws CatalogException { + JwtPayload jwtPayload = catalogManager.getUserManager().validateToken(sessionId); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, jwtPayload); + String organizationId = studyFqn.getOrganizationId(); + String assembly = ""; OpenCGAResult projectQueryResult; - projectQueryResult = catalogManager.getProjectManager().search(new Query(ProjectDBAdaptor.QueryParams.STUDY.key(), studyId), + projectQueryResult = catalogManager.getProjectManager().search(organizationId, new Query(ProjectDBAdaptor.QueryParams.STUDY.key(), studyId), new QueryOptions(QueryOptions.INCLUDE, ProjectDBAdaptor.QueryParams.ORGANISM.key()), sessionId); if (CollectionUtils.isNotEmpty(projectQueryResult.getResults())) { assembly = projectQueryResult.first().getOrganism().getAssembly(); @@ -852,140 +859,6 @@ public VariantStorageManager getVariantStorageManager() { /* P R I V A T E M E T H O D S */ /*--------------------------------------------------------------------------*/ - // FIXME Class path to a new section in storage-configuration.yml file - private void init() { - try { - this.database = catalogManager.getConfiguration().getDatabasePrefix() + "_clinical"; - - this.clinicalVariantEngine = getClinicalStorageEngine(); - } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) { - e.printStackTrace(); - } - - } - - private ClinicalVariantEngine getClinicalStorageEngine() throws ClassNotFoundException, IllegalAccessException, InstantiationException { - String clazz = this.storageConfiguration.getClinical().getManager(); - ClinicalVariantEngine storageEngine = (ClinicalVariantEngine) Class.forName(clazz).newInstance(); - storageEngine.setStorageConfiguration(this.storageConfiguration); - return storageEngine; - } - - private Query checkQueryPermissions(Query query, String token) throws ClinicalVariantException, CatalogException { - if (query == null) { - throw new ClinicalVariantException("Query object is null"); - } - - // Get userId from token and Study numeric IDs from the query - String userId = catalogManager.getUserManager().getUserId(token); - List studyIds = getStudyIds(userId, query); - - // If one specific clinical analysis, sample or individual is provided we expect a single valid study as well - if (isCaseProvided(query)) { - if (studyIds.size() == 1) { - // This checks that the user has permission to the clinical analysis, family, sample or individual - DataResult clinicalAnalysisQueryResult = catalogManager.getClinicalAnalysisManager() - .search(studyIds.get(0), query, QueryOptions.empty(), token); - - if (clinicalAnalysisQueryResult.getResults().isEmpty()) { - throw new ClinicalVariantException("Either the ID does not exist or the user does not have permissions to view it"); - } else { - if (!query.containsKey(ClinicalVariantEngine.QueryParams.CLINICAL_ANALYSIS_ID.key())) { - query.remove(ClinicalVariantEngine.QueryParams.FAMILY.key()); - query.remove(ClinicalVariantEngine.QueryParams.SAMPLE.key()); - query.remove(ClinicalVariantEngine.QueryParams.SUBJECT.key()); - String clinicalAnalysisList = StringUtils.join( - clinicalAnalysisQueryResult.getResults().stream().map(ClinicalAnalysis::getId).collect(Collectors.toList()), - ","); - query.put("clinicalAnalysisId", clinicalAnalysisList); - } - } - } else { - throw new ClinicalVariantException("No single valid study provided: " - + query.getString(ClinicalVariantEngine.QueryParams.STUDY.key())); - } - } else { - // Get the owner of all the studies - Set users = new HashSet<>(); - for (String studyFqn : studyIds) { - users.add(StringUtils.split(studyFqn, "@")[0]); - } - - // There must be one single owner for all the studies, we do nt allow to query multiple databases - if (users.size() == 1) { - Query studyQuery = new Query(StudyDBAdaptor.QueryParams.ID.key(), StringUtils.join(studyIds, ",")); - DataResult studyQueryResult = catalogManager.getStudyManager().search(studyQuery, QueryOptions.empty(), token); - - // If the user is the owner we do not have to check anything else - List studyAliases = new ArrayList<>(studyIds.size()); - if (users.contains(userId)) { - for (Study study : studyQueryResult.getResults()) { - studyAliases.add(study.getAlias()); - } - } else { - for (Study study : studyQueryResult.getResults()) { - for (Group group : study.getGroups()) { - if (group.getId().equalsIgnoreCase("@admins") && group.getUserIds().contains(userId)) { - studyAliases.add(study.getAlias()); - break; - } - } - } - } - - if (studyAliases.isEmpty()) { - throw new ClinicalVariantException("This user is not owner or admins for the provided studies"); - } else { - query.put(ClinicalVariantEngine.QueryParams.STUDY.key(), StringUtils.join(studyAliases, ",")); - } - } else { - throw new ClinicalVariantException(""); - } - } - return query; - } - - private void checkInterpretationPermissions(String study, long interpretationId, String token) - throws CatalogException, ClinicalVariantException { - // Get user ID from token and study numeric ID - String studyId = catalogManager.getStudyManager().get(study, StudyManager.INCLUDE_STUDY_IDS, token).first().getFqn(); - - // This checks that the user has permission to this interpretation - Query query = new Query(ClinicalAnalysisDBAdaptor.QueryParams.INTERPRETATION_ID.key(), interpretationId); - DataResult clinicalAnalysisQueryResult = catalogManager.getClinicalAnalysisManager() - .search(studyId, query, QueryOptions.empty(), token); - - if (clinicalAnalysisQueryResult.getResults().isEmpty()) { - throw new ClinicalVariantException("Either the interpretation ID (" + interpretationId + ") does not exist or the user does" - + " not have access permissions"); - } - } - - private List getStudyIds(String userId, Query query) throws CatalogException { - List studyIds = new ArrayList<>(); - - if (query != null && query.containsKey(ClinicalVariantEngine.QueryParams.STUDY.key())) { - String study = query.getString(ClinicalVariantEngine.QueryParams.STUDY.key()); - List studies = Arrays.asList(study.split(",")); - studyIds = catalogManager.getStudyManager().get(studies, StudyManager.INCLUDE_STUDY_IDS, false, userId) - .getResults() - .stream() - .map(Study::getFqn) - .collect(Collectors.toList()); - } - return studyIds; - } - - private boolean isCaseProvided(Query query) { - if (query != null) { - return query.containsKey(ClinicalVariantEngine.QueryParams.CLINICAL_ANALYSIS_ID.key()) - || query.containsKey(ClinicalVariantEngine.QueryParams.FAMILY.key()) - || query.containsKey(ClinicalVariantEngine.QueryParams.SUBJECT.key()) - || query.containsKey(ClinicalVariantEngine.QueryParams.SAMPLE.key()); - } - return false; - } - private void cleanQuery(Query query) { if (query.containsKey(VariantQueryParam.GENOTYPE.key())) { query.remove(VariantQueryParam.SAMPLE.key()); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/ClinicalTsvAnnotationLoader.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/ClinicalTsvAnnotationLoader.java index 1d2be95e82e..abdbca2e7b5 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/ClinicalTsvAnnotationLoader.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/ClinicalTsvAnnotationLoader.java @@ -32,7 +32,7 @@ import java.util.Collections; @Tool(id = ClinicalTsvAnnotationLoader.ID, resource = Enums.Resource.CLINICAL_ANALYSIS, type = Tool.Type.OPERATION, - description = "Load annotations from TSV file.") + description = "Load annotations from TSV file.", priority = Enums.Priority.HIGH) public class ClinicalTsvAnnotationLoader extends TsvAnnotationLoader { public final static String ID = "clinical-tsv-load"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/InterpretationAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/InterpretationAnalysis.java index 4fe57ad701b..81e4502625f 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/InterpretationAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/InterpretationAnalysis.java @@ -118,7 +118,7 @@ protected void saveInterpretation(String studyId, ClinicalAnalysis clinicalAnaly .setCommit(GitRepositoryState.getInstance().getCommitId()))); // Analyst - ClinicalAnalyst analyst = clinicalInterpretationManager.getAnalyst(token); + ClinicalAnalyst analyst = clinicalInterpretationManager.getAnalyst(studyId, token); List primaryFindings = readClinicalVariants(Paths.get(getOutDir().toString() + "/" + PRIMARY_FINDINGS_FILENAME)); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserClinicalVariantCreator.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserClinicalVariantCreator.java index 9d94df38540..595b7b0dfc1 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserClinicalVariantCreator.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserClinicalVariantCreator.java @@ -1,7 +1,7 @@ package org.opencb.opencga.analysis.clinical.exomiser; import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.opencb.biodata.models.clinical.ClinicalAcmg; import org.opencb.biodata.models.clinical.ClinicalDiscussion; import org.opencb.biodata.models.clinical.ClinicalProperty; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java index 55b291f156e..bbf685d1a03 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java @@ -30,8 +30,10 @@ import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.analysis.ConfigurationUtils; import org.opencb.opencga.analysis.clinical.InterpretationAnalysis; import org.opencb.opencga.analysis.individual.qc.IndividualQcUtils; +import org.opencb.opencga.analysis.wrappers.exomiser.ExomiserWrapperAnalysis; import org.opencb.opencga.analysis.wrappers.exomiser.ExomiserWrapperAnalysisExecutor; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.utils.ParamUtils; @@ -44,10 +46,10 @@ import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.individual.Individual; import org.opencb.opencga.core.response.OpenCGAResult; -import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.core.tools.annotations.Tool; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import java.io.BufferedReader; import java.io.File; @@ -69,6 +71,7 @@ public class ExomiserInterpretationAnalysis extends InterpretationAnalysis { private String clinicalAnalysisId; private String sampleId; private ClinicalAnalysis.Type clinicalAnalysisType; + private String exomiserVersion; private ClinicalAnalysis clinicalAnalysis; @@ -97,8 +100,7 @@ protected void check() throws Exception { try { clinicalAnalysisQueryResult = catalogManager.getClinicalAnalysisManager().get(studyId, clinicalAnalysisId, QueryOptions.empty(), token); - } catch ( - CatalogException e) { + } catch (CatalogException e) { throw new ToolException(e); } if (clinicalAnalysisQueryResult.getNumResults() != 1) { @@ -117,6 +119,7 @@ protected void check() throws Exception { } sampleId = clinicalAnalysis.getProband().getSamples().get(0).getId(); + // Check clinical analysis type if (clinicalAnalysis.getType() == ClinicalAnalysis.Type.FAMILY) { clinicalAnalysisType = ClinicalAnalysis.Type.FAMILY; } else { @@ -125,6 +128,13 @@ protected void check() throws Exception { logger.info("The clinical analysis type is {}, so the Exomiser will be run in mode {}", clinicalAnalysis.getType(), clinicalAnalysisType); + // Check exomiser version + if (StringUtils.isEmpty(exomiserVersion)) { + // Missing exomiser version use the default one + exomiserVersion = ConfigurationUtils.getToolDefaultVersion(ExomiserWrapperAnalysis.ID, configuration); + logger.warn("Missing exomiser version, using the default {}", exomiserVersion); + } + // Update executor params with OpenCGA home and session ID setUpStorageEngineExecutor(studyId); } @@ -134,28 +144,31 @@ protected void run() throws ToolException { step(() -> { executorParams.put(EXECUTOR_ID, ExomiserWrapperAnalysisExecutor.ID); - getToolExecutor(ExomiserWrapperAnalysisExecutor.class) + ExomiserWrapperAnalysisExecutor exomiserExecutor = getToolExecutor(ExomiserWrapperAnalysisExecutor.class) .setStudyId(studyId) .setSampleId(sampleId) .setClinicalAnalysisType(clinicalAnalysisType) - .execute(); + .setExomiserVersion(exomiserVersion); + + exomiserExecutor.execute(); - saveInterpretation(studyId, clinicalAnalysis); + saveInterpretation(studyId, clinicalAnalysis, exomiserExecutor.getDockerImageName(), exomiserExecutor.getDockerImageVersion()); }); } - protected void saveInterpretation(String studyId, ClinicalAnalysis clinicalAnalysis) throws ToolException, StorageEngineException, + protected void saveInterpretation(String studyId, ClinicalAnalysis clinicalAnalysis, String dockerImage, String dockerImageVersion) + throws ToolException, StorageEngineException, CatalogException, IOException { // Interpretation method InterpretationMethod method = new InterpretationMethod(getId(), GitRepositoryState.getInstance().getBuildVersion(), GitRepositoryState.getInstance().getCommitId(), Collections.singletonList( new Software() .setName("Exomiser") - .setRepository("Docker: " + ExomiserWrapperAnalysisExecutor.DOCKER_IMAGE_NAME) - .setVersion(ExomiserWrapperAnalysisExecutor.DOCKER_IMAGE_VERSION))); + .setRepository("Docker: " + dockerImage) + .setVersion(dockerImageVersion))); // Analyst - ClinicalAnalyst analyst = clinicalInterpretationManager.getAnalyst(token); + ClinicalAnalyst analyst = clinicalInterpretationManager.getAnalyst(studyId, token); List primaryFindings = getPrimaryFindings(); @@ -274,8 +287,17 @@ private List getPrimaryFindings() throws IOException, StorageEn // Convert variants to clinical variants for (Variant variant : variantResults.getResults()) { ClinicalVariant clinicalVariant = clinicalVariantCreator.create(variant); - List exomiserTranscripts = new ArrayList<>(variantTranscriptMap.get(normalizedToTsv - .get(variant.toStringSimple()))); + List exomiserTranscripts = new ArrayList<>(); + if (normalizedToTsv.containsKey(variant.toStringSimple())) { + if (variantTranscriptMap.containsKey(normalizedToTsv.get(variant.toStringSimple()))) { + exomiserTranscripts.addAll(variantTranscriptMap.get(normalizedToTsv.get(variant.toStringSimple()))); + } else { + logger.warn("Variant {} (normalizedToTsv {}), not found in map variantTranscriptMap", variant.toStringSimple(), + normalizedToTsv.get(variant.toStringSimple())); + } + } else { + logger.warn("Variant {} not found in map normalizedToTsv", variant.toStringSimple()); + } for (String[] fields : variantTsvMap.get(variant.toStringSimple())) { ClinicalProperty.ModeOfInheritance moi = getModeOfInheritance(fields[4]); Map attributes = getAttributesFromTsv(fields); @@ -463,4 +485,13 @@ public ExomiserInterpretationAnalysis setClinicalAnalysisId(String clinicalAnaly this.clinicalAnalysisId = clinicalAnalysisId; return this; } + + public String getExomiserVersion() { + return exomiserVersion; + } + + public ExomiserInterpretationAnalysis setExomiserVersion(String exomiserVersion) { + this.exomiserVersion = exomiserVersion; + return this; + } } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/rga/AuxiliarRgaAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/rga/AuxiliarRgaAnalysis.java index 37823aac657..d70f3f3d244 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/rga/AuxiliarRgaAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/rga/AuxiliarRgaAnalysis.java @@ -7,7 +7,7 @@ import org.opencb.opencga.core.tools.annotations.Tool; @Tool(id = AuxiliarRgaAnalysis.ID, resource = Enums.Resource.RGA, type = Tool.Type.OPERATION, - description = AuxiliarRgaAnalysis.DESCRIPTION) + description = AuxiliarRgaAnalysis.DESCRIPTION, priority = Enums.Priority.HIGH) public class AuxiliarRgaAnalysis extends OperationTool { public final static String ID = "rga-aux-index"; public final static String DESCRIPTION = ParamConstants.INDEX_AUXILIAR_COLLECTION_DESCRIPTION; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/rga/RgaAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/rga/RgaAnalysis.java index 0e2a56e644d..132b3d72464 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/rga/RgaAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/rga/RgaAnalysis.java @@ -6,7 +6,8 @@ import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.tools.annotations.Tool; -@Tool(id = RgaAnalysis.ID, resource = Enums.Resource.RGA, type = Tool.Type.OPERATION, description = "Index RGA study.") +@Tool(id = RgaAnalysis.ID, resource = Enums.Resource.RGA, type = Tool.Type.OPERATION, description = "Index RGA study.", + priority = Enums.Priority.HIGH) public class RgaAnalysis extends OperationTool { public final static String ID = "rga-index"; public final static String DESCRIPTION = "Generate Recessive Gene Analysis secondary index"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/cohort/CohortIndexTask.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/cohort/CohortIndexTask.java deleted file mode 100644 index c5b16bf144b..00000000000 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/cohort/CohortIndexTask.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright 2015-2020 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.analysis.cohort; - -import org.opencb.commons.datastore.core.Query; -import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.analysis.tools.OpenCgaTool; -import org.opencb.opencga.catalog.db.api.CohortDBAdaptor; -import org.opencb.opencga.catalog.db.api.DBAdaptor; -import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; -import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.stats.solr.CatalogSolrManager; -import org.opencb.opencga.catalog.stats.solr.converters.CatalogCohortToSolrCohortConverter; -import org.opencb.opencga.catalog.stats.solr.converters.SolrConverterUtil; -import org.opencb.opencga.core.api.ParamConstants; -import org.opencb.opencga.core.models.common.Enums; -import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.response.OpenCGAResult; -import org.opencb.opencga.core.tools.annotations.Tool; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Set; - -@Tool(id = CohortIndexTask.ID, resource = Enums.Resource.FILE, type = Tool.Type.OPERATION, description = "Index Cohort entries in Solr.") -public class CohortIndexTask extends OpenCgaTool { - - public final static String ID = "cohort-secondary-index"; - - private CatalogSolrManager catalogSolrManager; - - @Override - protected void check() throws Exception { - catalogSolrManager = new CatalogSolrManager(this.catalogManager); - } - - @Override - protected void run() throws Exception { - // Get all the studies - Query query = new Query(); - QueryOptions options = new QueryOptions() - .append(QueryOptions.INCLUDE, Arrays.asList(StudyDBAdaptor.QueryParams.UID.key(), - StudyDBAdaptor.QueryParams.ID.key(), StudyDBAdaptor.QueryParams.FQN.key(), - StudyDBAdaptor.QueryParams.VARIABLE_SET.key())) - .append(DBAdaptor.INCLUDE_ACLS, true); - OpenCGAResult studyDataResult = catalogManager.getStudyManager().search(query, options, token); - if (studyDataResult.getNumResults() == 0) { - throw new CatalogException("Could not index catalog into solr. No studies found"); - } - - // Create solr collections if they don't exist - catalogSolrManager.createSolrCollections(CatalogSolrManager.COHORT_SOLR_COLLECTION); - - for (Study study : studyDataResult.getResults()) { - Map> studyAcls = SolrConverterUtil - .parseInternalOpenCGAAcls((List>) study.getAttributes().get("OPENCGA_ACL")); - // We replace the current studyAcls for the parsed one - study.getAttributes().put("OPENCGA_ACL", studyAcls); - - indexCohort(catalogSolrManager, study); - } - - } - - private void indexCohort(CatalogSolrManager catalogSolrManager, Study study) throws CatalogException { - logger.info("Indexing cohorts of study {}", study.getFqn()); - - Query query = new Query(); - QueryOptions cohortQueryOptions = new QueryOptions() - .append(QueryOptions.INCLUDE, Arrays.asList(CohortDBAdaptor.QueryParams.UUID.key(), - CohortDBAdaptor.QueryParams.ID.key(), - CohortDBAdaptor.QueryParams.CREATION_DATE.key(), CohortDBAdaptor.QueryParams.INTERNAL_STATUS.key(), - CohortDBAdaptor.QueryParams.RELEASE.key(), CohortDBAdaptor.QueryParams.ANNOTATION_SETS.key(), - CohortDBAdaptor.QueryParams.SAMPLE_UIDS.key(), CohortDBAdaptor.QueryParams.TYPE.key())) - .append(DBAdaptor.INCLUDE_ACLS, true) - .append(ParamConstants.FLATTEN_ANNOTATIONS, true); - - catalogSolrManager.insertCatalogCollection(catalogManager.getCohortManager().iterator(study.getFqn(), query, - cohortQueryOptions, token), new CatalogCohortToSolrCohortConverter(study), CatalogSolrManager.COHORT_SOLR_COLLECTION); - } -} diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/cohort/CohortTsvAnnotationLoader.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/cohort/CohortTsvAnnotationLoader.java index 82b8c9e1bb4..7dcdc62c8fc 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/cohort/CohortTsvAnnotationLoader.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/cohort/CohortTsvAnnotationLoader.java @@ -25,7 +25,7 @@ import org.opencb.opencga.core.tools.annotations.Tool; @Tool(id = CohortTsvAnnotationLoader.ID, resource = Enums.Resource.COHORT, type = Tool.Type.OPERATION, - description = "Load annotations from TSV file.") + description = "Load annotations from TSV file.", priority = Enums.Priority.HIGH) public class CohortTsvAnnotationLoader extends TsvAnnotationLoader { public final static String ID = "cohort-tsv-load"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/FamilyIndexTask.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/FamilyIndexTask.java deleted file mode 100644 index 7f4d76b1fa4..00000000000 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/FamilyIndexTask.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright 2015-2020 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.analysis.family; - -import org.opencb.commons.datastore.core.Query; -import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.analysis.tools.OpenCgaTool; -import org.opencb.opencga.catalog.db.api.DBAdaptor; -import org.opencb.opencga.catalog.db.api.FamilyDBAdaptor; -import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; -import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.stats.solr.CatalogSolrManager; -import org.opencb.opencga.catalog.stats.solr.converters.CatalogFamilyToSolrFamilyConverter; -import org.opencb.opencga.catalog.stats.solr.converters.SolrConverterUtil; -import org.opencb.opencga.core.api.ParamConstants; -import org.opencb.opencga.core.models.common.Enums; -import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.response.OpenCGAResult; -import org.opencb.opencga.core.tools.annotations.Tool; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Set; - -@Tool(id = FamilyIndexTask.ID, resource = Enums.Resource.FILE, type = Tool.Type.OPERATION, description = "Index Family entries in Solr.") -public class FamilyIndexTask extends OpenCgaTool { - - public final static String ID = "family-secondary-index"; - - private CatalogSolrManager catalogSolrManager; - - @Override - protected void check() throws Exception { - catalogSolrManager = new CatalogSolrManager(this.catalogManager); - } - - @Override - protected void run() throws Exception { - // Get all the studies - Query query = new Query(); - QueryOptions options = new QueryOptions() - .append(QueryOptions.INCLUDE, Arrays.asList(StudyDBAdaptor.QueryParams.UID.key(), - StudyDBAdaptor.QueryParams.ID.key(), StudyDBAdaptor.QueryParams.FQN.key(), - StudyDBAdaptor.QueryParams.VARIABLE_SET.key())) - .append(DBAdaptor.INCLUDE_ACLS, true); - OpenCGAResult studyDataResult = catalogManager.getStudyManager().search(query, options, token); - if (studyDataResult.getNumResults() == 0) { - throw new CatalogException("Could not index catalog into solr. No studies found"); - } - - // Create solr collections if they don't exist - catalogSolrManager.createSolrCollections(CatalogSolrManager.FAMILY_SOLR_COLLECTION); - - for (Study study : studyDataResult.getResults()) { - Map> studyAcls = SolrConverterUtil - .parseInternalOpenCGAAcls((List>) study.getAttributes().get("OPENCGA_ACL")); - // We replace the current studyAcls for the parsed one - study.getAttributes().put("OPENCGA_ACL", studyAcls); - - indexFamily(catalogSolrManager, study); - } - } - - private void indexFamily(CatalogSolrManager catalogSolrManager, Study study) throws CatalogException { - logger.info("Indexing families of study {}", study.getFqn()); - - Query query = new Query(); - QueryOptions familyQueryOptions = new QueryOptions() - .append(QueryOptions.INCLUDE, Arrays.asList(FamilyDBAdaptor.QueryParams.UUID.key(), - FamilyDBAdaptor.QueryParams.CREATION_DATE.key(), FamilyDBAdaptor.QueryParams.INTERNAL_STATUS.key(), - FamilyDBAdaptor.QueryParams.MEMBER_UID.key(), FamilyDBAdaptor.QueryParams.RELEASE.key(), - FamilyDBAdaptor.QueryParams.VERSION.key(), FamilyDBAdaptor.QueryParams.ANNOTATION_SETS.key(), - FamilyDBAdaptor.QueryParams.PHENOTYPES.key(), FamilyDBAdaptor.QueryParams.EXPECTED_SIZE.key())) - .append(DBAdaptor.INCLUDE_ACLS, true) - .append(ParamConstants.FLATTEN_ANNOTATIONS, true); - - catalogSolrManager.insertCatalogCollection(catalogManager.getFamilyManager().iterator(study.getFqn(), query, - familyQueryOptions, token), new CatalogFamilyToSolrFamilyConverter(study), CatalogSolrManager.FAMILY_SOLR_COLLECTION); - } -} - diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/FamilyTsvAnnotationLoader.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/FamilyTsvAnnotationLoader.java index 6d4091fdce3..68efcc010b5 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/FamilyTsvAnnotationLoader.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/FamilyTsvAnnotationLoader.java @@ -25,7 +25,7 @@ import org.opencb.opencga.core.tools.annotations.Tool; @Tool(id = FamilyTsvAnnotationLoader.ID, resource = Enums.Resource.FAMILY, type = Tool.Type.OPERATION, - description = "Load annotations from TSV file.") + description = "Load annotations from TSV file.", priority = Enums.Priority.HIGH) public class FamilyTsvAnnotationLoader extends TsvAnnotationLoader { public final static String ID = "family-tsv-load"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/PedigreeGraphAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/PedigreeGraphAnalysis.java index 3d49b12c7fb..2e99c99c2fd 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/PedigreeGraphAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/PedigreeGraphAnalysis.java @@ -57,7 +57,7 @@ protected void check() throws Exception { OpenCGAResult familyResult = catalogManager.getFamilyManager().get(study, pedigreeParams.getFamilyId(), QueryOptions.empty(), token); if (familyResult.getNumResults() != 1) { - throw new ToolException("Unable to compute the pedigree graph imae. Family '" + pedigreeParams.getFamilyId() + "' not found"); + throw new ToolException("Unable to compute the pedigree graph image. Family '" + pedigreeParams.getFamilyId() + "' not found"); } family = familyResult.first(); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/FamilyQcAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/FamilyQcAnalysis.java index 54b3c971da2..febc214e9e7 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/FamilyQcAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/FamilyQcAnalysis.java @@ -23,7 +23,10 @@ import org.opencb.opencga.analysis.tools.OpenCgaTool; import org.opencb.opencga.analysis.variant.relatedness.RelatednessAnalysis; import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.utils.CatalogFqn; + import org.opencb.opencga.core.exceptions.ToolException; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.family.Family; import org.opencb.opencga.core.models.family.FamilyQualityControl; @@ -67,9 +70,13 @@ protected void check() throws Exception { // Check permissions try { + JwtPayload jwtPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, jwtPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = jwtPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().get(studyId, QueryOptions.empty(), token).first(); - String userId = catalogManager.getUserManager().getUserId(token); - catalogManager.getAuthorizationManager().checkStudyPermission(study.getUid(), userId, WRITE_FAMILIES); + catalogManager.getAuthorizationManager().checkStudyPermission(organizationId, study.getUid(), userId, WRITE_FAMILIES); } catch (CatalogException e) { throw new ToolException(e); } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/IBDComputation.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/IBDComputation.java index 4ae4a565214..7277c3d2429 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/IBDComputation.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/IBDComputation.java @@ -16,12 +16,10 @@ package org.opencb.opencga.analysis.family.qc; -import com.fasterxml.jackson.core.JsonProcessingException; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; import org.apache.commons.io.FileUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.commons.math3.ode.nonstiff.RungeKuttaFieldIntegrator; +import org.apache.commons.lang3.StringUtils; import org.opencb.biodata.models.clinical.qc.RelatednessReport; import org.opencb.biodata.models.clinical.qc.RelatednessScore; import org.opencb.biodata.models.variant.avro.VariantType; @@ -34,19 +32,16 @@ import org.opencb.opencga.analysis.variant.relatedness.RelatednessAnalysis; import org.opencb.opencga.analysis.wrappers.plink.PlinkWrapperAnalysisExecutor; import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.family.Family; import org.opencb.opencga.core.models.individual.Individual; import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; -import org.opencb.opencga.storage.core.variant.query.VariantQueryUtils; import java.io.*; import java.net.URL; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.*; import java.util.stream.Collectors; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FetchAndRegisterTask.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FetchAndRegisterTask.java index e400585f536..3dcd8079060 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FetchAndRegisterTask.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FetchAndRegisterTask.java @@ -22,8 +22,10 @@ import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogIOException; import org.opencb.opencga.catalog.managers.StudyManager; +import org.opencb.opencga.catalog.utils.CatalogFqn; import org.opencb.opencga.core.common.UriUtils; import org.opencb.opencga.core.exceptions.ToolException; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.file.File; import org.opencb.opencga.core.models.file.FileFetch; @@ -44,7 +46,7 @@ import java.util.List; @Tool(id = FetchAndRegisterTask.ID, resource = Enums.Resource.FILE, type = Tool.Type.OPERATION, - description = "Download an external file and register it in OpenCGA.") + description = "Download an external file and register it in OpenCGA.", priority = Enums.Priority.HIGH) public class FetchAndRegisterTask extends OpenCgaToolScopeStudy { public final static String ID = "files-fetch"; @@ -73,9 +75,11 @@ protected void check() throws Exception { } try { + JwtPayload jwtPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn catalogFqn= CatalogFqn.extractFqnFromStudy(studyFqn, jwtPayload); Study study = catalogManager.getStudyManager().get(studyFqn, StudyManager.INCLUDE_STUDY_IDS, token).first(); - OpenCGAResult parents = catalogManager.getFileManager().getParents(studyFqn, toolParams.getPath(), false, + OpenCGAResult parents = catalogManager.getFileManager().getParents(study.getFqn(), toolParams.getPath(), false, QueryOptions.empty(), token); if (parents.getNumResults() == 0) { throw new ToolException("No parent folders found for " + toolParams.getPath()); @@ -85,10 +89,10 @@ protected void check() throws Exception { throw new CatalogException("Parent path " + parents.first().getPath() + " is external. Cannot download to mounted folders"); } - String userId = catalogManager.getUserManager().getUserId(token); + String userId = jwtPayload.getUserId(catalogFqn.getOrganizationId()); // Check write permissions over the path - catalogManager.getAuthorizationManager() - .checkFilePermission(study.getUid(), parents.first().getUid(), userId, FilePermissions.WRITE); + catalogManager.getAuthorizationManager().checkFilePermission(catalogFqn.getOrganizationId(), study.getUid(), + parents.first().getUid(), userId, FilePermissions.WRITE); } catch (CatalogException e) { throw new ToolException(e); } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FileDeleteTask.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FileDeleteTask.java index 89edaa6e683..2750435e049 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FileDeleteTask.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FileDeleteTask.java @@ -42,7 +42,8 @@ import java.io.IOException; import java.util.*; -@Tool(id = FileDeleteTask.ID, resource = Enums.Resource.FILE, type = Tool.Type.OPERATION, description = "Delete files.") +@Tool(id = FileDeleteTask.ID, resource = Enums.Resource.FILE, type = Tool.Type.OPERATION, description = "Delete files.", + priority = Enums.Priority.HIGH) public class FileDeleteTask extends OpenCgaTool { public final static String ID = "files-delete"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FileIndexTask.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FileIndexTask.java deleted file mode 100644 index 50728337279..00000000000 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FileIndexTask.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright 2015-2020 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.analysis.file; - -import org.opencb.commons.datastore.core.Query; -import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.analysis.tools.OpenCgaTool; -import org.opencb.opencga.catalog.db.api.DBAdaptor; -import org.opencb.opencga.catalog.db.api.FileDBAdaptor; -import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; -import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.stats.solr.CatalogSolrManager; -import org.opencb.opencga.catalog.stats.solr.converters.CatalogFileToSolrFileConverter; -import org.opencb.opencga.catalog.stats.solr.converters.SolrConverterUtil; -import org.opencb.opencga.core.api.ParamConstants; -import org.opencb.opencga.core.models.common.Enums; -import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.response.OpenCGAResult; -import org.opencb.opencga.core.tools.annotations.Tool; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Set; - -@Tool(id = FileIndexTask.ID, resource = Enums.Resource.FILE, type = Tool.Type.OPERATION, description = "Index File entries in Solr.") -public class FileIndexTask extends OpenCgaTool { - - public final static String ID = "file-secondary-index"; - - private CatalogSolrManager catalogSolrManager; - - @Override - protected void check() throws Exception { - catalogSolrManager = new CatalogSolrManager(this.catalogManager); -// catalogSolrManager.existsCollection() -// if (!catalogSolrManager.isAlive(configuration.getDatabasePrefix() + CatalogSolrManager.FILE_SOLR_COLLECTION)) { -// throw new ToolException("Solr not found"); -// } - } - - @Override - protected void run() throws Exception { - // Get all the studies - Query query = new Query(); - QueryOptions options = new QueryOptions() - .append(QueryOptions.INCLUDE, Arrays.asList(StudyDBAdaptor.QueryParams.UID.key(), - StudyDBAdaptor.QueryParams.ID.key(), StudyDBAdaptor.QueryParams.FQN.key(), - StudyDBAdaptor.QueryParams.VARIABLE_SET.key())) - .append(DBAdaptor.INCLUDE_ACLS, true); - OpenCGAResult studyDataResult = catalogManager.getStudyManager().search(query, options, token); - if (studyDataResult.getNumResults() == 0) { - throw new CatalogException("Could not index catalog into solr. No studies found"); - } - - // Create solr collections if they don't exist - catalogSolrManager.createSolrCollections(CatalogSolrManager.FILE_SOLR_COLLECTION); - - for (Study study : studyDataResult.getResults()) { - Map> studyAcls = SolrConverterUtil - .parseInternalOpenCGAAcls((List>) study.getAttributes().get("OPENCGA_ACL")); - // We replace the current studyAcls for the parsed one - study.getAttributes().put("OPENCGA_ACL", studyAcls); - - indexFile(catalogSolrManager, study); - } - - } - - private void indexFile(CatalogSolrManager catalogSolrManager, Study study) throws CatalogException { - logger.info("Indexing files of study {}", study.getFqn()); - - Query query = new Query(); - QueryOptions fileQueryOptions = new QueryOptions() - .append(QueryOptions.INCLUDE, Arrays.asList(FileDBAdaptor.QueryParams.UUID.key(), - FileDBAdaptor.QueryParams.NAME.key(), FileDBAdaptor.QueryParams.TYPE.key(), - FileDBAdaptor.QueryParams.FORMAT.key(), - FileDBAdaptor.QueryParams.CREATION_DATE.key(), FileDBAdaptor.QueryParams.BIOFORMAT.key(), - FileDBAdaptor.QueryParams.RELEASE.key(), FileDBAdaptor.QueryParams.INTERNAL_STATUS.key(), - FileDBAdaptor.QueryParams.EXTERNAL.key(), FileDBAdaptor.QueryParams.SIZE.key(), - FileDBAdaptor.QueryParams.SOFTWARE.key(), FileDBAdaptor.QueryParams.EXPERIMENT.key(), - FileDBAdaptor.QueryParams.RELATED_FILES.key(), FileDBAdaptor.QueryParams.SAMPLE_IDS.key(), - FileDBAdaptor.QueryParams.ANNOTATION_SETS.key())) - .append(DBAdaptor.INCLUDE_ACLS, true) - .append(ParamConstants.FLATTEN_ANNOTATIONS, true); - - catalogSolrManager.insertCatalogCollection(catalogManager.getFileManager().iterator(study.getFqn(), query, - fileQueryOptions, token), new CatalogFileToSolrFileConverter(study), CatalogSolrManager.FILE_SOLR_COLLECTION); - } -} diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FileLinkTask.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FileLinkTask.java index d6506bb6d81..88cb038215a 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FileLinkTask.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FileLinkTask.java @@ -18,7 +18,8 @@ import java.util.List; import java.util.Map; -@Tool(id = FileLinkTask.ID, resource = Enums.Resource.FILE, type = Tool.Type.OPERATION, description = FileLinkTask.DESCRIPTION) +@Tool(id = FileLinkTask.ID, resource = Enums.Resource.FILE, type = Tool.Type.OPERATION, description = FileLinkTask.DESCRIPTION, + priority = Enums.Priority.HIGH) public class FileLinkTask extends OpenCgaToolScopeStudy { public static final String ID = "file-link"; @@ -69,7 +70,7 @@ protected void run() throws Exception { Job postLinkJob = catalogManager.getJobManager() .submit(getStudy(), PostLinkSampleAssociation.ID, Enums.Priority.MEDIUM, params, null, "Job generated by " + getId() + " - " + getJobId(), Collections.emptyList(), Collections.emptyList(), - getToken()).first(); + getJobId(), null, false, getToken()).first(); logger.info("Submit post-link job : " + postLinkJob.getId()); } } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FileTsvAnnotationLoader.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FileTsvAnnotationLoader.java index ddd77ca22af..f52871a7b6e 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FileTsvAnnotationLoader.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FileTsvAnnotationLoader.java @@ -32,7 +32,7 @@ import java.util.Collections; @Tool(id = FileTsvAnnotationLoader.ID, resource = Enums.Resource.FILE, type = Tool.Type.OPERATION, - description = "Load annotations from TSV file.") + description = "Load annotations from TSV file.", priority = Enums.Priority.HIGH) public class FileTsvAnnotationLoader extends TsvAnnotationLoader { public final static String ID = "file-tsv-load"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FileUnlinkTask.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FileUnlinkTask.java index 4340046ff50..24a45fe40b9 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FileUnlinkTask.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FileUnlinkTask.java @@ -38,7 +38,8 @@ import java.util.*; -@Tool(id = FileUnlinkTask.ID, resource = Enums.Resource.FILE, type = Tool.Type.OPERATION, description = "Unlink files.") +@Tool(id = FileUnlinkTask.ID, resource = Enums.Resource.FILE, type = Tool.Type.OPERATION, description = "Unlink files.", + priority = Enums.Priority.HIGH) public class FileUnlinkTask extends OpenCgaTool { public final static String ID = "files-unlink"; @@ -183,8 +184,7 @@ private void restore(Query query, FileUpdateParams updateParams, QueryOptions op } private void addCriticalError(CatalogException e) { - CatalogException exception = new CatalogException("Critical: Could not restore status of pending files to " - + FileStatus.READY, e); + CatalogException exception = new CatalogException("Critical: Could not restore status of pending files to " + FileStatus.READY, e); logger.error("{}", e.getMessage(), e); try { addError(exception); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/PostLinkSampleAssociation.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/PostLinkSampleAssociation.java index 60d67f05996..d624d2e4828 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/PostLinkSampleAssociation.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/PostLinkSampleAssociation.java @@ -24,7 +24,7 @@ import java.util.*; @Tool(id = PostLinkSampleAssociation.ID, resource = Enums.Resource.FILE, type = Tool.Type.OPERATION, - description = PostLinkSampleAssociation.DESCRIPTION) + description = PostLinkSampleAssociation.DESCRIPTION, priority = Enums.Priority.HIGH) public class PostLinkSampleAssociation extends OpenCgaToolScopeStudy { public static final String ID = "postlink"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/individual/IndividualIndexTask.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/individual/IndividualIndexTask.java deleted file mode 100644 index 48ddf3ae512..00000000000 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/individual/IndividualIndexTask.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright 2015-2020 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.analysis.individual; - -import org.opencb.commons.datastore.core.Query; -import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.analysis.tools.OpenCgaTool; -import org.opencb.opencga.catalog.db.api.DBAdaptor; -import org.opencb.opencga.catalog.db.api.IndividualDBAdaptor; -import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; -import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.stats.solr.CatalogSolrManager; -import org.opencb.opencga.catalog.stats.solr.converters.CatalogIndividualToSolrIndividualConverter; -import org.opencb.opencga.catalog.stats.solr.converters.SolrConverterUtil; -import org.opencb.opencga.core.api.ParamConstants; -import org.opencb.opencga.core.models.common.Enums; -import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.response.OpenCGAResult; -import org.opencb.opencga.core.tools.annotations.Tool; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Set; - -@Tool(id = IndividualIndexTask.ID, resource = Enums.Resource.FILE, type = Tool.Type.OPERATION, - description = "Index Individual entries in Solr.") -public class IndividualIndexTask extends OpenCgaTool { - - public final static String ID = "individual-secondary-index"; - - private CatalogSolrManager catalogSolrManager; - - @Override - protected void check() throws Exception { - catalogSolrManager = new CatalogSolrManager(this.catalogManager); - } - - @Override - protected void run() throws Exception { - // Get all the studies - Query query = new Query(); - QueryOptions options = new QueryOptions() - .append(QueryOptions.INCLUDE, Arrays.asList(StudyDBAdaptor.QueryParams.UID.key(), - StudyDBAdaptor.QueryParams.ID.key(), StudyDBAdaptor.QueryParams.FQN.key(), - StudyDBAdaptor.QueryParams.VARIABLE_SET.key())) - .append(DBAdaptor.INCLUDE_ACLS, true); - OpenCGAResult studyDataResult = catalogManager.getStudyManager().search(query, options, token); - if (studyDataResult.getNumResults() == 0) { - throw new CatalogException("Could not index catalog into solr. No studies found"); - } - - // Create solr collections if they don't exist - catalogSolrManager.createSolrCollections(CatalogSolrManager.INDIVIDUAL_SOLR_COLLECTION); - - for (Study study : studyDataResult.getResults()) { - Map> studyAcls = SolrConverterUtil - .parseInternalOpenCGAAcls((List>) study.getAttributes().get("OPENCGA_ACL")); - // We replace the current studyAcls for the parsed one - study.getAttributes().put("OPENCGA_ACL", studyAcls); - - indexIndividual(catalogSolrManager, study); - } - - } - - private void indexIndividual(CatalogSolrManager catalogSolrManager, Study study) throws CatalogException { - logger.info("Indexing individuals of study {}", study.getFqn()); - - Query query = new Query() - .append(IndividualDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - QueryOptions individualQueryOptions = new QueryOptions() - .append(QueryOptions.INCLUDE, Arrays.asList(IndividualDBAdaptor.QueryParams.UUID.key(), - IndividualDBAdaptor.QueryParams.FATHER_UID.key(), IndividualDBAdaptor.QueryParams.MOTHER_UID.key(), - IndividualDBAdaptor.QueryParams.SEX_ID.key(), - IndividualDBAdaptor.QueryParams.ETHNICITY_ID.key(), IndividualDBAdaptor.QueryParams.POPULATION_NAME.key(), - IndividualDBAdaptor.QueryParams.RELEASE.key(), IndividualDBAdaptor.QueryParams.CREATION_DATE.key(), - IndividualDBAdaptor.QueryParams.VERSION.key(), - IndividualDBAdaptor.QueryParams.INTERNAL_STATUS.key(), IndividualDBAdaptor.QueryParams.LIFE_STATUS.key(), - IndividualDBAdaptor.QueryParams.PHENOTYPES.key(), - IndividualDBAdaptor.QueryParams.SAMPLE_UIDS.key(), IndividualDBAdaptor.QueryParams.PARENTAL_CONSANGUINITY.key(), - IndividualDBAdaptor.QueryParams.KARYOTYPIC_SEX.key(), IndividualDBAdaptor.QueryParams.ANNOTATION_SETS.key())) - .append(DBAdaptor.INCLUDE_ACLS, true) - .append(ParamConstants.FLATTEN_ANNOTATIONS, true); - - catalogSolrManager.insertCatalogCollection(catalogManager.getIndividualManager().iterator(study.getFqn(), query, - individualQueryOptions, token), new CatalogIndividualToSolrIndividualConverter(study), - CatalogSolrManager.INDIVIDUAL_SOLR_COLLECTION); - } -} diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/individual/IndividualTsvAnnotationLoader.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/individual/IndividualTsvAnnotationLoader.java index 3c53ac46524..b6776329c50 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/individual/IndividualTsvAnnotationLoader.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/individual/IndividualTsvAnnotationLoader.java @@ -25,7 +25,7 @@ import org.opencb.opencga.core.tools.annotations.Tool; @Tool(id = IndividualTsvAnnotationLoader.ID, resource = Enums.Resource.INDIVIDUAL, type = Tool.Type.OPERATION, - description = "Load annotations from TSV file.") + description = "Load annotations from TSV file.", priority = Enums.Priority.HIGH) public class IndividualTsvAnnotationLoader extends TsvAnnotationLoader { public final static String ID = "individual-tsv-load"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/individual/qc/IndividualQcAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/individual/qc/IndividualQcAnalysis.java index f347169a21e..b8eb489031d 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/individual/qc/IndividualQcAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/individual/qc/IndividualQcAnalysis.java @@ -23,9 +23,14 @@ import org.opencb.opencga.analysis.tools.OpenCgaTool; import org.opencb.opencga.analysis.variant.inferredSex.InferredSexAnalysis; import org.opencb.opencga.catalog.exceptions.CatalogException; + import org.opencb.opencga.catalog.managers.CatalogManager; + +import org.opencb.opencga.catalog.utils.CatalogFqn; + import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.exceptions.ToolException; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.individual.Individual; import org.opencb.opencga.core.models.individual.IndividualQualityControl; @@ -85,6 +90,19 @@ protected void check() throws Exception { inferredSexMethod = COVERAGE_RATIO_INFERRED_SEX_METHOD; } + // Check permissions + try { + Study study = catalogManager.getStudyManager().get(studyId, QueryOptions.empty(), token).first(); + + JwtPayload jwtPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, jwtPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = jwtPayload.getUserId(organizationId); + catalogManager.getAuthorizationManager().checkStudyPermission(organizationId, study.getUid(), userId, WRITE_INDIVIDUALS); + } catch (CatalogException e) { + throw new ToolException(e); + } + // Main check (this function is shared with the endpoint individual/qc/run) checkParameters(individualId, sampleId, inferredSexMethod, studyId, catalogManager, token); @@ -239,8 +257,11 @@ public static void checkParameters(String individualId, String sampleId, String // Check permissions try { Study study = catalogManager.getStudyManager().get(studyId, QueryOptions.empty(), token).first(); - String userId = catalogManager.getUserManager().getUserId(token); - catalogManager.getAuthorizationManager().checkStudyPermission(study.getUid(), userId, WRITE_INDIVIDUALS); + JwtPayload jwtPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, jwtPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = jwtPayload.getUserId(organizationId); + catalogManager.getAuthorizationManager().checkStudyPermission(organizationId, study.getUid(), userId, WRITE_INDIVIDUALS); } catch (CatalogException e) { throw new ToolException(e); } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/individual/qc/IndividualQcUtils.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/individual/qc/IndividualQcUtils.java index 091cea875a8..6c1f4b94ed7 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/individual/qc/IndividualQcUtils.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/individual/qc/IndividualQcUtils.java @@ -29,7 +29,9 @@ import org.opencb.opencga.catalog.db.api.ProjectDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.CatalogManager; +import org.opencb.opencga.catalog.utils.CatalogFqn; import org.opencb.opencga.core.exceptions.ToolException; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.common.InternalStatus; import org.opencb.opencga.core.models.family.Family; import org.opencb.opencga.core.models.individual.Individual; @@ -132,7 +134,12 @@ public static List getSamples(String study, String familyId, CatalogMana public static String getAssembly(String study, CatalogManager catalogManager, String token) throws CatalogException { String assembly = ""; OpenCGAResult projectQueryResult; - projectQueryResult = catalogManager.getProjectManager().search(new Query(ProjectDBAdaptor.QueryParams.STUDY.key(), study), + + JwtPayload jwtPayload = new JwtPayload(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(study, jwtPayload); + String organizationId = studyFqn.getOrganizationId(); + + projectQueryResult = catalogManager.getProjectManager().search(organizationId, new Query(ProjectDBAdaptor.QueryParams.STUDY.key(), study), new QueryOptions(QueryOptions.INCLUDE, ProjectDBAdaptor.QueryParams.ORGANISM.key()), token); if (CollectionUtils.isNotEmpty(projectQueryResult.getResults())) { assembly = projectQueryResult.first().getOrganism().getAssembly(); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/job/JobIndexTask.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/job/JobIndexTask.java deleted file mode 100644 index 7eec33c901d..00000000000 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/job/JobIndexTask.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright 2015-2020 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.analysis.job; - -import org.opencb.commons.datastore.core.Query; -import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.analysis.tools.OpenCgaTool; -import org.opencb.opencga.catalog.db.api.DBAdaptor; -import org.opencb.opencga.catalog.db.api.JobDBAdaptor; -import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; -import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.stats.solr.CatalogSolrManager; -import org.opencb.opencga.catalog.stats.solr.converters.JobSolrConverter; -import org.opencb.opencga.catalog.stats.solr.converters.SolrConverterUtil; -import org.opencb.opencga.core.models.common.Enums; -import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.response.OpenCGAResult; -import org.opencb.opencga.core.tools.annotations.Tool; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Set; - -@Tool(id = JobIndexTask.ID, resource = Enums.Resource.JOB, type = Tool.Type.OPERATION, description = "Index Job entries in Solr.") -public class JobIndexTask extends OpenCgaTool { - - public final static String ID = "job-secondary-index"; - - private CatalogSolrManager catalogSolrManager; - - @Override - protected void check() throws Exception { - catalogSolrManager = new CatalogSolrManager(this.catalogManager); - } - - @Override - protected void run() throws Exception { - // Get all the studies - Query query = new Query(); - QueryOptions options = new QueryOptions() - .append(QueryOptions.INCLUDE, Arrays.asList(StudyDBAdaptor.QueryParams.UID.key(), - StudyDBAdaptor.QueryParams.ID.key(), StudyDBAdaptor.QueryParams.FQN.key(), - StudyDBAdaptor.QueryParams.VARIABLE_SET.key())) - .append(DBAdaptor.INCLUDE_ACLS, true); - OpenCGAResult studyDataResult = catalogManager.getStudyManager().search(query, options, token); - if (studyDataResult.getNumResults() == 0) { - throw new CatalogException("Could not index catalog into solr. No studies found"); - } - - // Create solr collections if they don't exist - catalogSolrManager.createSolrCollections(CatalogSolrManager.JOB_SOLR_COLLECTION); - - for (Study study : studyDataResult.getResults()) { - Map> studyAcls = SolrConverterUtil - .parseInternalOpenCGAAcls((List>) study.getAttributes().get("OPENCGA_ACL")); - // We replace the current studyAcls for the parsed one - study.getAttributes().put("OPENCGA_ACL", studyAcls); - - indexJob(catalogSolrManager, study); - } - - } - - private void indexJob(CatalogSolrManager catalogSolrManager, Study study) throws CatalogException { - logger.info("Indexing jobs of study {}", study.getFqn()); - - Query query = new Query(JobDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), Arrays.asList( - Enums.ExecutionStatus.ERROR, Enums.ExecutionStatus.DONE, Enums.ExecutionStatus.ABORTED)); - QueryOptions jobQueryOptions = new QueryOptions() - .append(QueryOptions.INCLUDE, Arrays.asList(JobDBAdaptor.QueryParams.UID.key(), JobDBAdaptor.QueryParams.UUID.key(), - JobDBAdaptor.QueryParams.STUDY_UID.key(), JobDBAdaptor.QueryParams.CREATION_DATE.key(), - JobDBAdaptor.QueryParams.RELEASE.key(), JobDBAdaptor.QueryParams.INTERNAL_STATUS.key(), JobDBAdaptor.QueryParams.TOOL.key(), - JobDBAdaptor.QueryParams.USER_ID.key(), JobDBAdaptor.QueryParams.PRIORITY.key(), - JobDBAdaptor.QueryParams.TAGS.key(), JobDBAdaptor.QueryParams.EXECUTION.key())) - .append(DBAdaptor.INCLUDE_ACLS, true); - - catalogSolrManager.insertCatalogCollection(catalogManager.getJobManager().iterator(study.getFqn(), query, jobQueryOptions, token), - new JobSolrConverter(study), CatalogSolrManager.JOB_SOLR_COLLECTION); - } -} diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/models/StudyInfo.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/models/StudyInfo.java index ba7cf98961c..a89301b5a0d 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/models/StudyInfo.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/models/StudyInfo.java @@ -51,15 +51,6 @@ public StudyInfo() { this.fileInfos = new ArrayList<>(); } - public String getUserId() { - return userId; - } - - public StudyInfo setUserId(String userId) { - this.userId = userId; - return this; - } - public long getProjectUid() { return projectUid; } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/rga/RgaManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/rga/RgaManager.java index 5acff933d6e..a09f5f7f6c1 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/rga/RgaManager.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/rga/RgaManager.java @@ -19,15 +19,18 @@ import org.opencb.opencga.analysis.rga.exceptions.RgaException; import org.opencb.opencga.analysis.rga.iterators.RgaIterator; import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; +import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager; import org.opencb.opencga.catalog.db.api.SampleDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.AbstractManager; import org.opencb.opencga.catalog.managers.CatalogManager; import org.opencb.opencga.catalog.managers.FileManager; import org.opencb.opencga.catalog.managers.SampleManager; +import org.opencb.opencga.catalog.utils.CatalogFqn; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.storage.StorageConfiguration; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.analysis.knockout.*; import org.opencb.opencga.core.models.common.RgaIndex; import org.opencb.opencga.core.models.file.File; @@ -142,10 +145,16 @@ public void index(String studyStr, String fileStr, String token) throws CatalogE */ public void index(String study, Path file, String token) throws CatalogException, IOException, RgaException { checkStorageReadMode(); - String userId = catalogManager.getUserManager().getUserId(token); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(study, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study studyObject = catalogManager.getStudyManager().get(study, QueryOptions.empty(), token).first(); try { - catalogManager.getAuthorizationManager().isOwnerOrAdmin(studyObject.getUid(), userId); + AuthorizationManager authorizationManager = catalogManager.getAuthorizationManager(); + long studyId = studyObject.getUid(); + authorizationManager.checkIsAtLeastStudyAdministrator(organizationId, studyId, userId); } catch (CatalogException e) { logger.error(e.getMessage(), e); throw new CatalogException("Only owners or admins can index", e.getCause()); @@ -225,10 +234,17 @@ private void load(String study, Path file, String token) throws RgaException { public void generateAuxiliarCollection(String studyStr, String token) throws CatalogException, RgaException, IOException { checkStorageReadMode(); - String userId = catalogManager.getUserManager().getUserId(token); + + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); Study study = catalogManager.getStudyManager().get(studyStr, QueryOptions.empty(), token).first(); + try { - catalogManager.getAuthorizationManager().isOwnerOrAdmin(study.getUid(), userId); + AuthorizationManager authorizationManager = catalogManager.getAuthorizationManager(); + long studyId = study.getUid(); + authorizationManager.checkIsAtLeastStudyAdministrator(organizationId, studyId, userId); } catch (CatalogException e) { logger.error(e.getMessage(), e); throw new CatalogException("Only owners or admins can generate the auxiliary RGA collection", e.getCause()); @@ -432,11 +448,15 @@ private AuxiliarRgaDataModel getAuxiliarRgaDataModel(String mainCollection, Stri private OpenCGAResult updateRgaInternalIndexStatus(String studyStr, List sampleIds, RgaIndex.Status status, String token) throws CatalogException, RgaException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); Study study = catalogManager.getStudyManager().get(studyStr, QueryOptions.empty(), token).first(); - String userId = catalogManager.getUserManager().getUserId(token); + String collection = getMainCollectionName(study.getFqn()); - catalogManager.getAuthorizationManager().checkIsOwnerOrAdmin(study.getUid(), userId); + catalogManager.getAuthorizationManager().checkIsAtLeastStudyAdministrator(organizationId, study.getUid(), userId); if (!rgaEngine.isAlive(collection)) { throw new RgaException("Missing RGA indexes for study '" + study.getFqn() + "' or solr server not alive"); @@ -475,11 +495,15 @@ private OpenCGAResult updateRgaInternalIndexStatus(String studyStr, List updateRgaInternalIndexStatus(String studyStr, String token) throws CatalogException, IOException, RgaException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); Study study = catalogManager.getStudyManager().get(studyStr, QueryOptions.empty(), token).first(); - String userId = catalogManager.getUserManager().getUserId(token); + String collection = getMainCollectionName(study.getFqn()); - catalogManager.getAuthorizationManager().checkIsOwnerOrAdmin(study.getUid(), userId); + catalogManager.getAuthorizationManager().checkIsAtLeastStudyAdministrator(organizationId, study.getUid(), userId); if (!rgaEngine.isAlive(collection)) { throw new RgaException("Missing RGA indexes for study '" + study.getFqn() + "' or solr server not alive"); @@ -535,12 +559,17 @@ public OpenCGAResult individualQuery(String studyStr, Quer return cacheResults; } + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); Study study = catalogManager.getStudyManager().get(studyStr, QueryOptions.empty(), token).first(); + String collection = getMainCollectionName(study.getFqn()); Preprocess preprocess; try { - preprocess = individualQueryPreprocess(study, query, options, token); + preprocess = individualQueryPreprocess(organizationId, study, query, options, userId, token); } catch (RgaException e) { if (RgaException.NO_RESULTS_FOUND.equals(e.getMessage())) { stopWatch.stop(); @@ -577,7 +606,8 @@ public OpenCGAResult individualQuery(String studyStr, Quer } } // Check parent permissions... - Set authorisedSamples = getAuthorisedSamples(study.getFqn(), parentSampleIds, null, preprocess.getUserId(), token); + Set authorisedSamples = getAuthorisedSamples(organizationId, study.getFqn(), parentSampleIds, null, + preprocess.getUserId(), token); if (authorisedSamples.size() < parentSampleIds.size()) { // Filter out parent sample ids for (KnockoutByIndividual knockoutByIndividual : knockoutByIndividuals) { @@ -618,8 +648,12 @@ public OpenCGAResult geneQuery(String studyStr, Query query, return cacheResults; } + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); Study study = catalogManager.getStudyManager().get(studyStr, QueryOptions.empty(), token).first(); - String userId = catalogManager.getUserManager().getUserId(token); + String collection = getMainCollectionName(study.getFqn()); if (!rgaEngine.isAlive(collection)) { throw new RgaException("Missing RGA indexes for study '" + study.getFqn() + "' or solr server not alive"); @@ -630,7 +664,9 @@ public OpenCGAResult geneQuery(String studyStr, Query query, QueryOptions queryOptions = setDefaultLimit(options); List includeIndividuals = queryOptions.getAsStringList(RgaQueryParams.INCLUDE_INDIVIDUAL); - Boolean isOwnerOrAdmin = catalogManager.getAuthorizationManager().isOwnerOrAdmin(study.getUid(), userId); + AuthorizationManager authorizationManager = catalogManager.getAuthorizationManager(); + long studyId = study.getUid(); + boolean isOwnerOrAdmin = authorizationManager.isAtLeastStudyAdministrator(organizationId, studyId, userId); Query auxQuery = query != null ? new Query(query) : new Query(); // Get number of matches @@ -668,7 +704,7 @@ public OpenCGAResult geneQuery(String studyStr, Query query, .append(ACL_PARAM, userId + ":" + SamplePermissions.VIEW_VARIANTS) .append(SampleDBAdaptor.QueryParams.INTERNAL_RGA_STATUS.key(), RgaIndex.Status.INDEXED) .append(SampleDBAdaptor.QueryParams.ID.key(), includeIndividuals); - OpenCGAResult authorisedSampleIdResult = catalogManager.getSampleManager().distinct(study.getFqn(), + OpenCGAResult authorisedSampleIdResult = catalogManager.getSampleManager().distinct(organizationId, study.getFqn(), SampleDBAdaptor.QueryParams.ID.key(), sampleQuery, token); includeSampleIds = new HashSet<>((List) authorisedSampleIdResult.getResults()); } else { @@ -684,7 +720,7 @@ public OpenCGAResult geneQuery(String studyStr, Query query, // 3. Get list of sample ids for which the user has permissions Query sampleQuery = new Query(ACL_PARAM, userId + ":" + SamplePermissions.VIEW_VARIANTS) .append(SampleDBAdaptor.QueryParams.ID.key(), sampleIds); - OpenCGAResult authorisedSampleIdResult = catalogManager.getSampleManager().distinct(study.getFqn(), + OpenCGAResult authorisedSampleIdResult = catalogManager.getSampleManager().distinct(organizationId, study.getFqn(), SampleDBAdaptor.QueryParams.ID.key(), sampleQuery, token); // TODO: The number of samples to include could be really high includeSampleIds = new HashSet<>((List) authorisedSampleIdResult.getResults()); @@ -701,7 +737,7 @@ public OpenCGAResult geneQuery(String studyStr, Query query, logger.warn("Include only the samples that are actually necessary"); } - OpenCGAResult sampleResult = catalogManager.getSampleManager().distinct(study.getFqn(), + OpenCGAResult sampleResult = catalogManager.getSampleManager().distinct(organizationId, study.getFqn(), SampleDBAdaptor.QueryParams.ID.key(), sampleQuery, token); includeSampleIds = new HashSet<>((List) sampleResult.getResults()); } @@ -752,8 +788,12 @@ public OpenCGAResult variantQuery(String studyStr, Query quer return cacheResults; } + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); Study study = catalogManager.getStudyManager().get(studyStr, QueryOptions.empty(), token).first(); - String userId = catalogManager.getUserManager().getUserId(token); + String collection = getMainCollectionName(study.getFqn()); String auxCollection = getAuxCollectionName(study.getFqn()); if (!rgaEngine.isAlive(collection)) { @@ -767,7 +807,9 @@ public OpenCGAResult variantQuery(String studyStr, Query quer QueryOptions queryOptions = setDefaultLimit(options); List includeIndividuals = queryOptions.getAsStringList(RgaQueryParams.INCLUDE_INDIVIDUAL); - Boolean isOwnerOrAdmin = catalogManager.getAuthorizationManager().isOwnerOrAdmin(study.getUid(), userId); + AuthorizationManager authorizationManager = catalogManager.getAuthorizationManager(); + long studyId = study.getUid(); + boolean isOwnerOrAdmin = authorizationManager.isAtLeastStudyAdministrator(organizationId, studyId, userId); Query auxQuery = query != null ? new Query(query) : new Query(); ResourceIds resourceIds; @@ -789,7 +831,7 @@ public OpenCGAResult variantQuery(String studyStr, Query quer .append(ACL_PARAM, userId + ":" + SamplePermissions.VIEW_VARIANTS) .append(SampleDBAdaptor.QueryParams.INTERNAL_RGA_STATUS.key(), RgaIndex.Status.INDEXED) .append(SampleDBAdaptor.QueryParams.INDIVIDUAL_ID.key(), includeIndividuals); - OpenCGAResult authorisedSampleIdResult = catalogManager.getSampleManager().distinct(study.getFqn(), + OpenCGAResult authorisedSampleIdResult = catalogManager.getSampleManager().distinct(organizationId, study.getFqn(), SampleDBAdaptor.QueryParams.ID.key(), sampleQuery, token); includeSampleIds = new HashSet<>((List) authorisedSampleIdResult.getResults()); } else { @@ -808,7 +850,7 @@ public OpenCGAResult variantQuery(String studyStr, Query quer Query sampleQuery = new Query() .append(ACL_PARAM, userId + ":" + SamplePermissions.VIEW_VARIANTS) .append(SampleDBAdaptor.QueryParams.ID.key(), sampleIds); - OpenCGAResult authorisedSampleIdResult = catalogManager.getSampleManager().distinct(study.getFqn(), + OpenCGAResult authorisedSampleIdResult = catalogManager.getSampleManager().distinct(organizationId, study.getFqn(), SampleDBAdaptor.QueryParams.ID.key(), sampleQuery, token); includeSampleIds = new HashSet<>((List) authorisedSampleIdResult.getResults()); } @@ -823,7 +865,7 @@ public OpenCGAResult variantQuery(String studyStr, Query quer logger.warn("Include only the samples that are actually necessary"); } - OpenCGAResult sampleResult = catalogManager.getSampleManager().distinct(study.getFqn(), + OpenCGAResult sampleResult = catalogManager.getSampleManager().distinct(organizationId, study.getFqn(), SampleDBAdaptor.QueryParams.ID.key(), sampleQuery, token); includeSampleIds = new HashSet<>((List) sampleResult.getResults()); } @@ -920,7 +962,12 @@ public OpenCGAResult individualSummary(String study return cacheResults; } + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); Study study = catalogManager.getStudyManager().get(studyStr, QueryOptions.empty(), token).first(); + String collection = getMainCollectionName(study.getFqn()); ExecutorService executor = Executors.newFixedThreadPool(4); @@ -943,7 +990,7 @@ public OpenCGAResult individualSummary(String study Preprocess preprocess; try { - preprocess = individualQueryPreprocess(study, query, options, token); + preprocess = individualQueryPreprocess(organizationId, study, query, options, userId, token); } catch (RgaException e) { if (RgaException.NO_RESULTS_FOUND.equals(e.getMessage())) { stopWatch.stop(); @@ -954,7 +1001,7 @@ public OpenCGAResult individualSummary(String study throw e; } - catalogManager.getAuthorizationManager().checkStudyPermission(study.getUid(), preprocess.getUserId(), + catalogManager.getAuthorizationManager().checkStudyPermission(organizationId, study.getUid(), preprocess.getUserId(), StudyPermissions.Permissions.VIEW_AGGREGATED_VARIANTS); List sampleIds = preprocess.getQuery().getAsStringList(RgaQueryParams.SAMPLE_ID.key()); @@ -995,7 +1042,8 @@ public OpenCGAResult individualSummary(String study if (!parentSampleIds.isEmpty()) { // Check parent permissions... - Set authorisedSamples = getAuthorisedSamples(study.getFqn(), parentSampleIds, null, preprocess.getUserId(), token); + Set authorisedSamples = getAuthorisedSamples(organizationId, study.getFqn(), parentSampleIds, null, + preprocess.getUserId(), token); // Filter out parent sample ids if (authorisedSamples.size() < parentSampleIds.size()) { for (KnockoutByIndividualSummary knockoutByIndividualSummary : knockoutByIndividualSummaryList) { @@ -1047,14 +1095,18 @@ public OpenCGAResult geneSummary(String studyStr, Query q return cacheResults; } + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); Study study = catalogManager.getStudyManager().get(studyStr, QueryOptions.empty(), token).first(); - String userId = catalogManager.getUserManager().getUserId(token); + String collection = getMainCollectionName(study.getFqn()); if (!rgaEngine.isAlive(collection)) { throw new RgaException("Missing RGA indexes for study '" + study.getFqn() + "' or solr server not alive"); } - catalogManager.getAuthorizationManager().checkStudyPermission(study.getUid(), userId, + catalogManager.getAuthorizationManager().checkStudyPermission(organizationId, study.getUid(), userId, StudyPermissions.Permissions.VIEW_AGGREGATED_VARIANTS); ExecutorService executor = Executors.newFixedThreadPool(4); @@ -1131,8 +1183,12 @@ public OpenCGAResult variantSummary(String studyStr, Q return cacheResults; } + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); Study study = catalogManager.getStudyManager().get(studyStr, QueryOptions.empty(), token).first(); - String userId = catalogManager.getUserManager().getUserId(token); + String collection = getMainCollectionName(study.getFqn()); String auxCollection = getAuxCollectionName(study.getFqn()); if (!rgaEngine.isAlive(collection)) { @@ -1142,7 +1198,7 @@ public OpenCGAResult variantSummary(String studyStr, Q throw new RgaException("Missing auxiliar RGA collection for study '" + study.getFqn() + "'"); } - catalogManager.getAuthorizationManager().checkStudyPermission(study.getUid(), userId, + catalogManager.getAuthorizationManager().checkStudyPermission(organizationId, study.getUid(), userId, StudyPermissions.Permissions.VIEW_AGGREGATED_VARIANTS); ExecutorService executor = Executors.newFixedThreadPool(4); @@ -1237,10 +1293,13 @@ public OpenCGAResult variantSummary(String studyStr, Q public OpenCGAResult aggregationStats(String studyStr, Query query, QueryOptions options, String fields, String token) throws CatalogException, IOException, RgaException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); Study study = catalogManager.getStudyManager().get(studyStr, QueryOptions.empty(), token).first(); - String userId = catalogManager.getUserManager().getUserId(token); - catalogManager.getAuthorizationManager().checkCanViewStudy(study.getUid(), userId); + catalogManager.getAuthorizationManager().checkCanViewStudy(organizationId, study.getUid(), userId); String collection = getMainCollectionName(study.getFqn()); if (!rgaEngine.isAlive(collection)) { @@ -1706,27 +1765,29 @@ private KnockoutByIndividualSummary calculateIndividualSummary(String collection return knockoutByIndividualSummary; } - private Set getAuthorisedSamples(String study, Set sampleIds, List otherPermissions, - String userId, String token) throws CatalogException { + private Set getAuthorisedSamples(String organizationId, String study, Set sampleIds, + List otherPermissions, String userId, String token) + throws CatalogException { Query query = new Query(SampleDBAdaptor.QueryParams.ID.key(), sampleIds); if (org.apache.commons.collections4.CollectionUtils.isNotEmpty(otherPermissions)) { query.put(ACL_PARAM, userId + ":" + StringUtils.join(otherPermissions, ",")); } - OpenCGAResult distinct = catalogManager.getSampleManager().distinct(study, SampleDBAdaptor.QueryParams.ID.key(), query, token); + OpenCGAResult distinct = catalogManager.getSampleManager().distinct(organizationId, study, SampleDBAdaptor.QueryParams.ID.key(), + query, token); return distinct.getResults().stream().map(String::valueOf).collect(Collectors.toSet()); } - private Preprocess individualQueryPreprocess(Study study, Query query, QueryOptions options, String token) - throws RgaException, CatalogException, IOException { - - String userId = catalogManager.getUserManager().getUserId(token); + private Preprocess individualQueryPreprocess(String organizationId, Study study, Query query, QueryOptions options, String userId, + String token) throws RgaException, CatalogException, IOException { String collection = getMainCollectionName(study.getFqn()); if (!rgaEngine.isAlive(collection)) { throw new RgaException("Missing RGA indexes for study '" + study.getFqn() + "' or solr server not alive"); } - Boolean isOwnerOrAdmin = catalogManager.getAuthorizationManager().isOwnerOrAdmin(study.getUid(), userId); + AuthorizationManager authorizationManager = catalogManager.getAuthorizationManager(); + long studyId = study.getUid(); + boolean isOwnerOrAdmin = authorizationManager.isAtLeastStudyAdministrator(organizationId, studyId, userId); Preprocess preprocessResult = new Preprocess(); preprocessResult.setUserId(userId); @@ -1813,7 +1874,7 @@ private Preprocess individualQueryPreprocess(Study study, Query query, QueryOpti List tmpValues = values.subList(currentBatch, Math.min(values.size(), batchSize + currentBatch)); sampleQuery.put(sampleQueryField, tmpValues); - OpenCGAResult authorisedSampleIdResult = catalogManager.getSampleManager().distinct(study.getFqn(), + OpenCGAResult authorisedSampleIdResult = catalogManager.getSampleManager().distinct(organizationId, study.getFqn(), SampleDBAdaptor.QueryParams.ID.key(), sampleQuery, token); authorisedSamples.addAll((Collection) authorisedSampleIdResult.getResults()); @@ -1866,11 +1927,15 @@ public void testConnection() throws StorageEngineException { } private String getMainCollectionName(String study) { - return catalogManager.getConfiguration().getDatabasePrefix() + "-rga-" + study.replace("@", "_").replace(":", "_"); + CatalogFqn fqn = CatalogFqn.extractFqnFromStudyFqn(study); + return catalogManager.getConfiguration().getDatabasePrefix() + "-rga-" + + fqn.getOrganizationId() + "_" + fqn.getProjectId() + "_" + fqn.getStudyId(); } private String getAuxCollectionName(String study) { - return catalogManager.getConfiguration().getDatabasePrefix() + "-rga-aux-" + study.replace("@", "_").replace(":", "_"); + CatalogFqn fqn = CatalogFqn.extractFqnFromStudyFqn(study); + return catalogManager.getConfiguration().getDatabasePrefix() + "-rga-aux-" + + fqn.getOrganizationId() + "_" + fqn.getProjectId() + "_" + fqn.getStudyId(); } @Override diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/sample/SampleIndexTask.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/sample/SampleIndexTask.java deleted file mode 100644 index 33c0800742b..00000000000 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/sample/SampleIndexTask.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright 2015-2020 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.analysis.sample; - -import org.opencb.commons.datastore.core.Query; -import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.analysis.tools.OpenCgaTool; -import org.opencb.opencga.catalog.db.api.DBAdaptor; -import org.opencb.opencga.catalog.db.api.SampleDBAdaptor; -import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; -import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.stats.solr.CatalogSolrManager; -import org.opencb.opencga.catalog.stats.solr.converters.CatalogSampleToSolrSampleConverter; -import org.opencb.opencga.catalog.stats.solr.converters.SolrConverterUtil; -import org.opencb.opencga.core.api.ParamConstants; -import org.opencb.opencga.core.models.common.Enums; -import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.response.OpenCGAResult; -import org.opencb.opencga.core.tools.annotations.Tool; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Set; - -@Tool(id = SampleIndexTask.ID, resource = Enums.Resource.SAMPLE, type = Tool.Type.OPERATION, description = "Index Sample entries in Solr.") -public class SampleIndexTask extends OpenCgaTool { - - public final static String ID = "sample-secondary-index"; - - private CatalogSolrManager catalogSolrManager; - - @Override - protected void check() throws Exception { - catalogSolrManager = new CatalogSolrManager(this.catalogManager); - } - - @Override - protected void run() throws Exception { - // Get all the studies - Query query = new Query(); - QueryOptions options = new QueryOptions() - .append(QueryOptions.INCLUDE, Arrays.asList(StudyDBAdaptor.QueryParams.UID.key(), - StudyDBAdaptor.QueryParams.ID.key(), StudyDBAdaptor.QueryParams.FQN.key(), - StudyDBAdaptor.QueryParams.VARIABLE_SET.key())) - .append(DBAdaptor.INCLUDE_ACLS, true); - OpenCGAResult studyDataResult = catalogManager.getStudyManager().search(query, options, token); - if (studyDataResult.getNumResults() == 0) { - throw new CatalogException("Could not index catalog into solr. No studies found"); - } - - // Create solr collections if they don't exist - catalogSolrManager.createSolrCollections(CatalogSolrManager.SAMPLE_SOLR_COLLECTION); - - for (Study study : studyDataResult.getResults()) { - Map> studyAcls = SolrConverterUtil - .parseInternalOpenCGAAcls((List>) study.getAttributes().get("OPENCGA_ACL")); - // We replace the current studyAcls for the parsed one - study.getAttributes().put("OPENCGA_ACL", studyAcls); - - indexSample(catalogSolrManager, study); - } - - } - - private void indexSample(CatalogSolrManager catalogSolrManager, Study study) throws CatalogException { - logger.info("Indexing samples of study {}", study.getFqn()); - - Query query = new Query() - .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - QueryOptions sampleQueryOptions = new QueryOptions() - .append(QueryOptions.INCLUDE, Arrays.asList(SampleDBAdaptor.QueryParams.UUID.key(), - SampleDBAdaptor.QueryParams.RELEASE.key(), SampleDBAdaptor.QueryParams.VERSION.key(), - SampleDBAdaptor.QueryParams.PROCESSING.key(), SampleDBAdaptor.QueryParams.COLLECTION.key(), - SampleDBAdaptor.QueryParams.CREATION_DATE.key(), SampleDBAdaptor.QueryParams.INTERNAL_STATUS.key(), - SampleDBAdaptor.QueryParams.SOMATIC.key(), - SampleDBAdaptor.QueryParams.PHENOTYPES.key(), SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key(), - SampleDBAdaptor.QueryParams.UID.key())) - .append(DBAdaptor.INCLUDE_ACLS, true) - .append(ParamConstants.FLATTEN_ANNOTATIONS, true); - - catalogSolrManager.insertCatalogCollection(catalogManager.getSampleManager().iterator(study.getFqn(), query, sampleQueryOptions, - token), new CatalogSampleToSolrSampleConverter(study), CatalogSolrManager.SAMPLE_SOLR_COLLECTION); - } -} diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/sample/SampleTsvAnnotationLoader.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/sample/SampleTsvAnnotationLoader.java index adeb819720b..c3fa3ad84b1 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/sample/SampleTsvAnnotationLoader.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/sample/SampleTsvAnnotationLoader.java @@ -25,7 +25,7 @@ import org.opencb.opencga.core.tools.annotations.Tool; @Tool(id = SampleTsvAnnotationLoader.ID, resource = Enums.Resource.SAMPLE, type = Tool.Type.OPERATION, - description = "Load annotations from TSV file.") + description = "Load annotations from TSV file.", priority = Enums.Priority.HIGH) public class SampleTsvAnnotationLoader extends TsvAnnotationLoader { public final static String ID = "sample-tsv-load"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/sample/qc/SampleQcAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/sample/qc/SampleQcAnalysis.java index a103568d020..b8d460824c1 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/sample/qc/SampleQcAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/sample/qc/SampleQcAnalysis.java @@ -18,34 +18,26 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; -import org.opencb.biodata.formats.alignment.picard.HsMetrics; -import org.opencb.biodata.formats.alignment.samtools.SamtoolsFlagstats; -import org.opencb.biodata.formats.alignment.samtools.SamtoolsStats; -import org.opencb.biodata.formats.sequence.fastqc.FastQcMetrics; -import org.opencb.biodata.models.clinical.qc.*; +import org.opencb.biodata.models.clinical.qc.GenomePlot; +import org.opencb.biodata.models.clinical.qc.GenomePlotConfig; +import org.opencb.biodata.models.clinical.qc.SampleQcVariantStats; import org.opencb.commons.datastore.core.Event; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.analysis.AnalysisUtils; -import org.opencb.opencga.analysis.alignment.qc.AlignmentFastQcMetricsAnalysis; -import org.opencb.opencga.analysis.alignment.qc.AlignmentFlagStatsAnalysis; -import org.opencb.opencga.analysis.alignment.qc.AlignmentHsMetricsAnalysis; -import org.opencb.opencga.analysis.alignment.qc.AlignmentStatsAnalysis; import org.opencb.opencga.analysis.individual.qc.IndividualQcUtils; import org.opencb.opencga.analysis.tools.OpenCgaToolScopeStudy; import org.opencb.opencga.analysis.variant.genomePlot.GenomePlotAnalysis; import org.opencb.opencga.analysis.variant.mutationalSignature.MutationalSignatureAnalysis; import org.opencb.opencga.analysis.variant.stats.SampleVariantStatsAnalysis; import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.utils.CatalogFqn; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.exceptions.ToolException; -import org.opencb.opencga.core.models.alignment.AlignmentFileQualityControl; -import org.opencb.opencga.core.models.alignment.AlignmentQcParams; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.file.File; -import org.opencb.opencga.core.models.file.FileQualityControl; -import org.opencb.opencga.core.models.file.FileUpdateParams; import org.opencb.opencga.core.models.job.Job; import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.sample.SampleQualityControl; @@ -63,7 +55,6 @@ import java.nio.file.Paths; import java.util.*; -import static org.opencb.opencga.analysis.variant.mutationalSignature.MutationalSignatureAnalysis.*; import static org.opencb.opencga.core.models.study.StudyPermissions.Permissions.WRITE_SAMPLES; @Tool(id = SampleQcAnalysis.ID, resource = Enums.Resource.SAMPLE, description = SampleQcAnalysis.DESCRIPTION) @@ -93,9 +84,13 @@ protected void check() throws Exception { // Check permissions try { + JwtPayload jwtPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(study, jwtPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = jwtPayload.getUserId(organizationId); + long studyUid = catalogManager.getStudyManager().get(getStudy(), QueryOptions.empty(), token).first().getUid(); - String userId = catalogManager.getUserManager().getUserId(token); - catalogManager.getAuthorizationManager().checkStudyPermission(studyUid, userId, WRITE_SAMPLES); + catalogManager.getAuthorizationManager().checkStudyPermission(organizationId, studyUid, userId, WRITE_SAMPLES); } catch (CatalogException e) { throw new ToolException(e); } @@ -230,7 +225,8 @@ protected void run() throws ToolException { OpenCGAResult variantStatsJobResult = catalogManager.getJobManager() .submit(study, SampleVariantStatsAnalysis.ID, Enums.Priority.MEDIUM, params, null, "Job generated by " - + getId() + " - " + getJobId(), Collections.emptyList(), Collections.emptyList(), token); + + getId() + " - " + getJobId(), Collections.emptyList(), Collections.emptyList(), getJobId(), null, + false, token); variantStatsJobId = variantStatsJobResult.first().getId(); addEvent(Event.Type.INFO, "Submit job " + variantStatsJobId + " to compute stats (" + SampleVariantStatsAnalysis.ID + ")"); @@ -272,7 +268,8 @@ protected void run() throws ToolException { OpenCGAResult signatureJobResult = catalogManager.getJobManager() .submit(getStudy(), MutationalSignatureAnalysis.ID, Enums.Priority.MEDIUM, params, null, "Job generated by " - + getId() + " - " + getJobId(), Collections.emptyList(), Collections.emptyList(), token); + + getId() + " - " + getJobId(), Collections.emptyList(), Collections.emptyList(), getJobId(), null, + false, token); signatureJobId = signatureJobResult.first().getId(); logger.info("Submitted job {} to compute the mutational signature analysis {}", signatureJobId, MutationalSignatureAnalysis.ID); @@ -294,7 +291,7 @@ protected void run() throws ToolException { OpenCGAResult genomePlotJobResult = catalogManager.getJobManager() .submit(getStudy(), GenomePlotAnalysis.ID, Enums.Priority.MEDIUM, params, null, "Job generated by " + getId() + " - " + getJobId(), Collections.emptyList(), Collections.emptyList(), - token); + getJobId(), null, false, token); genomePlotJobId = genomePlotJobResult.first().getId(); addEvent(Event.Type.INFO, "Submit job " + genomePlotJobId + " to compute genome plot (" + GenomePlotAnalysis.ID + ")"); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/templates/TemplateRunner.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/templates/TemplateRunner.java index 08797e1185e..cf8390d1a61 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/templates/TemplateRunner.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/templates/TemplateRunner.java @@ -15,7 +15,7 @@ import java.nio.file.Paths; @Tool(id = TemplateRunner.ID, description = TemplateRunner.DESCRIPTION, type = Tool.Type.OPERATION, resource = Enums.Resource.STUDY, - scope = Tool.Scope.STUDY) + scope = Tool.Scope.STUDY, priority = Enums.Priority.HIGH) public class TemplateRunner extends OperationTool { public static final String ID = "templates"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaTool.java index c80a799a050..6024761d596 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaTool.java @@ -67,6 +67,7 @@ public abstract class OpenCgaTool { private String jobId; private String opencgaHome; + private boolean dryRun; protected String token; protected final ObjectMap params; @@ -92,19 +93,20 @@ public OpenCgaTool() { } public final OpenCgaTool setUp(String opencgaHome, CatalogManager catalogManager, StorageEngineFactory engineFactory, - ObjectMap params, Path outDir, String jobId, String token) { + ObjectMap params, Path outDir, String jobId, boolean dryRun, String token) { VariantStorageManager manager = new VariantStorageManager(catalogManager, engineFactory); - return setUp(opencgaHome, catalogManager, manager, params, outDir, jobId, token); + return setUp(opencgaHome, catalogManager, manager, params, outDir, jobId, dryRun, token); } public final OpenCgaTool setUp(String opencgaHome, CatalogManager catalogManager, VariantStorageManager variantStorageManager, - ObjectMap params, Path outDir, String jobId, String token) { + ObjectMap params, Path outDir, String jobId, boolean dryRun, String token) { this.opencgaHome = opencgaHome; this.catalogManager = catalogManager; this.configuration = catalogManager.getConfiguration(); this.variantStorageManager = variantStorageManager; this.storageConfiguration = variantStorageManager.getStorageConfiguration(); this.jobId = jobId; + this.dryRun = dryRun; this.token = token; if (params != null) { this.params.putAll(params); @@ -188,13 +190,14 @@ public final ExecutionResult start() throws ToolException { exception = e; } if (!erm.isClosed()) { - privateLogger.error("Unexpected system shutdown!"); + String message = "Unexpected system shutdown. Job killed by the system."; + privateLogger.error(message); try { if (scratchDir != null) { deleteScratchDirectory(); } if (exception == null) { - exception = new RuntimeException("Unexpected system shutdown"); + exception = new RuntimeException(message); } logException(exception); ExecutionResult result = erm.close(exception); @@ -243,21 +246,32 @@ public final ExecutionResult start() throws ToolException { try { currentStep = "check"; privateCheck(); - check(); - currentStep = null; - erm.setSteps(getSteps()); - run(); + if (dryRun) { + logger.info("Dry run enabled. Sleep for 5 seconds and skip execution."); + Thread.sleep(5000); + } else { + currentStep = null; + erm.setSteps(getSteps()); + run(); + } } catch (ToolException e) { throw e; } catch (Exception e) { throw new ToolException(e); } + Runtime.getRuntime().removeShutdownHook(hook); } catch (Throwable e) { exception = e; + // Do not use a finally block to remove shutdownHook, as finally blocks will be executed even if the JVM is killed, + // and this would throw IllegalStateException("Shutdown in progress"); + try { + Runtime.getRuntime().removeShutdownHook(hook); + } catch (Exception e1) { + e.addSuppressed(e1); + } throw e; } finally { deleteScratchDirectory(); - Runtime.getRuntime().removeShutdownHook(hook); stopMemoryMonitor(); result = erm.close(exception); logException(exception); @@ -308,6 +322,7 @@ private void privateCheck() throws Exception { if (toolParams != null) { toolParams.updateParams(getParams()); } + check(); } /** @@ -537,7 +552,7 @@ protected final T getToolExecutor(Class clazz toolExecutor.getSource(), toolExecutor.getFramework())); - toolExecutor.setUp(erm, executorParams, outDir); + toolExecutor.setUp(erm, executorParams, outDir, configuration); return toolExecutor; } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/ToolFactory.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/ToolFactory.java index 4fac5476896..ba31a2eb4a4 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/ToolFactory.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/ToolFactory.java @@ -37,14 +37,16 @@ public class ToolFactory { private static Map>> duplicatedTools; private static List> toolsList; - private static synchronized Map> loadTools() { + public static final String DEFAULT_PACKAGE = "org.opencb.opencga"; + + private static synchronized Map> loadTools(List packages) { if (toolsCache == null) { Reflections reflections = new Reflections(new ConfigurationBuilder() .setScanners( new SubTypesScanner(), new TypeAnnotationsScanner().filterResultsBy(s -> StringUtils.equals(s, Tool.class.getName())) ) - .addUrls(getUrls()) + .addUrls(getUrlsFromPackages(packages)) .filterInputsBy(input -> input != null && input.endsWith(".class")) ); @@ -85,15 +87,26 @@ private static synchronized Map> loadTools( } return toolsCache; } + static Collection getUrlsFromPackages(List packages) { + Collection urls = new LinkedList<>(); + for (String pack :packages){ + for (URL url : ClasspathHelper.forPackage(pack)) { + String name = url.getPath().substring(url.getPath().lastIndexOf('/') + 1); + if (name.isEmpty() || (name.contains("opencga") && !name.contains("opencga-hadoop-shaded"))) { + urls.add(url); + } + } + } + return urls; + } static Collection getUrls() { - // TODO: What if there are third party libraries that implement Tools? // Currently they must contain "opencga" in the jar name. // e.g. acme-rockets-opencga-5.4.0.jar Collection urls = new LinkedList<>(); for (URL url : ClasspathHelper.forPackage("org.opencb.opencga")) { String name = url.getPath().substring(url.getPath().lastIndexOf('/') + 1); - if (name.isEmpty() || (name.contains("opencga") && !name.contains("opencga-storage-hadoop-deps"))) { + if (name.isEmpty() || (name.contains("opencga") && !name.contains("opencga-hadoop-shaded"))) { urls.add(url); } } @@ -101,6 +114,10 @@ static Collection getUrls() { } public final Class getToolClass(String toolId) throws ToolException { + return getToolClass(toolId, Collections.singletonList(DEFAULT_PACKAGE)); + } + + public final Class getToolClass(String toolId, List packages) throws ToolException { Objects.requireNonNull(toolId); Class aClass = null; @@ -112,7 +129,7 @@ public final Class getToolClass(String toolId) throws Too } catch (ClassNotFoundException ignore) { } if (aClass == null) { - aClass = loadTools().get(toolId); + aClass = loadTools(packages).get(toolId); } if (aClass == null) { throw new ToolException("Tool '" + toolId + "' not found"); @@ -121,11 +138,19 @@ public final Class getToolClass(String toolId) throws Too } public Tool getTool(String toolId) throws ToolException { - return getToolClass(toolId).getAnnotation(Tool.class); + return getTool(toolId, Collections.singletonList(DEFAULT_PACKAGE)); + } + + public Tool getTool(String toolId, List packages) throws ToolException { + return getToolClass(toolId, packages).getAnnotation(Tool.class); } public final OpenCgaTool createTool(String toolId) throws ToolException { - return createTool(getToolClass(toolId)); + return createTool(toolId, Collections.singletonList(DEFAULT_PACKAGE)); + } + + public final OpenCgaTool createTool(String toolId, List packages) throws ToolException { + return createTool(getToolClass(toolId, packages)); } public final OpenCgaTool createTool(Class aClass) throws ToolException { @@ -141,12 +166,22 @@ public final OpenCgaTool createTool(Class aClass) throws } public Collection> getTools() { - loadTools(); + loadTools(Collections.singletonList(DEFAULT_PACKAGE)); + return toolsList; + } + + public Collection> getTools(List packages) { + loadTools(packages); return toolsList; } public Map>> getDuplicatedTools() { - loadTools(); + loadTools(Collections.singletonList(DEFAULT_PACKAGE)); + return duplicatedTools; + } + + public Map>> getDuplicatedTools(List packages) { + loadTools(packages); return duplicatedTools; } } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/ToolRunner.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/ToolRunner.java index 88a96004a3f..4a1dd9cf02d 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/ToolRunner.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/ToolRunner.java @@ -16,12 +16,14 @@ package org.opencb.opencga.analysis.tools; +import org.apache.commons.collections4.CollectionUtils; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.CatalogManager; import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.job.Job; import org.opencb.opencga.core.tools.ToolParams; @@ -42,16 +44,30 @@ public class ToolRunner { private final String opencgaHome; private final ToolFactory toolFactory; + private final Configuration configuration; + public ToolRunner(String opencgaHome, CatalogManager catalogManager, StorageEngineFactory storageEngineFactory) { this(opencgaHome, catalogManager, new VariantStorageManager(catalogManager, storageEngineFactory)); } + public ToolRunner(String opencgaHome, CatalogManager catalogManager, StorageEngineFactory storageEngineFactory, + Configuration configuration) { + this(opencgaHome, catalogManager, new VariantStorageManager(catalogManager, storageEngineFactory), configuration); + } + public ToolRunner(String opencgaHome, CatalogManager catalogManager, VariantStorageManager variantStorageManager) { + this(opencgaHome, catalogManager, variantStorageManager, null); + } + + public ToolRunner(String opencgaHome, CatalogManager catalogManager, VariantStorageManager variantStorageManager, + Configuration configuration) { this.opencgaHome = opencgaHome; this.catalogManager = catalogManager; this.variantStorageManager = variantStorageManager; this.toolFactory = new ToolFactory(); + + this.configuration = configuration; } /** @@ -89,73 +105,90 @@ public ExecutionResult execute(Job job, String token) throws CatalogException, T * @throws ToolException if the execution fails */ public ExecutionResult execute(Job job, Path outDir, String token) throws CatalogException, ToolException { - return execute(job.getTool().getId(), new ObjectMap(job.getParams()), outDir, job.getId(), token); + return execute(job.getTool().getId(), new ObjectMap(job.getParams()), outDir, job.getId(), job.isDryRun(), token); } /** * Execute a tool + * * @param toolId Tool identifier. It can be either the tool id itself, or the class name. * @param params Params for the execution. * @param outDir Output directory. Mandatory - * @param token session id of the user that will execute the tool. + * @param dryRun Dry-run mode. + * @param token session id of the user that will execute the tool. * @return Execution result * @throws ToolException if the execution fails */ - public ExecutionResult execute(String toolId, ObjectMap params, Path outDir, String jobId, String token) throws ToolException { - return toolFactory - .createTool(toolId) - .setUp(opencgaHome, catalogManager, variantStorageManager, params, outDir, jobId, token) + public ExecutionResult execute(String toolId, ObjectMap params, Path outDir, String jobId, boolean dryRun, String token) + throws ToolException { + OpenCgaTool tool; + if (configuration != null && configuration.getAnalysis() != null + && CollectionUtils.isNotEmpty(configuration.getAnalysis().getPackages())) { + tool = toolFactory.createTool(toolId, configuration.getAnalysis().getPackages()); + } else { + tool = toolFactory.createTool(toolId); + } + return tool.setUp(opencgaHome, catalogManager, variantStorageManager, params, outDir, jobId, dryRun, token) .start(); } /** * Execute a tool - * @param tool Tool class - * @param study Study id + * + * @param tool Tool class + * @param study Study id * @param toolParams Specific ToolParams for the execution. - * @param outDir Output directory. Mandatory - * @param jobId Job Id (if any) - * @param token session id of the user that will execute the tool. + * @param outDir Output directory. Mandatory + * @param jobId Job Id (if any) + * @param dryRun Dry-run mode. + * @param token session id of the user that will execute the tool. * @return Execution result * @throws ToolException if the execution fails */ - public ExecutionResult execute(Class tool, String study, ToolParams toolParams, Path outDir, String jobId, String token) + public ExecutionResult execute(Class tool, String study, ToolParams toolParams, Path outDir, String jobId, + boolean dryRun, String token) throws ToolException { ObjectMap params = new ObjectMap(); params.putIfNotEmpty(ParamConstants.STUDY_PARAM, study); - return execute(tool, toolParams, params, outDir, jobId, token); + return execute(tool, toolParams, params, outDir, jobId, dryRun, token); } /** * Execute a tool - * @param tool Tool class + * + * @param tool Tool class * @param toolParams Specific ToolParams for the execution. - * @param params Params for the execution. - * @param outDir Output directory. Mandatory - * @param jobId Job Id (if any) - * @param token session id of the user that will execute the tool. + * @param params Params for the execution. + * @param outDir Output directory. Mandatory + * @param jobId Job Id (if any) + * @param dryRun Dry-run mode. + * @param token session id of the user that will execute the tool. * @return Execution result * @throws ToolException if the execution fails */ - public ExecutionResult execute(Class tool, ToolParams toolParams, ObjectMap params, Path outDir, String jobId, String token) + public ExecutionResult execute(Class tool, ToolParams toolParams, ObjectMap params, Path outDir, String jobId, + boolean dryRun, String token) throws ToolException { if (toolParams != null) { params = toolParams.toObjectMap(params); } - return execute(tool, params, outDir, jobId, token); + return execute(tool, params, outDir, jobId, dryRun, token); } /** * Execute a tool - * @param tool Tool class + * + * @param tool Tool class * @param toolParams Specific ToolParams for the execution. - * @param outDir Output directory. Mandatory - * @param jobId Job Id (if any) - * @param token session id of the user that will execute the tool. + * @param outDir Output directory. Mandatory + * @param jobId Job Id (if any) + * @param dryRun Dry-run mode. + * @param token session id of the user that will execute the tool. * @return Execution result * @throws ToolException if the execution fails */ - public ExecutionResult execute(Class tool, ToolParams toolParams, Path outDir, String jobId, String token) + public ExecutionResult execute(Class tool, ToolParams toolParams, Path outDir, String jobId, boolean dryRun, + String token) throws ToolException { ObjectMap params; if (toolParams != null) { @@ -163,23 +196,27 @@ public ExecutionResult execute(Class tool, ToolParams too } else { params = new ObjectMap(); } - return execute(tool, params, outDir, jobId, token); + return execute(tool, params, outDir, jobId, dryRun, token); } /** * Execute a tool - * @param tool Tool class + * + * @param tool Tool class * @param params Params for the execution. * @param outDir Output directory. Mandatory - * @param jobId Job Id (if any) - * @param token session id of the user that will execute the tool. + * @param jobId Job Id (if any) + * @param dryRun + * @param token session id of the user that will execute the tool. * @return Execution result * @throws ToolException if the execution fails */ - public ExecutionResult execute(Class tool, ObjectMap params, Path outDir, String jobId, String token) throws ToolException { + public ExecutionResult execute(Class tool, ObjectMap params, Path outDir, String jobId, boolean dryRun, + String token) throws ToolException { + return toolFactory .createTool(tool) - .setUp(opencgaHome, catalogManager, variantStorageManager, params, outDir, jobId, token) + .setUp(opencgaHome, catalogManager, variantStorageManager, params, outDir, jobId, dryRun, token) .start(); } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/gwas/GwasAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/gwas/GwasAnalysis.java index a5d2484b2d7..270e9e0b213 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/gwas/GwasAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/gwas/GwasAnalysis.java @@ -295,7 +295,7 @@ protected void run() throws ToolException { step("gwas", () -> { GwasAnalysisExecutor gwasExecutor = getToolExecutor(GwasAnalysisExecutor.class); - gwasExecutor.setConfiguration(gwasConfiguration) + gwasExecutor.setGwasConfiguration(gwasConfiguration) .setStudy(study) .setSampleList1(caseCohortSamples) .setSampleList2(controlCohortSamples) diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/julie/JulieTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/julie/JulieTool.java index 5c25bf2a4ff..49ccd95298a 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/julie/JulieTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/julie/JulieTool.java @@ -7,6 +7,7 @@ import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.cohort.Cohort; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.user.User; @@ -24,7 +25,8 @@ resource = Enums.Resource.VARIANT, type = Tool.Type.OPERATION, scope = Tool.Scope.PROJECT, - description = JulieTool.DESCRIPTION) + description = JulieTool.DESCRIPTION, + priority = Enums.Priority.HIGH) public class JulieTool extends OpenCgaTool { public static final String ID = "julie"; @@ -39,8 +41,10 @@ protected void check() throws Exception { String project = getParams().getString(ParamConstants.PROJECT_PARAM); if (StringUtils.isEmpty(project)) { - String userId = getCatalogManager().getUserManager().getUserId(getToken()); - User user = catalogManager.getUserManager().get(userId, null, getToken()).first(); + JwtPayload payload = getCatalogManager().getUserManager().validateToken(getToken()); + String organizationId = payload.getOrganization(); + String userId = payload.getUserId(); + User user = catalogManager.getUserManager().get(organizationId, userId, null, getToken()).first(); if (CollectionUtils.isEmpty(user.getProjects()) || user.getProjects().size() > 1) { throw new CatalogException("Missing '" + ParamConstants.PROJECT_PARAM + "' parameter"); } else { diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/CatalogUtils.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/CatalogUtils.java index e13193675d5..9e35b8c09fb 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/CatalogUtils.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/CatalogUtils.java @@ -24,6 +24,8 @@ import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.CatalogManager; import org.opencb.opencga.catalog.managers.StudyManager; +import org.opencb.opencga.catalog.utils.CatalogFqn; +import org.opencb.opencga.catalog.utils.FqnUtils; import org.opencb.opencga.core.models.project.Project; import org.opencb.opencga.core.models.study.Study; import org.opencb.opencga.storage.core.variant.adaptors.VariantField; @@ -120,7 +122,8 @@ public List getStudies(Query query, String sessionId) throws CatalogExce Collections.singleton(VariantField.STUDIES)); } else { // Get all studies from user. - return catalogManager.getStudyManager().search(new Query(), StudyManager.INCLUDE_STUDY_IDS, sessionId).getResults() + return catalogManager.getStudyManager().searchInOrganization(null, new Query(), StudyManager.INCLUDE_STUDY_IDS, sessionId) + .getResults() .stream() .map(Study::getFqn) .collect(Collectors.toList()); @@ -139,7 +142,9 @@ public Project getProjectFromQuery(Query query, String sessionId, QueryOptions o } else { String studyFqn = getAnyStudy(query, sessionId); String project = catalogManager.getStudyManager().getProjectFqn(studyFqn); - return catalogManager.getProjectManager().search(new Query(ProjectDBAdaptor.QueryParams.FQN.key(), project), options, sessionId) + String organizationId = CatalogFqn.extractFqnFromProjectFqn(project).getOrganizationId(); + return catalogManager.getProjectManager().search(organizationId, + new Query(ProjectDBAdaptor.QueryParams.FQN.key(), project), options, sessionId) .first(); } } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantCatalogQueryUtils.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantCatalogQueryUtils.java index 7a588c900f5..b26a7f6beff 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantCatalogQueryUtils.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantCatalogQueryUtils.java @@ -40,6 +40,7 @@ import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.*; import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.PrivateStudyUid; import org.opencb.opencga.core.models.cohort.Cohort; import org.opencb.opencga.core.models.common.Enums; @@ -279,7 +280,8 @@ public Query parseQuery(Query query, QueryOptions queryOptions, CellBaseUtils ce if (isValidParam(query, SAVED_FILTER)) { String savedFilter = query.getString(SAVED_FILTER.key()); - String userId = catalogManager.getUserManager().getUserId(token); + JwtPayload payload = catalogManager.getUserManager().validateToken(token); + String userId = payload.getUserId(); UserFilter userFilter = catalogManager.getUserManager().getFilter(userId, savedFilter, token).first(); if (!userFilter.getResource().equals(Enums.Resource.VARIANT)) { throw VariantQueryException.malformedParam(SAVED_FILTER, savedFilter, @@ -1607,7 +1609,8 @@ protected List getValuesToValidate(List rawValues) { protected List validate(String defaultStudyStr, List values, Integer release, VariantQueryParam param, String sessionId) throws CatalogException { if (release == null) { - String userId = catalogManager.getUserManager().getUserId(sessionId); + JwtPayload payload = catalogManager.getUserManager().validateToken(sessionId); + String userId = payload.getUserId(); // DataResult samples = catalogManager.getSampleManager().get(defaultStudyStr, values, // SampleManager.INCLUDE_SAMPLE_IDS, sessionId); long numMatches = catalogManager.getSampleManager() diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java index 944bbc9b980..bd71414a4e8 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java @@ -46,7 +46,9 @@ import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.managers.CatalogManager; +import org.opencb.opencga.catalog.managers.ProjectManager; import org.opencb.opencga.catalog.managers.StudyManager; +import org.opencb.opencga.catalog.utils.CatalogFqn; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.cellbase.CellBaseValidator; import org.opencb.opencga.core.common.ExceptionUtils; @@ -54,6 +56,7 @@ import org.opencb.opencga.core.config.storage.CellBaseConfiguration; import org.opencb.opencga.core.config.storage.SampleIndexConfiguration; import org.opencb.opencga.core.config.storage.StorageConfiguration; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.audit.AuditRecord; import org.opencb.opencga.core.models.cohort.Cohort; import org.opencb.opencga.core.models.common.Enums; @@ -68,8 +71,9 @@ import org.opencb.opencga.core.models.sample.SamplePermissions; import org.opencb.opencga.core.models.study.Study; import org.opencb.opencga.core.models.study.StudyPermissions; +import org.opencb.opencga.core.models.study.VariantSetupResult; +import org.opencb.opencga.core.models.variant.VariantSetupParams; import org.opencb.opencga.core.response.OpenCGAResult; -import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.core.tools.ToolParams; import org.opencb.opencga.storage.core.StorageEngineFactory; import org.opencb.opencga.storage.core.StoragePipelineResult; @@ -85,6 +89,7 @@ import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIterator; import org.opencb.opencga.storage.core.variant.io.VariantWriterFactory.VariantOutputFormat; import org.opencb.opencga.storage.core.variant.query.ParsedQuery; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.variant.query.VariantQueryUtils; import org.opencb.opencga.storage.core.variant.query.projection.VariantQueryProjectionParser; import org.opencb.opencga.storage.core.variant.score.VariantScoreFormatDescriptor; @@ -115,11 +120,6 @@ public VariantStorageManager(CatalogManager catalogManager, StorageEngineFactory catalogUtils = new VariantCatalogQueryUtils(catalogManager); } - public void clearCache(String studyId, String type, String token) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(token); - - } - // -------------------------// // Import/Export methods // // -------------------------// @@ -492,6 +492,18 @@ public void aggregate(String studyStr, VariantAggregateParams params, String tok }); } + public VariantSetupResult variantSetup(String studyStr, VariantSetupParams params, String token) + throws CatalogException, StorageEngineException { + return secureOperation(VariantSetupOperationManager.ID, studyStr, params.toObjectMap(), token, + engine -> new VariantSetupOperationManager(this, engine).setup(getStudyFqn(studyStr, token), params, token)); + } + + public boolean hasVariantSetup(String studyStr, String token) throws CatalogException { + Study study = catalogManager.getStudyManager().get(studyStr, + new QueryOptions(INCLUDE, StudyDBAdaptor.QueryParams.INTERNAL_CONFIGURATION_VARIANT_ENGINE.key()), token).first(); + return VariantSetupOperationManager.hasVariantSetup(study); + } + public ObjectMap configureProject(String projectStr, ObjectMap params, String token) throws CatalogException, StorageEngineException { return secureOperationByProject("configure", projectStr, params, token, engine -> { DataStore dataStore = getDataStoreByProjectId(projectStr, token); @@ -565,7 +577,7 @@ public OpenCGAResult configureSampleIndex(String studyStr, SampleIndexConfi /** * Update Cellbase configuration. * - * @param project Study identifier + * @param project Project identifier * @param cellbaseConfiguration New cellbase configuration * @param annotate Launch variant annotation if needed * @param annotationSaveId Save previous variant annotation before annotating @@ -839,7 +851,7 @@ public DataResult getSampleData(String variant, String study, QueryOpti numReadSamples += samplesInResult.size(); StopWatch checkPermissionsStopWatch = StopWatch.createStarted(); - String userId = catalogManager.getUserManager().getUserId(token); + String userId = catalogManager.getUserManager().validateToken(token).getUserId(); List validSamples = catalogManager.getSampleManager() .search(study, new Query(SampleDBAdaptor.QueryParams.ID.key(), samplesInResult) @@ -977,12 +989,8 @@ protected VariantStorageEngine getVariantStorageEngineForStudyOperation(String s .first(); DataStore dataStore = getDataStore(study.getFqn(), token); - VariantStorageEngine variantStorageEngine = storageEngineFactory - .getVariantStorageEngine(dataStore.getStorageEngine(), dataStore.getDbName(), study.getFqn()); - setCellbaseConfiguration(variantStorageEngine, getProjectFqn(null, studyStr, token), token); - if (dataStore.getOptions() != null) { - variantStorageEngine.getOptions().putAll(dataStore.getOptions()); - } + String projectFqn = getProjectFqn(null, studyStr, token); + VariantStorageEngine variantStorageEngine = getVariantStorageEngineByDatastore(dataStore, study.getFqn(), projectFqn, token); if (study.getInternal() != null && study.getInternal().getConfiguration() != null && study.getInternal().getConfiguration().getVariantEngine() != null @@ -1007,16 +1015,25 @@ protected VariantStorageEngine getVariantStorageEngine(String study, String toke protected VariantStorageEngine getVariantStorageEngineByProject(String project, ObjectMap params, String token) throws StorageEngineException, CatalogException { - DataStore dataStore = getDataStoreByProjectId(project, token); + String projectFqn = getProjectFqn(project, Collections.emptyList(), token); + DataStore dataStore = getDataStoreByProjectId(projectFqn, token); + VariantStorageEngine variantStorageEngine = getVariantStorageEngineByDatastore(dataStore, null, projectFqn, token); + if (params != null) { + variantStorageEngine.getOptions().putAll(params); + } + return variantStorageEngine; + } + + private VariantStorageEngine getVariantStorageEngineByDatastore(DataStore dataStore, String alias, String projectFqnStr, String token) throws StorageEngineException, CatalogException { VariantStorageEngine variantStorageEngine = storageEngineFactory - .getVariantStorageEngine(dataStore.getStorageEngine(), dataStore.getDbName()); - setCellbaseConfiguration(variantStorageEngine, project, token); + .getVariantStorageEngine(dataStore.getStorageEngine(), dataStore.getDbName(), alias); + CatalogFqn projectFqn = CatalogFqn.fromProjectFqn(projectFqnStr); + variantStorageEngine.getOptions().put("catalog.organization", projectFqn.getOrganizationId()); + variantStorageEngine.getOptions().put("catalog.project", projectFqn.getProjectId()); + setCellbaseConfiguration(variantStorageEngine, projectFqnStr, token); if (dataStore.getOptions() != null) { variantStorageEngine.getOptions().putAll(dataStore.getOptions()); } - if (params != null) { - variantStorageEngine.getOptions().putAll(params); - } return variantStorageEngine; } @@ -1178,7 +1195,7 @@ private interface VariantOperationFunction { private R secureOperationByProject(String operationName, String project, ObjectMap params, String token, VariantOperationFunction operation) throws CatalogException, StorageEngineException { try (VariantStorageEngine variantStorageEngine = getVariantStorageEngineByProject(project, params, token)) { - return secureTool(operationName, true, params, token, variantStorageEngine, operation); + return secureTool(operationName, true, null, params, token, variantStorageEngine, operation); } catch (IOException e) { throw new StorageEngineException("Error closing the VariantStorageEngine", e); } @@ -1187,7 +1204,7 @@ private R secureOperationByProject(String operationName, String project, Obj private R secureOperation(String operationName, String study, ObjectMap params, String token, VariantOperationFunction operation) throws CatalogException, StorageEngineException { try (VariantStorageEngine variantStorageEngine = getVariantStorageEngineForStudyOperation(study, params, token)) { - return secureTool(operationName, true, params, token, variantStorageEngine, operation); + return secureTool(operationName, true, study, params, token, variantStorageEngine, operation); } catch (IOException e) { throw new StorageEngineException("Error closing the VariantStorageEngine", e); } @@ -1196,7 +1213,7 @@ private R secureOperation(String operationName, String study, ObjectMap para private R secureAnalysis(String operationName, String study, ObjectMap params, String token, VariantOperationFunction operation) throws CatalogException, StorageEngineException { try (VariantStorageEngine variantStorageEngine = getVariantStorageEngineForStudyOperation(study, params, token)) { - return secureTool(operationName, false, params, token, variantStorageEngine, operation); + return secureTool(operationName, false, study, params, token, variantStorageEngine, operation); } catch (IOException e) { throw new StorageEngineException("Error closing the VariantStorageEngine", e); } @@ -1218,7 +1235,7 @@ private R secureOperationByProject(String operationName, String projectStr, return secureOperationByProject(operationName, projectStr, params, token, operation); } - private R secureTool(String toolId, boolean isOperation, ObjectMap params, String token, + private R secureTool(String toolId, boolean isOperation, String study, ObjectMap params, String token, VariantStorageEngine variantStorageEngine, VariantOperationFunction operation) throws CatalogException, StorageEngineException { @@ -1229,7 +1246,7 @@ private R secureTool(String toolId, boolean isOperation, ObjectMap params, S // deprecated .append("operationName", toolId); R result = null; - String userId = catalogManager.getUserManager().getUserId(token); + JwtPayload payload = catalogManager.getUserManager().validateToken(token); Exception exception = null; StopWatch totalStopWatch = StopWatch.createStarted(); @@ -1238,6 +1255,15 @@ private R secureTool(String toolId, boolean isOperation, ObjectMap params, S throw new StorageEngineException("Unable to execute operation '" + toolId + "'. " + "The storage engine is in mode=" + storageConfiguration.getMode()); } + if (isOperation && study != null && !VariantSetupOperationManager.ID.equals(toolId)) { + // Ensure that the variant setup has been executed + // do not check for the setup operation itself + // Project level operations can not be checked for setup. + if (!hasVariantSetup(study, token)) { + throw new StorageEngineException("Unable to execute operation '" + toolId + "'. " + + "The variant storage has not been setup for study '" + study + "'"); + } + } result = operation.apply(variantStorageEngine); return result; } catch (CatalogException | StorageEngineException e) { @@ -1266,7 +1292,9 @@ private R secureTool(String toolId, boolean isOperation, ObjectMap params, S } logger.debug("dbTime = " + auditAttributes.getInt("dbTime")); logger.debug("totalTimeMillis = " + auditAttributes.getInt("totalTimeMillis")); - catalogManager.getAuditManager().audit(userId, Enums.Action.VARIANT_STORAGE_OPERATION, Enums.Resource.VARIANT, + String organizationId = variantStorageEngine.getOptions().getString("catalog.organization"); + String userId = payload.getUserId(organizationId); + catalogManager.getAuditManager().audit(organizationId, userId, Enums.Action.VARIANT_STORAGE_OPERATION, Enums.Resource.VARIANT, "", "", "", "", params, status, @@ -1290,13 +1318,12 @@ private R secure(Query query, QueryOptions queryOptions, String token, Enums .append("query", new Query(query)) .append("queryOptions", new QueryOptions(queryOptions)); R result = null; - String userId = catalogManager.getUserManager().getUserId(token); + JwtPayload payload = catalogManager.getUserManager().validateToken(token); Exception exception = null; StopWatch totalStopWatch = StopWatch.createStarted(); StopWatch storageStopWatch = null; + VariantStorageEngine variantStorageEngine = getVariantStorageEngine(query, token); try { - VariantStorageEngine variantStorageEngine = getVariantStorageEngine(query, token); - StopWatch stopWatch = StopWatch.createStarted(); query = catalogUtils.parseQuery(query, queryOptions, variantStorageEngine.getCellBaseUtils(), token); auditAttributes.append("catalogParseQueryTimeMillis", stopWatch.getTime(TimeUnit.MILLISECONDS)); @@ -1339,7 +1366,9 @@ private R secure(Query query, QueryOptions queryOptions, String token, Enums logger.debug("storageTimeMillis = " + auditAttributes.getInt("storageTimeMillis")); logger.debug("dbTime = " + auditAttributes.getInt("dbTime")); logger.debug("totalTimeMillis = " + auditAttributes.getInt("totalTimeMillis")); - catalogManager.getAuditManager().audit(userId, auditAction, Enums.Resource.VARIANT, "", "", "", "", new ObjectMap(), + String organizationId = variantStorageEngine.getOptions().getString("catalog.organization"); + String userId = payload.getUserId(organizationId); + catalogManager.getAuditManager().audit(organizationId, userId, auditAction, Enums.Resource.VARIANT, "", "", "", "", new ObjectMap(), status, auditAttributes); } } @@ -1360,7 +1389,7 @@ Map> checkSamplesPermissions(Query query, QueryOptions quer Enums.Action auditAction, String token) throws CatalogException { final Map> samplesMap = new HashMap<>(); - String userId = catalogManager.getUserManager().getUserId(token); + String userId = catalogManager.getUserManager().validateToken(token).getUserId(); Set returnedFields = VariantField.getIncludeFields(queryOptions); if (auditAction == Enums.Action.FACET) { if (VariantQueryProjectionParser.isIncludeNoSamples(query, VariantField.all())) { @@ -1488,13 +1517,17 @@ private void checkStudyPermissions(Collection studies, String userId, St } private void checkStudyPermissions(String study, String userId, String token) throws CatalogException { + JwtPayload payload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(study, payload); + String organizationId = studyFqn.getOrganizationId(); + long studyUid = catalogManager.getStudyManager().get(study, StudyManager.INCLUDE_STUDY_IDS, token).first().getUid(); CatalogAuthorizationException exception = null; // Check VIEW_AGGREGATED_VARIANTS try { catalogManager.getAuthorizationManager() - .checkStudyPermission(studyUid, userId, StudyPermissions.Permissions.VIEW_AGGREGATED_VARIANTS); + .checkStudyPermission(organizationId, studyUid, userId, StudyPermissions.Permissions.VIEW_AGGREGATED_VARIANTS); return; } catch (CatalogAuthorizationException e) { exception = e; @@ -1503,7 +1536,7 @@ private void checkStudyPermissions(String study, String userId, String token) th // Check VIEW_SAMPLE_VARIANTS try { catalogManager.getAuthorizationManager() - .checkStudyPermission(studyUid, userId, StudyPermissions.Permissions.VIEW_SAMPLE_VARIANTS); + .checkStudyPermission(organizationId, studyUid, userId, StudyPermissions.Permissions.VIEW_SAMPLE_VARIANTS); return; } catch (CatalogAuthorizationException e) { // Ignore this exception. Throw exception of missing VIEW_AGGREGATED_VARIANTS @@ -1638,15 +1671,7 @@ private String getProjectFqn(String projectStr, String study, String token) thro } private String getProjectFqn(String projectStr, List studies, String token) throws CatalogException { - if (CollectionUtils.isEmpty(studies) && StringUtils.isEmpty(projectStr)) { - List projects = catalogManager.getProjectManager().search(new Query(), new QueryOptions(), token).getResults(); - if (projects.size() == 1) { - projectStr = projects.get(0).getFqn(); - } else { - throw new IllegalArgumentException("Expected either studies or project to annotate"); - } - } - + final String projectFqn; if (CollectionUtils.isNotEmpty(studies)) { // Ensure all studies are valid. Convert to FQN studies = catalogManager.getStudyManager() @@ -1656,7 +1681,11 @@ private String getProjectFqn(String projectStr, List studies, String tok .map(Study::getFqn) .collect(Collectors.toList()); - projectStr = catalogManager.getStudyManager().getProjectFqn(studies.get(0)); + if (StringUtils.isEmpty(projectStr)) { + projectFqn = catalogManager.getStudyManager().getProjectFqn(studies.get(0)); + } else { + projectFqn = catalogManager.getProjectManager().get(projectStr, ProjectManager.INCLUDE_PROJECT_IDS, token).first().getFqn(); + } if (studies.size() > 1) { for (String studyStr : studies) { @@ -1665,8 +1694,22 @@ private String getProjectFqn(String projectStr, List studies, String tok } } } + } else if (StringUtils.isNotEmpty(projectStr)) { + projectFqn = catalogManager.getProjectManager().get(projectStr, ProjectManager.INCLUDE_PROJECT_IDS, token).first().getFqn(); + } else { + // Extract organization from token + JwtPayload jwtPayload = new JwtPayload(token); + String organizationId = jwtPayload.getOrganization(); + // Look for projects from own organization + List projects = catalogManager.getProjectManager().search(organizationId, new Query(), new QueryOptions(), token) + .getResults(); + if (projects.size() == 1) { + projectFqn = projects.get(0).getFqn(); + } else { + throw new IllegalArgumentException("Expected either studies or project to annotate"); + } } - return projectStr; + return projectFqn; } public DataStore getDataStore(String study, String token) throws CatalogException { @@ -1701,7 +1744,7 @@ public static DataStore getDataStoreByProjectId(CatalogManager catalogManager, S } if (dataStore == null) { //get default datastore - dataStore = defaultDataStore(catalogManager, project, token); + dataStore = defaultDataStore(catalogManager, project); } if (dataStore.getOptions() == null) { dataStore.setOptions(new ObjectMap()); @@ -1710,24 +1753,22 @@ public static DataStore getDataStoreByProjectId(CatalogManager catalogManager, S return dataStore; } - public static DataStore defaultDataStore(CatalogManager catalogManager, Project project, String token) throws CatalogException { - return defaultDataStore(catalogManager, project, catalogManager.getConfiguration().getDatabasePrefix(), token); + public static DataStore defaultDataStore(CatalogManager catalogManager, Project project) throws CatalogException { + return defaultDataStore(catalogManager.getConfiguration().getDatabasePrefix(), project.getFqn()); } - public static DataStore defaultDataStore(CatalogManager catalogManager, Project project, String databasePrefix, String token) + public static DataStore defaultDataStore(String databasePrefix, String projectFqnStr) throws CatalogException { - DataStore dataStore; - //Must use the UserByStudyId instead of the file owner. - String userId = catalogManager.getProjectManager().getOwner(project.getUid()); - // Replace possible dots at the userId. Usually a special character in almost all databases. See #532 - userId = userId.replace('.', '_'); - - String dbName = buildDatabaseName(databasePrefix, userId, project.getId()); - dataStore = new DataStore(StorageEngineFactory.get().getDefaultStorageEngineId(), dbName); - return dataStore; + CatalogFqn projectFqn = CatalogFqn.extractFqnFromProjectFqn(projectFqnStr); + + String dbName = buildDatabaseName(databasePrefix, projectFqn.getOrganizationId(), projectFqn.getProjectId()); + return new DataStore(StorageEngineFactory.get().getDefaultStorageEngineId(), dbName); } - public static String buildDatabaseName(String databasePrefix, String userId, String projectId) { + public static String buildDatabaseName(String databasePrefix, String organizationId, String projectId) { + // Replace possible dots at the organization. Usually a special character in almost all databases. See #532 + organizationId = organizationId.replace('.', '_'); + String prefix; if (StringUtils.isNotEmpty(databasePrefix)) { prefix = databasePrefix; @@ -1737,14 +1778,14 @@ public static String buildDatabaseName(String databasePrefix, String userId, Str } else { prefix = "opencga_"; } - // Project id might contain the userId: - // userId@projectId + // Project id might contain the organization: + // organization@projectId int idx = projectId.indexOf('@'); if (idx >= 0) { projectId = projectId.substring(idx + 1); } - return prefix + userId + '_' + projectId; + return prefix + organizationId + '_' + projectId; } } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantDeleteOperationManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantDeleteOperationManager.java index 882240d2865..c0f57f0b04f 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantDeleteOperationManager.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantDeleteOperationManager.java @@ -27,6 +27,8 @@ import org.opencb.opencga.storage.core.metadata.models.TaskMetadata; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.net.URI; import java.util.ArrayList; @@ -34,6 +36,8 @@ public class VariantDeleteOperationManager extends OperationManager { + private final Logger logger = LoggerFactory.getLogger(VariantDeleteOperationManager.class); + public VariantDeleteOperationManager(VariantStorageManager variantStorageManager, VariantStorageEngine engine) { super(variantStorageManager, engine); } @@ -63,7 +67,14 @@ public void removeFile(String study, List inputFiles, URI outdir, String String catalogIndexStatus = file.getInternal().getVariant().getIndex().getStatus().getId(); if (!catalogIndexStatus.equals(VariantIndexStatus.READY)) { // Might be partially loaded in VariantStorage. Check FileMetadata - FileMetadata fileMetadata = variantStorageEngine.getMetadataManager().getFileMetadata(studyMetadata.getId(), fileStr); + FileMetadata fileMetadata = variantStorageEngine.getMetadataManager() + .getFileMetadata(studyMetadata.getId(), file.getName()); + if (fileMetadata != null && !fileMetadata.getPath().equals(file.getUri().getPath())) { + // FileMetadata path does not match the catalog path. This file is not registered in the storage. + throw new CatalogException("Unable to remove variants from file '" + file.getPath() + "'. " + + "File is not registered in the storage. " + + "Instead, found file with same name but different path '" + fileMetadata.getPath() + "'"); + } boolean canBeRemoved; if (force) { // When forcing remove, just require the file to be registered in the storage @@ -73,17 +84,18 @@ public void removeFile(String study, List inputFiles, URI outdir, String canBeRemoved = fileMetadata != null && fileMetadata.getIndexStatus() != TaskMetadata.Status.NONE; } if (!canBeRemoved) { - throw new CatalogException("Unable to remove variants from file " + file.getName() + ". " - + "IndexStatus = " + catalogIndexStatus); + throw new CatalogException("Unable to remove variants from file '" + file.getPath() + "'. " + + "IndexStatus = " + catalogIndexStatus + "." + + (fileMetadata == null ? " File not found in storage." : "")); } } fileNames.add(file.getName()); // filePaths.add(file.getPath()); } - if (fileNames.isEmpty()) { - throw new CatalogException("Nothing to do!"); - } + } + if (fileNames.isEmpty()) { + throw new CatalogException("Nothing to do!"); } variantStorageEngine.removeFiles(study, fileNames, outdir); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantFileIndexerOperationManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantFileIndexerOperationManager.java index 94bfada60d5..55e40d9dddb 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantFileIndexerOperationManager.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantFileIndexerOperationManager.java @@ -34,7 +34,9 @@ import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.FileUtils; +import org.opencb.opencga.catalog.utils.CatalogFqn; import org.opencb.opencga.core.common.UriUtils; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.cohort.Cohort; import org.opencb.opencga.core.models.cohort.CohortCreateParams; import org.opencb.opencga.core.models.cohort.CohortStatus; @@ -86,6 +88,7 @@ public class VariantFileIndexerOperationManager extends OperationManager { private final Logger logger; private String studyFqn; + private String organizationId; private List files; private boolean calculateStats; @@ -128,6 +131,10 @@ public List index(String study, List files, URI o private void check(String study, ObjectMap params, String token) throws Exception { studyFqn = getStudyFqn(study, token); + JwtPayload jwtPayload = new JwtPayload(token); + CatalogFqn catalogFqn = CatalogFqn.extractFqnFromStudy(studyFqn, jwtPayload); + this.organizationId = catalogFqn.getOrganizationId(); + this.keepIntermediateFiles = params.getBoolean(OperationTool.KEEP_INTERMEDIATE_FILES); transform = true; @@ -184,8 +191,7 @@ private void updateProject(String studyFqn, String token) throws CatalogExceptio * @throws URISyntaxException * @throws StorageEngineException */ - private List findFilesToIndex(ObjectMap params, String token) - throws CatalogException, URISyntaxException, StorageEngineException { + private List findFilesToIndex(ObjectMap params, String token) throws CatalogException, URISyntaxException, StorageEngineException { synchronizer = new CatalogStorageMetadataSynchronizer(catalogManager, variantStorageEngine.getMetadataManager()); List inputFiles = new ArrayList<>(); @@ -279,7 +285,7 @@ private List findFilesToIndex(ObjectMap params, String token) FileInternalVariantIndex index = file.getInternal().getVariant().getIndex(); index.setRelease(release); index.setStatus(new VariantIndexStatus(fileStatus, fileStatusMessage)); - catalogManager.getFileManager().updateFileInternalVariantIndex(file, index, token); + catalogManager.getFileManager().updateFileInternalVariantIndex(studyFqn, file, index, token); } } return fileUris; @@ -466,7 +472,7 @@ private void updateFileInfo(String study, List filesToIndex, VariantReader // Update index status index.setRelease(release); index.setStatus(new VariantIndexStatus(indexStatusId, indexStatusMessage)); - catalogManager.getFileManager().updateFileInternalVariantIndex(indexedFile, index, sessionId); + catalogManager.getFileManager().updateFileInternalVariantIndex(studyFqn, indexedFile, index, sessionId); boolean calculateStats = options.getBoolean(VariantStorageOptions.STATS_CALCULATE.key()); if (indexStatusId.equals(VariantIndexStatus.READY) && calculateStats) { diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantSetupOperationManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantSetupOperationManager.java new file mode 100644 index 00000000000..0003ca105bf --- /dev/null +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantSetupOperationManager.java @@ -0,0 +1,166 @@ +package org.opencb.opencga.analysis.variant.manager.operations; + +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; +import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.models.study.Study; +import org.opencb.opencga.core.models.study.VariantSetupResult; +import org.opencb.opencga.core.models.variant.VariantSetupParams; +import org.opencb.opencga.storage.core.exceptions.StorageEngineException; +import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; +import org.opencb.opencga.storage.core.variant.VariantStorageEngine; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class VariantSetupOperationManager extends OperationManager { + + + public static final String ID = "variant-setup"; + private static Logger logger = LoggerFactory.getLogger(VariantSetupOperationManager.class); + + public VariantSetupOperationManager(VariantStorageManager variantStorageManager, VariantStorageEngine variantStorageEngine) { + super(variantStorageManager, variantStorageEngine); + } + + public VariantSetupResult setup(String studyFqn, VariantSetupParams params, String token) + throws CatalogException, StorageEngineException { + // Copy params to avoid modifying input object + params = new VariantSetupParams(params); + check(studyFqn, params, token); + + VariantSetupResult result = new VariantSetupResult(); + result.setDate(TimeUtils.getTime()); + result.setUserId(catalogManager.getUserManager().getUserIdContextStudy(studyFqn, token)); + result.setParams(params.toObjectMap()); + result.setStatus(VariantSetupResult.Status.READY); + + inferParams(params); + + ObjectMap options = variantStorageEngine.inferConfigurationParams(params); + result.setOptions(options); + + catalogManager.getStudyManager().setVariantEngineSetupOptions(studyFqn, result, token); + + return result; + } + + /** + * Infer some parameters from others. + * - averageFileSize inferred from fileType + * - samplesPerFile inferred from dataDistribution or expectedSamplesNumber and expectedFilesNumber + * - numberOfVariantsPerSample inferred from fileType + * @param params params to infer + */ + private void inferParams(VariantSetupParams params) { + if (params.getFileType() != null) { + switch (params.getFileType()) { + case GENOME_gVCF: + if (params.getAverageFileSize() == null) { + params.setAverageFileSize("1GiB"); + } + if (params.getVariantsPerSample() == null) { + params.setVariantsPerSample(5000000); + } + break; + case GENOME_VCF: + if (params.getAverageFileSize() == null) { + params.setAverageFileSize("500MiB"); + } + if (params.getVariantsPerSample() == null) { + params.setVariantsPerSample(5000000); + } + break; + case EXOME: + if (params.getAverageFileSize() == null) { + params.setAverageFileSize("100MiB"); + } + if (params.getVariantsPerSample() == null) { + params.setVariantsPerSample(100000); + } + break; + default: + throw new IllegalArgumentException("Unknown fileType " + params.getFileType()); + } + } + // Unable to tell. Use a default value for numberOfVariantsPerSample + if (params.getVariantsPerSample() == null) { + params.setVariantsPerSample(5000000); + } + + if (params.getAverageSamplesPerFile() == null) { + if (params.getDataDistribution() == null) { + params.setAverageSamplesPerFile(params.getExpectedSamples().floatValue() / params.getExpectedFiles().floatValue()); + } else { + switch (params.getDataDistribution()) { + case SINGLE_SAMPLE_PER_FILE: + params.setAverageSamplesPerFile(1f); + break; + case MULTIPLE_SAMPLES_PER_FILE: + params.setAverageSamplesPerFile(params.getExpectedSamples().floatValue() / params.getExpectedFiles().floatValue()); + break; + case MULTIPLE_FILES_PER_SAMPLE: + // Hard to tell. Let's assume 2 samples per file + params.setAverageSamplesPerFile(2f); + break; + case FILES_SPLIT_BY_CHROMOSOME: + case FILES_SPLIT_BY_REGION: + params.setAverageSamplesPerFile(params.getExpectedSamples().floatValue()); + break; + default: + throw new IllegalArgumentException("Unknown dataDistribution " + params.getDataDistribution()); + } + } + } + } + + private void check(String studyStr, VariantSetupParams params, String token) throws CatalogException, StorageEngineException { + Study study = catalogManager.getStudyManager().get(studyStr, + new QueryOptions(QueryOptions.INCLUDE, StudyDBAdaptor.QueryParams.INTERNAL_CONFIGURATION_VARIANT_ENGINE.key()), token) + .first(); + + VariantStorageMetadataManager metadataManager = variantStorageEngine.getMetadataManager(); + if (metadataManager.studyExists(studyStr)) { + int studyId = metadataManager.getStudyId(studyStr); + if (!metadataManager.getIndexedFiles(studyId).isEmpty()) { + throw new IllegalArgumentException("Unable to execute variant-setup on study '" + studyStr + "'. " + + "It already has indexed files."); + } + } + if (hasVariantSetup(study)) { + logger.info("Study {} was already setup. Re executing variant-setup", studyStr); + } + + if (params.getExpectedFiles() == null || params.getExpectedFiles() <= 0) { + throw new IllegalArgumentException("Missing expectedFiles"); + } + if (params.getExpectedSamples() == null || params.getExpectedSamples() <= 0) { + throw new IllegalArgumentException("Missing expectedSamples"); + } + + if (params.getAverageFileSize() == null && params.getFileType() == null) { + throw new IllegalArgumentException("Missing averageFileSize or fileType"); + } + } + + public static boolean hasVariantSetup(Study study) { + boolean hasSetup = false; + VariantSetupResult setup = getVariantSetupResult(study); + if (setup != null && setup.getStatus() == VariantSetupResult.Status.READY) { + hasSetup = true; + } + return hasSetup; + } + + private static VariantSetupResult getVariantSetupResult(Study study) { + if (study.getInternal() != null + && study.getInternal().getConfiguration() != null + && study.getInternal().getConfiguration().getVariantEngine() != null) { + return study.getInternal().getConfiguration().getVariantEngine().getSetup(); + } + return null; + } + +} diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/metadata/CatalogStorageMetadataSynchronizer.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/metadata/CatalogStorageMetadataSynchronizer.java index 3d97af2c238..bb9b7fff211 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/metadata/CatalogStorageMetadataSynchronizer.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/metadata/CatalogStorageMetadataSynchronizer.java @@ -623,7 +623,7 @@ protected boolean synchronizeFiles(StudyMetadata study, List files, String logger.info("File \"{}\" change status from {} to {}", file.getName(), VariantIndexStatus.READY, newStatus); index.setStatus(new VariantIndexStatus(newStatus, "Not indexed, regarding Storage Metadata")); - catalogManager.getFileManager().updateFileInternalVariantIndex(file, index, token); + catalogManager.getFileManager().updateFileInternalVariantIndex(study.getName(), file, index, token); modified = true; } } @@ -682,7 +682,7 @@ protected boolean synchronizeFiles(StudyMetadata study, List files, String prevStatus, newStatus); index.setStatus(new VariantIndexStatus(newStatus, "Error loading. Reset status to " + newStatus)); - catalogManager.getFileManager().updateFileInternalVariantIndex(file, index, token); + catalogManager.getFileManager().updateFileInternalVariantIndex(study.getName(), file, index, token); modified = true; } else { // Running job. Might be transforming, or have just started. Do not modify the status! @@ -723,7 +723,7 @@ protected boolean synchronizeFiles(StudyMetadata study, List files, String newStatus = VariantIndexStatus.INDEXING; } index.setStatus(new VariantIndexStatus(newStatus, "File is being loaded regarding Storage")); - catalogManager.getFileManager().updateFileInternalVariantIndex(file, index, token); + catalogManager.getFileManager().updateFileInternalVariantIndex(study.getName(), file, index, token); modified = true; } } @@ -749,7 +749,7 @@ private boolean synchronizeIndexedFile(StudyMetadata study, File file, Map samp SAMPLE_QUERY_OPTIONS, token)) { while (iterator.hasNext()) { Sample sample = iterator.next(); - if (synchronizeSample(sampleMetadataMap.get(sample.getId()), sample, sampleIndexVersion, token)) { + if (synchronizeSample(study, sampleMetadataMap.get(sample.getId()), sample, sampleIndexVersion, token)) { modified = true; modifiedSamples++; } @@ -831,8 +831,8 @@ private boolean synchronizeSamples(StudyMetadata study, Collection samp return modified; } - private boolean synchronizeSample(SampleMetadata sampleMetadata, Sample sample, int lastSampleIndexVersion, String token) - throws CatalogException { + private boolean synchronizeSample(StudyMetadata study, SampleMetadata sampleMetadata, Sample sample, int lastSampleIndexVersion, + String token) throws CatalogException { boolean modified = false; String catalogIndexStatus = secureGet(sample, s -> s.getInternal().getVariant().getIndex().getStatus().getId(), null); @@ -842,7 +842,7 @@ private boolean synchronizeSample(SampleMetadata sampleMetadata, Sample sample, || catalogNumFiles != sampleMetadata.getFiles().size() || catalogMultiFile != sampleMetadata.isMultiFileSample()) { catalogManager.getSampleManager() - .updateSampleInternalVariantIndex(sample, + .updateSampleInternalVariantIndex(study.getName(), sample, new SampleInternalVariantIndex( new IndexStatus(sampleMetadata.getIndexStatus().name()), sampleMetadata.getFiles().size(), @@ -852,7 +852,7 @@ private boolean synchronizeSample(SampleMetadata sampleMetadata, Sample sample, String catalogAnnotationIndexStatus = secureGet(sample, s -> s.getInternal().getVariant().getAnnotationIndex().getStatus().getId(), null); if (!sampleMetadata.getAnnotationStatus().name().equals(catalogAnnotationIndexStatus)) { catalogManager.getSampleManager() - .updateSampleInternalVariantAnnotationIndex(sample, + .updateSampleInternalVariantAnnotationIndex(study.getName(), sample, new SampleInternalVariantAnnotationIndex( new IndexStatus(sampleMetadata.getAnnotationStatus().name())), token); modified = true; @@ -862,7 +862,7 @@ private boolean synchronizeSample(SampleMetadata sampleMetadata, Sample sample, s -> s.getInternal().getVariant().getSecondaryAnnotationIndex().getStatus().getId(), null); if (!sampleMetadata.getSecondaryAnnotationIndexStatus().name().equals(catalogSecondaryAnnotationIndexStatus)) { catalogManager.getSampleManager() - .updateSampleInternalVariantSecondaryAnnotationIndex(sample, + .updateSampleInternalVariantSecondaryAnnotationIndex(study.getName(), sample, new SampleInternalVariantSecondaryAnnotationIndex( new IndexStatus(sampleMetadata.getSecondaryAnnotationIndexStatus().name())), token); modified = true; @@ -930,7 +930,7 @@ private boolean synchronizeSample(SampleMetadata sampleMetadata, Sample sample, if (catalogVariantSecondarySampleIndexModified) { catalogManager.getSampleManager() - .updateSampleInternalVariantSecondarySampleIndex(sample, catalogVariantSecondarySampleIndex, token); + .updateSampleInternalVariantSecondarySampleIndex(study.getName(), sample, catalogVariantSecondarySampleIndex, token); modified = true; } return modified; @@ -950,7 +950,7 @@ public void synchronizeRemovedStudyFromStorage(String study, String token) throw .iterator(study, INDEXED_FILES_QUERY, INDEXED_FILES_QUERY_OPTIONS, token)) { while (iterator.hasNext()) { File file = iterator.next(); - catalogManager.getFileManager().updateFileInternalVariantIndex(file, FileInternalVariantIndex.init() + catalogManager.getFileManager().updateFileInternalVariantIndex(study, file, FileInternalVariantIndex.init() .setStatus(new VariantIndexStatus(VariantIndexStatus.NONE)), token); } } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/metadata/CatalogVariantMetadataFactory.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/metadata/CatalogVariantMetadataFactory.java index cbffe4e9e21..1dde494f3d0 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/metadata/CatalogVariantMetadataFactory.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/metadata/CatalogVariantMetadataFactory.java @@ -164,7 +164,7 @@ private void fillSamples(String studyId, List(sample.getAnnotations())); for (AnnotationSet annotationSet : annotationSets) { - String prefix = annotationSets.size() > 1 ? annotationSet.getName() + '.' : ""; + String prefix = annotationSets.size() > 1 ? annotationSet.getId() + '.' : ""; Map annotations = annotationSet.getAnnotations(); for (Map.Entry annotationEntry : annotations.entrySet()) { Object value = annotationEntry.getValue(); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureAnalysis.java index c81b966dcaf..56a95cfa92e 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureAnalysis.java @@ -51,9 +51,6 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.List; -import java.util.Locale; - -import static org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam.STUDY; @Tool(id = MutationalSignatureAnalysis.ID, resource = Enums.Resource.VARIANT) public class MutationalSignatureAnalysis extends OpenCgaToolScopeStudy { diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAggregateFamilyOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAggregateFamilyOperationTool.java index c00f4943e08..57efbe0ef60 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAggregateFamilyOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAggregateFamilyOperationTool.java @@ -25,7 +25,8 @@ @Tool(id = VariantAggregateFamilyOperationTool.ID, description = VariantAggregateFamilyOperationTool.DESCRIPTION, type = Tool.Type.OPERATION, - resource = Enums.Resource.VARIANT) + resource = Enums.Resource.VARIANT, + priority = Enums.Priority.HIGH) public class VariantAggregateFamilyOperationTool extends OperationTool { public static final String ID = "variant-aggregate-family"; public static final String DESCRIPTION = "Find variants where not all the samples are present, and fill the empty values."; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAggregateOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAggregateOperationTool.java index 49f799500bd..ab28fd9dea4 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAggregateOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAggregateOperationTool.java @@ -16,14 +16,13 @@ package org.opencb.opencga.analysis.variant.operations; -import org.opencb.opencga.core.tools.annotations.Tool; -import org.opencb.opencga.core.models.operations.variant.VariantAggregateParams; import org.opencb.opencga.core.models.common.Enums; +import org.opencb.opencga.core.models.operations.variant.VariantAggregateParams; +import org.opencb.opencga.core.tools.annotations.Tool; import org.opencb.opencga.core.tools.annotations.ToolParams; -import org.opencb.opencga.storage.core.variant.VariantStorageOptions; @Tool(id = VariantAggregateOperationTool.ID, description = VariantAggregateOperationTool.DESCRIPTION, - type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT) + type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT, priority = Enums.Priority.HIGH) public class VariantAggregateOperationTool extends OperationTool { public static final String ID = "variant-aggregate"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAnnotationDeleteOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAnnotationDeleteOperationTool.java index 07f5e396ef7..47d36b85333 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAnnotationDeleteOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAnnotationDeleteOperationTool.java @@ -24,7 +24,7 @@ @Tool(id = VariantAnnotationDeleteOperationTool.ID, description = VariantAnnotationDeleteOperationTool.DESCRIPTION, type = Tool.Type.OPERATION, scope = Tool.Scope.PROJECT, - resource = Enums.Resource.VARIANT) + resource = Enums.Resource.VARIANT, priority = Enums.Priority.HIGH) public class VariantAnnotationDeleteOperationTool extends OperationTool { public static final String ID = "variant-annotation-delete"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAnnotationIndexOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAnnotationIndexOperationTool.java index d3405dcaecd..aa08de950c7 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAnnotationIndexOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAnnotationIndexOperationTool.java @@ -32,7 +32,8 @@ @Tool(id = VariantAnnotationIndexOperationTool.ID, description = VariantAnnotationIndexOperationTool.DESCRIPTION, type = Tool.Type.OPERATION, scope = Tool.Scope.PROJECT, - resource = Enums.Resource.VARIANT) + resource = Enums.Resource.VARIANT, + priority = Enums.Priority.HIGH) public class VariantAnnotationIndexOperationTool extends OperationTool { public static final String ID = "variant-annotation-index"; public static final String DESCRIPTION = "Create and load variant annotations into the database"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAnnotationRebuilderOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAnnotationRebuilderOperationTool.java index 891838e34b3..b684d612519 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAnnotationRebuilderOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAnnotationRebuilderOperationTool.java @@ -22,7 +22,8 @@ @Tool(id = VariantAnnotationRebuilderOperationTool.ID, description = VariantAnnotationRebuilderOperationTool.ID, type = Tool.Type.OPERATION, scope = Tool.Scope.PROJECT, - resource = Enums.Resource.VARIANT) + resource = Enums.Resource.VARIANT, + priority = Enums.Priority.HIGH) public class VariantAnnotationRebuilderOperationTool extends OperationTool { public static final String ID = "variant-annotation-rebuilder"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAnnotationSaveOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAnnotationSaveOperationTool.java index 33e400bcb12..9f7994935b0 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAnnotationSaveOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAnnotationSaveOperationTool.java @@ -23,7 +23,8 @@ @Tool(id = VariantAnnotationSaveOperationTool.ID, description = VariantAnnotationSaveOperationTool.DESCRIPTION, type = Tool.Type.OPERATION, scope = Tool.Scope.PROJECT, - resource = Enums.Resource.VARIANT) + resource = Enums.Resource.VARIANT, + priority = Enums.Priority.HIGH) public class VariantAnnotationSaveOperationTool extends OperationTool { public static final String ID = "variant-annotation-save"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantFamilyIndexOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantFamilyIndexOperationTool.java index c7258c79f74..65dc97676cb 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantFamilyIndexOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantFamilyIndexOperationTool.java @@ -31,7 +31,7 @@ @Deprecated @Tool(id = VariantFamilyIndexOperationTool.ID, description = VariantFamilyIndexOperationTool.DESCRIPTION, - type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT) + type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT, priority = Enums.Priority.HIGH) public class VariantFamilyIndexOperationTool extends OperationTool { public static final String ID = "variant-family-index"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantFileDeleteOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantFileDeleteOperationTool.java index 85b4f8b1b48..db2da6c58e4 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantFileDeleteOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantFileDeleteOperationTool.java @@ -33,7 +33,7 @@ * @author Jacobo Coll <jacobo167@gmail.com> */ @Tool(id = VariantFileDeleteOperationTool.ID, description = VariantFileDeleteOperationTool.DESCRIPTION, - type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT) + type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT, priority = Enums.Priority.HIGH) public class VariantFileDeleteOperationTool extends OperationTool { public static final String ID = "variant-file-delete"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantFileIndexJobLauncherTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantFileIndexJobLauncherTool.java index 3dd3f5584ea..f45b7011650 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantFileIndexJobLauncherTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantFileIndexJobLauncherTool.java @@ -31,7 +31,7 @@ import static org.opencb.opencga.catalog.db.api.FileDBAdaptor.QueryParams.*; @Tool(id = VariantFileIndexJobLauncherTool.ID, description = VariantFileIndexJobLauncherTool.DESCRIPTION, - type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT) + type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT, priority = Enums.Priority.HIGH) public class VariantFileIndexJobLauncherTool extends OpenCgaToolScopeStudy { public static final String ID = "variant-index-job-launcher"; @@ -142,7 +142,7 @@ protected void run() throws Exception { String jobId = buildJobId(file); Job job = catalogManager.getJobManager().submit(getStudy(), VariantIndexOperationTool.ID, Enums.Priority.MEDIUM, indexParams.toParams(new ObjectMap(ParamConstants.STUDY_PARAM, study)), jobId, "Job generated by " + getId(), - Collections.emptyList(), jobTags, getToken()).first(); + Collections.emptyList(), jobTags, getJobId(), null, false, getToken()).first(); submittedJobs++; logger.info("[{}] Create variant-index job '{}' for file '{}'{}", submittedJobs, diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantIndexOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantIndexOperationTool.java index 70a882d4d80..766962db671 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantIndexOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantIndexOperationTool.java @@ -35,7 +35,7 @@ import static org.opencb.opencga.analysis.variant.manager.operations.VariantFileIndexerOperationManager.*; @Tool(id = VariantIndexOperationTool.ID, description = VariantIndexOperationTool.DESCRIPTION, - type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT) + type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT, priority = Enums.Priority.HIGH) public class VariantIndexOperationTool extends OperationTool { public static final String ID = "variant-index"; public static final String DESCRIPTION = "Index variant files into the variant storage"; @@ -77,7 +77,9 @@ protected void check() throws Exception { params.putIfNotEmpty(VariantStorageOptions.INCLUDE_GENOTYPE.key(), indexParams.getIncludeGenotypes()); params.put(VariantStorageOptions.STATS_AGGREGATION.key(), indexParams.getAggregated()); params.putIfNotEmpty(VariantStorageOptions.STATS_AGGREGATION_MAPPING_FILE.key(), indexParams.getAggregationMappingFile()); - params.put(VariantStorageOptions.GVCF.key(), indexParams.isGvcf()); + if (indexParams.isGvcf()) { + params.put(VariantStorageOptions.GVCF.key(), indexParams.isGvcf()); + } // queryOptions.putIfNotNull(VariantFileIndexerStorageOperation.TRANSFORMED_FILES, indexParams.transformedPaths); @@ -92,7 +94,9 @@ protected void check() throws Exception { params.put(VariantStorageOptions.FAMILY.key(), indexParams.isFamily()); params.put(VariantStorageOptions.SOMATIC.key(), indexParams.isSomatic()); params.putIfNotEmpty(VariantStorageOptions.LOAD_SPLIT_DATA.key(), indexParams.getLoadSplitData()); - params.put(VariantStorageOptions.LOAD_MULTI_FILE_DATA.key(), indexParams.isLoadMultiFileData()); + if (indexParams.isLoadMultiFileData()) { + params.put(VariantStorageOptions.LOAD_MULTI_FILE_DATA.key(), indexParams.isLoadMultiFileData()); + } params.putIfNotEmpty(VariantStorageOptions.LOAD_SAMPLE_INDEX.key(), indexParams.getLoadSampleIndex()); params.putIfNotEmpty(VariantStorageOptions.LOAD_ARCHIVE.key(), indexParams.getLoadArchive()); params.putIfNotEmpty(VariantStorageOptions.LOAD_HOM_REF.key(), indexParams.getLoadHomRef()); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantPruneOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantPruneOperationTool.java index 58838a2ea1e..f2426fa5f04 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantPruneOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantPruneOperationTool.java @@ -6,7 +6,7 @@ import org.opencb.opencga.core.tools.annotations.ToolParams; @Tool(id = VariantPruneOperationTool.ID, description = VariantPruneOperationTool.DESCRIPTION, - type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT) + type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT, priority = Enums.Priority.HIGH) public class VariantPruneOperationTool extends OperationTool { public static final String DESCRIPTION = "Prune orphan variants from studies in a project."; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSampleDeleteOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSampleDeleteOperationTool.java index 067367a3f28..8c5620291be 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSampleDeleteOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSampleDeleteOperationTool.java @@ -33,7 +33,7 @@ * @author Jacobo Coll <jacobo167@gmail.com> */ @Tool(id = VariantSampleDeleteOperationTool.ID, description = VariantSampleDeleteOperationTool.DESCRIPTION, - type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT) + type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT, priority = Enums.Priority.HIGH) public class VariantSampleDeleteOperationTool extends OperationTool { public static final String ID = "variant-sample-delete"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantScoreDeleteOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantScoreDeleteOperationTool.java index b2b6734cf87..a26afb81955 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantScoreDeleteOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantScoreDeleteOperationTool.java @@ -21,7 +21,7 @@ import org.opencb.opencga.core.models.common.Enums; @Tool(id= VariantScoreDeleteOperationTool.ID, - type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT) + type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT, priority = Enums.Priority.HIGH) public class VariantScoreDeleteOperationTool extends OperationTool { public static final String ID = "variant-score-delete"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantScoreIndexOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantScoreIndexOperationTool.java index d7049df006e..e863a630167 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantScoreIndexOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantScoreIndexOperationTool.java @@ -16,18 +16,16 @@ package org.opencb.opencga.analysis.variant.operations; -import org.opencb.opencga.core.tools.annotations.Tool; -import org.opencb.opencga.core.models.operations.variant.VariantScoreIndexParams; import org.opencb.opencga.core.common.UriUtils; import org.opencb.opencga.core.models.common.Enums; +import org.opencb.opencga.core.models.operations.variant.VariantScoreIndexParams; +import org.opencb.opencga.core.tools.annotations.Tool; import org.opencb.opencga.storage.core.variant.score.VariantScoreFormatDescriptor; import java.net.URI; -import static org.opencb.opencga.core.api.ParamConstants.STUDY_PARAM; - @Tool(id= VariantScoreIndexOperationTool.ID, description = VariantScoreIndexOperationTool.DESCRIPTION, - type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT) + type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT, priority = Enums.Priority.HIGH) public class VariantScoreIndexOperationTool extends OperationTool { public static final String ID = "variant-score-index"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSecondaryAnnotationIndexOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSecondaryAnnotationIndexOperationTool.java index 24f54414f78..a812c8165c5 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSecondaryAnnotationIndexOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSecondaryAnnotationIndexOperationTool.java @@ -23,7 +23,8 @@ @Tool(id = VariantSecondaryAnnotationIndexOperationTool.ID, description = VariantSecondaryAnnotationIndexOperationTool.DESCRIPTION, type = Tool.Type.OPERATION, scope = Tool.Scope.PROJECT, - resource = Enums.Resource.VARIANT) + resource = Enums.Resource.VARIANT, + priority = Enums.Priority.HIGH) public class VariantSecondaryAnnotationIndexOperationTool extends OperationTool { public static final String ID = "variant-secondary-annotation-index"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSecondaryIndexSamplesDeleteOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSecondaryIndexSamplesDeleteOperationTool.java index b452fffb820..92e5643dedc 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSecondaryIndexSamplesDeleteOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSecondaryIndexSamplesDeleteOperationTool.java @@ -22,7 +22,8 @@ @Tool(id = VariantSecondaryIndexSamplesDeleteOperationTool.ID, description = VariantSecondaryIndexSamplesDeleteOperationTool.DESCRIPTION, type = Tool.Type.OPERATION, - resource = Enums.Resource.VARIANT) + resource = Enums.Resource.VARIANT, + priority = Enums.Priority.HIGH) @Deprecated public class VariantSecondaryIndexSamplesDeleteOperationTool extends OperationTool { diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSecondaryIndexSamplesOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSecondaryIndexSamplesOperationTool.java index e31f32a62a1..9671c0fbda2 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSecondaryIndexSamplesOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSecondaryIndexSamplesOperationTool.java @@ -21,7 +21,7 @@ import org.opencb.opencga.core.models.common.Enums; @Tool(id = VariantSecondaryIndexSamplesOperationTool.ID, - type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT) + type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT, priority = Enums.Priority.HIGH) @Deprecated public class VariantSecondaryIndexSamplesOperationTool extends OperationTool { diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSecondarySampleIndexOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSecondarySampleIndexOperationTool.java index d4dd7ab8a89..259433a3b9e 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSecondarySampleIndexOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSecondarySampleIndexOperationTool.java @@ -30,7 +30,7 @@ import java.util.List; @Tool(id = VariantSecondarySampleIndexOperationTool.ID, description = VariantSecondarySampleIndexOperationTool.DESCRIPTION, - type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT) + type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT, priority = Enums.Priority.HIGH) public class VariantSecondarySampleIndexOperationTool extends OperationTool { public static final String ID = "variant-secondary-sample-index"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStatsDeleteOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStatsDeleteOperationTool.java index 43124238717..0853cfa5d5c 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStatsDeleteOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStatsDeleteOperationTool.java @@ -25,7 +25,8 @@ @Tool(id = VariantStatsDeleteOperationTool.ID, description = VariantStatsDeleteOperationTool.DESCRIPTION, type = Tool.Type.OPERATION, scope = Tool.Scope.STUDY, - resource = Enums.Resource.VARIANT) + resource = Enums.Resource.VARIANT, + priority = Enums.Priority.HIGH) public class VariantStatsDeleteOperationTool extends OperationTool { public static final String ID = "variant-stats-delete"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStatsIndexOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStatsIndexOperationTool.java index ae8576a84de..4ab8f6c58c9 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStatsIndexOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStatsIndexOperationTool.java @@ -26,7 +26,6 @@ import org.opencb.opencga.storage.core.variant.VariantStorageOptions; import org.opencb.opencga.storage.core.variant.stats.DefaultVariantStatisticsManager; -import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -40,7 +39,7 @@ * Created by jacobo on 06/03/15. */ @Tool(id = VariantStatsIndexOperationTool.ID, resource = Enums.Resource.VARIANT, type = Tool.Type.OPERATION, - description = VariantStatsIndexOperationTool.DESCRIPTION) + description = VariantStatsIndexOperationTool.DESCRIPTION, priority = Enums.Priority.HIGH) public class VariantStatsIndexOperationTool extends OperationTool { public final static String ID = "variant-stats-index"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStorageMetadataRepairTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStorageMetadataRepairTool.java index 29ed6efb996..6121cf1e247 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStorageMetadataRepairTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStorageMetadataRepairTool.java @@ -5,10 +5,9 @@ import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.exceptions.ToolException; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.operations.variant.VariantStorageMetadataRepairToolParams; import org.opencb.opencga.core.models.project.DataStore; @@ -27,7 +26,7 @@ import static org.opencb.opencga.core.models.operations.variant.VariantStorageMetadataRepairToolParams.What.*; @Tool(id = VariantStorageMetadataRepairTool.ID, description = VariantStorageMetadataRepairTool.DESCRIPTION, - type = Tool.Type.OPERATION, scope = Tool.Scope.GLOBAL, resource = Enums.Resource.VARIANT) + type = Tool.Type.OPERATION, scope = Tool.Scope.GLOBAL, resource = Enums.Resource.VARIANT, priority = Enums.Priority.HIGH) public class VariantStorageMetadataRepairTool extends OperationTool { public static final String ID = "variant-storage-metadata-repair"; public static final String DESCRIPTION = "Execute some repairs on Variant Storage Metadata. Advanced users only."; @@ -39,25 +38,26 @@ public class VariantStorageMetadataRepairTool extends OperationTool { protected void check() throws Exception { super.check(); - String userId = getCatalogManager().getUserManager().getUserId(getToken()); - if (!userId.equals(ParamConstants.OPENCGA_USER_ID)) { - throw new CatalogAuthenticationException("Only user '" + ParamConstants.OPENCGA_USER_ID + "' can run this operation!"); - } + JwtPayload payload = getCatalogManager().getUserManager().validateToken(getToken()); + getCatalogManager().getAuthorizationManager().checkIsOpencgaAdministrator(payload); } @Override protected void run() throws Exception { if (CollectionUtils.isEmpty(toolParams.getStudies())) { - // Get all studies - Query query = new Query(); - QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList( - StudyDBAdaptor.QueryParams.UID.key(), - StudyDBAdaptor.QueryParams.ID.key(), - StudyDBAdaptor.QueryParams.FQN.key())); - OpenCGAResult studyDataResult = catalogManager.getStudyManager().search(query, options, token); - - for (Study study : studyDataResult.getResults()) { - fixStudy(study.getFqn(), true); + for (String organizationId : catalogManager.getOrganizationManager().getOrganizationIds(getToken())) { + // Get all studies + Query query = new Query(); + QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList( + StudyDBAdaptor.QueryParams.UID.key(), + StudyDBAdaptor.QueryParams.ID.key(), + StudyDBAdaptor.QueryParams.FQN.key())); + OpenCGAResult studyDataResult = catalogManager.getStudyManager() + .searchInOrganization(organizationId, query, options, token); + + for (Study study : studyDataResult.getResults()) { + fixStudy(study.getFqn(), true); + } } } else { for (String study : toolParams.getStudies()) { diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStorageMetadataSynchronizeOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStorageMetadataSynchronizeOperationTool.java index 193c92d359c..8f7cac4f6a2 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStorageMetadataSynchronizeOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStorageMetadataSynchronizeOperationTool.java @@ -1,14 +1,12 @@ package org.opencb.opencga.analysis.variant.operations; -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.operations.variant.VariantStorageMetadataSynchronizeParams; import org.opencb.opencga.core.tools.annotations.Tool; import org.opencb.opencga.core.tools.annotations.ToolParams; @Tool(id = VariantStorageMetadataSynchronizeOperationTool.ID, resource = Enums.Resource.VARIANT, type = Tool.Type.OPERATION, - scope = Tool.Scope.STUDY, description = VariantStorageMetadataSynchronizeOperationTool.DESCRIPTION) + scope = Tool.Scope.STUDY, description = VariantStorageMetadataSynchronizeOperationTool.DESCRIPTION, priority = Enums.Priority.HIGH) public class VariantStorageMetadataSynchronizeOperationTool extends OperationTool { public static final String ID = "variant-storage-metadata-synchronize"; public static final String DESCRIPTION = "Synchronize catalog with variant storage metadata"; @@ -19,11 +17,8 @@ public class VariantStorageMetadataSynchronizeOperationTool extends OperationToo @Override protected void check() throws Exception { super.check(); - - String userId = getCatalogManager().getUserManager().getUserId(getToken()); - if (!userId.equals(ParamConstants.OPENCGA_USER_ID)) { - throw new CatalogAuthenticationException("Only user '" + ParamConstants.OPENCGA_USER_ID + "' can run this operation!"); - } + getCatalogManager().getAuthorizationManager().checkIsOpencgaAdministrator( + catalogManager.getUserManager().validateToken(getToken())); } @Override diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStudyDeleteOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStudyDeleteOperationTool.java index 643c2c989d8..baa52c6f2c2 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStudyDeleteOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStudyDeleteOperationTool.java @@ -32,7 +32,7 @@ * @author Jacobo Coll <jacobo167@gmail.com> */ @Tool(id = VariantStudyDeleteOperationTool.ID, description = VariantStudyDeleteOperationTool.DESCRIPTION, - type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT) + type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT, priority = Enums.Priority.HIGH) public class VariantStudyDeleteOperationTool extends OperationTool { public static final String ID = "variant-study-delete"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/relatedness/RelatednessAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/relatedness/RelatednessAnalysis.java index ee410da895c..6fedc83d6a8 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/relatedness/RelatednessAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/relatedness/RelatednessAnalysis.java @@ -18,7 +18,6 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; -import org.opencb.commons.utils.FileUtils; import org.opencb.opencga.analysis.AnalysisUtils; import org.opencb.opencga.analysis.family.qc.FamilyQcAnalysis; import org.opencb.opencga.analysis.individual.qc.IndividualQcUtils; @@ -31,13 +30,8 @@ import org.opencb.opencga.core.tools.annotations.Tool; import org.opencb.opencga.core.tools.variant.IBDRelatednessAnalysisExecutor; -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.IOException; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/stats/SampleVariantStatsAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/stats/SampleVariantStatsAnalysis.java index fc15bc487f4..3d91b877bdd 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/stats/SampleVariantStatsAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/stats/SampleVariantStatsAnalysis.java @@ -87,7 +87,7 @@ protected void check() throws Exception { } } if (toolParams.isIndex()) { - String userId = getCatalogManager().getUserManager().getUserId(getToken()); + String userId = getCatalogManager().getUserManager().validateToken(getToken()).getUserId(); Study study = getCatalogManager().getStudyManager().get(this.study, new QueryOptions(), getToken()).first(); boolean isOwner = study.getFqn().startsWith(userId + "@"); if (!isOwner) { diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java index d68f363b36d..0f1fc2bec67 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java @@ -7,8 +7,8 @@ import org.apache.commons.lang3.tuple.Pair; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.exec.Command; -import org.opencb.opencga.analysis.wrappers.deeptools.DeeptoolsWrapperAnalysis; import org.opencb.opencga.core.common.GitRepositoryState; +import org.opencb.opencga.core.config.AnalysisTool; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.tools.OpenCgaToolExecutor; import org.slf4j.Logger; @@ -18,12 +18,9 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.*; -public abstract class DockerWrapperAnalysisExecutor extends OpenCgaToolExecutor { +public abstract class DockerWrapperAnalysisExecutor extends OpenCgaToolExecutor { public final static String DOCKER_INPUT_PATH = "/data/input"; public final static String DOCKER_OUTPUT_PATH = "/data/output"; @@ -31,14 +28,51 @@ public abstract class DockerWrapperAnalysisExecutor extends OpenCgaToolExecutor public static final String STDOUT_FILENAME = "stdout.txt"; public static final String STDERR_FILENAME = "stderr.txt"; + public String getDockerImageName() throws ToolException { + return getConfiguration().getAnalysis().getOpencgaExtTools().split(":")[0]; + } + public static final String DOCKER_CLI_MSG = "Docker CLI: "; - public String getDockerImageName() { - return "opencb/opencga-ext-tools"; + public String getDockerImageVersion() throws ToolException { + if (getConfiguration().getAnalysis().getOpencgaExtTools().contains(":")) { + return getConfiguration().getAnalysis().getOpencgaExtTools().split(":")[1]; + } else { + return GitRepositoryState.getInstance().getBuildVersion(); + } + } + + protected AnalysisTool getAnalysisTool(String toolId, String version) throws ToolException { + for (AnalysisTool tool : getConfiguration().getAnalysis().getTools()) { + if (toolId.equals(tool.getId()) && version.equals(tool.getVersion())) { + return tool; + } + } + throw new ToolException("Missing analyis tool (ID = " + toolId + ", version = " + version + ") in configuration file"); + } + + public String getDockerImageName(String toolId, String version) throws ToolException { + AnalysisTool tool = getAnalysisTool(toolId, version); + return tool.getDockerId().split(":")[0]; + } + + public String getDockerImageVersion(String toolId, String version) throws ToolException { + AnalysisTool tool = getAnalysisTool(toolId, version); + if (tool.getDockerId().contains(":")) { + return tool.getDockerId().split(":")[1]; + } else { + return null; + } } - public String getDockerImageVersion() { - return GitRepositoryState.getInstance().getBuildVersion(); + protected String getToolResource(String toolId, String version, String resourceKey) throws ToolException { + // Get resources from the configuration file + AnalysisTool tool = getAnalysisTool(toolId, version); + if (!tool.getResources().containsKey(resourceKey)) { + throw new ToolException("Error getting resource " + resourceKey + " of analysis tool (ID = " + toolId + ", version = " + + version + "): it does not exist in the configuration file"); + } + return tool.getResources().get(resourceKey); } private Logger privateLogger = LoggerFactory.getLogger(DockerWrapperAnalysisExecutor.class); @@ -59,6 +93,14 @@ protected StringBuilder initCommandLine() { return new StringBuilder("docker run --log-driver=none -a stdin -a stdout -a stderr "); } + protected StringBuilder initCommandLine(String user) { + StringBuilder sb = initCommandLine(); + if (StringUtils.isNotEmpty(user)) { + sb.append("--user ").append(user); + } + return sb; + } + protected Map appendMounts(List> inputFilenames, StringBuilder sb) { Map mountMap = new HashMap<>(); @@ -84,7 +126,7 @@ protected Map appendMounts(List> inputFilen return mountMap; } - protected void appendCommand(String command, StringBuilder sb) { + protected void appendCommand(String command, StringBuilder sb) throws ToolException { // Docker image and version sb.append(getDockerImageName()); if (StringUtils.isNotEmpty(getDockerImageVersion())) { diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysis.java index c4a9d87d4f7..a6d34af3f37 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysis.java @@ -17,6 +17,7 @@ package org.opencb.opencga.analysis.wrappers.exomiser; import org.apache.commons.lang3.StringUtils; +import org.opencb.opencga.analysis.ConfigurationUtils; import org.opencb.opencga.analysis.tools.OpenCgaToolScopeStudy; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.clinical.ClinicalAnalysis; @@ -34,6 +35,14 @@ public class ExomiserWrapperAnalysis extends OpenCgaToolScopeStudy { public final static String DESCRIPTION = "The Exomiser is a Java program that finds potential disease-causing variants" + " from whole-exome or whole-genome sequencing data."; + // It must match the tool prefix in the tool keys for exomiser in the configuration file + public final static String EXOMISER_PREFIX = "exomiser-"; + + // It must match the resources key in the exomiser/tool section in the configuration file + public final static String HG19_RESOURCE_KEY = "HG19"; + public final static String HG38_RESOURCE_KEY = "HG38"; + public final static String PHENOTYPE_RESOURCE_KEY = "PHENOTYPE"; + @ToolParams protected final ExomiserWrapperParams analysisParams = new ExomiserWrapperParams(); @@ -43,6 +52,14 @@ protected void check() throws Exception { if (StringUtils.isEmpty(getStudy())) { throw new ToolException("Missing study"); } + + // Check exomiser version + if (StringUtils.isEmpty(analysisParams.getExomiserVersion())) { + // Missing exomiser version use the default one + String exomiserVersion = ConfigurationUtils.getToolDefaultVersion(ExomiserWrapperAnalysis.ID, configuration); + logger.warn("Missing exomiser version, using the default {}", exomiserVersion); + analysisParams.setExomiserVersion(exomiserVersion); + } } @Override @@ -53,6 +70,7 @@ protected void run() throws Exception { getToolExecutor(ExomiserWrapperAnalysisExecutor.class) .setStudyId(study) .setSampleId(analysisParams.getSample()) + .setExomiserVersion(analysisParams.getExomiserVersion()) .setClinicalAnalysisType(ClinicalAnalysis.Type.valueOf(analysisParams.getClinicalAnalysisType())) .execute(); }); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java index a6325932fb9..2d0d0dfb0ab 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java @@ -1,7 +1,6 @@ package org.opencb.opencga.analysis.wrappers.exomiser; import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.opencb.biodata.models.clinical.Disorder; import org.opencb.biodata.models.clinical.Phenotype; @@ -10,6 +9,7 @@ import org.opencb.biodata.models.pedigree.IndividualProperty; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.exec.Command; +import org.opencb.commons.utils.FileUtils; import org.opencb.opencga.analysis.ResourceUtils; import org.opencb.opencga.analysis.StorageToolExecutor; import org.opencb.opencga.analysis.individual.qc.IndividualQcUtils; @@ -30,10 +30,14 @@ import java.io.*; import java.net.URL; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; +import static org.opencb.commons.utils.FileUtils.copyFile; +import static org.opencb.opencga.analysis.wrappers.exomiser.ExomiserWrapperAnalysis.*; + @ToolExecutor(id = ExomiserWrapperAnalysisExecutor.ID, tool = ExomiserWrapperAnalysis.ID, source = ToolExecutor.Source.STORAGE, @@ -46,24 +50,32 @@ public class ExomiserWrapperAnalysisExecutor extends DockerWrapperAnalysisExecut private final static String EXOMISER_PROPERTIES_TEMPLATE_FILENAME = "application.properties"; private static final String EXOMISER_OUTPUT_OPTIONS_FILENAME = "output.yml"; - public final static String DOCKER_IMAGE_NAME = "exomiser/exomiser-cli"; - public final static String DOCKER_IMAGE_VERSION = "13.1.0"; - private String studyId; private String sampleId; + private String exomiserVersion; private ClinicalAnalysis.Type clinicalAnalysisType; private Logger logger = LoggerFactory.getLogger(this.getClass()); @Override - public void run() throws ToolException { + public void run() throws ToolException, IOException, CatalogException { // Check HPOs, it will use a set to avoid duplicate HPOs, // and it will check both phenotypes and disorders - logger.info("{}: Checking individual for sample {} in study {}", ID, sampleId, studyId); + logger.info("Checking individual for sample {} in study {}", sampleId, studyId); Set hpos = new HashSet<>(); Individual individual = IndividualQcUtils.getIndividualBySampleId(studyId, sampleId, getVariantStorageManager().getCatalogManager(), getToken()); + // Check assembly + String assembly = IndividualQcUtils.getAssembly(studyId, getVariantStorageManager().getCatalogManager(), getToken()); + if (assembly.equalsIgnoreCase("GRCh38")) { + assembly = "hg38"; +// } else if (assembly.equalsIgnoreCase("GRCh37")) { +// assembly = "hg19"; + } else { + throw new ToolException("Invalid assembly '" + assembly + "'. Supported assemblies are: GRCh38"); + } + // Set father and mother if necessary (family ?) if (individual.getFather() != null && StringUtils.isNotEmpty(individual.getFather().getId())) { individual.setFather(IndividualQcUtils.getIndividualById(studyId, individual.getFather().getId(), @@ -74,7 +86,7 @@ public void run() throws ToolException { getVariantStorageManager().getCatalogManager(), getToken())); } - logger.info("{}: Individual found: {}", ID, individual.getId()); + logger.info("Individual found: {}", individual.getId()); if (CollectionUtils.isNotEmpty(individual.getPhenotypes())) { for (Phenotype phenotype : individual.getPhenotypes()) { if (phenotype.getId().startsWith("HP:")) { @@ -94,7 +106,7 @@ public void run() throws ToolException { throw new ToolException("Missing phenotypes, i.e. HPO terms, for individual/sample (" + individual.getId() + "/" + sampleId + ")"); } - logger.info("{}: Getting HPO for individual {}: {}", ID, individual.getId(), StringUtils.join(hpos, ",")); + logger.info("Getting HPO for individual {}: {}", individual.getId(), StringUtils.join(hpos, ",")); List samples = new ArrayList<>(); samples.add(sampleId); @@ -137,8 +149,8 @@ public void run() throws ToolException { QueryOptions queryOptions = new QueryOptions(QueryOptions.INCLUDE, "id,studies.samples"); - logger.info("{}: Exomiser exports variants using the query: {}", ID, query.toJson()); - logger.info("{}: Exomiser exports variants using the query options: {}", ID, queryOptions.toJson()); + logger.info("Exomiser exports variants using the query: {}", query.toJson()); + logger.info("Exomiser exports variants using the query options: {}", queryOptions.toJson()); try { getVariantStorageManager().exportData(vcfPath.toString(), VariantWriterFactory.VariantOutputFormat.VCF_GZ, null, query, @@ -153,30 +165,34 @@ public void run() throws ToolException { // Copy the analysis try { - FileUtils.copyFile(openCgaHome.resolve("analysis/exomiser/" + EXOMISER_ANALYSIS_TEMPLATE_FILENAME).toFile(), - getOutDir().resolve(EXOMISER_ANALYSIS_TEMPLATE_FILENAME).toFile()); + copyFile(openCgaHome.resolve("analysis/exomiser").resolve(exomiserVersion).resolve(EXOMISER_ANALYSIS_TEMPLATE_FILENAME) + .toFile(), getOutDir().resolve(EXOMISER_ANALYSIS_TEMPLATE_FILENAME).toFile()); } catch (IOException e) { throw new ToolException("Error copying Exomiser analysis file", e); } - // Copy the application.properties + // Copy the application.properties and update data according to Exomiser version try { - FileUtils.copyFile(openCgaHome.resolve("analysis/exomiser/" + EXOMISER_PROPERTIES_TEMPLATE_FILENAME).toFile(), - getOutDir().resolve(EXOMISER_PROPERTIES_TEMPLATE_FILENAME).toFile()); + Path target = getOutDir().resolve(EXOMISER_PROPERTIES_TEMPLATE_FILENAME); + copyFile(openCgaHome.resolve("analysis/exomiser").resolve(exomiserVersion).resolve(EXOMISER_PROPERTIES_TEMPLATE_FILENAME) + .toFile(), target.toFile()); } catch (IOException e) { throw new ToolException("Error copying Exomiser properties file", e); } // Copy the output options try { - FileUtils.copyFile(openCgaHome.resolve("analysis/exomiser/" + EXOMISER_OUTPUT_OPTIONS_FILENAME).toFile(), + copyFile(openCgaHome.resolve("analysis/exomiser").resolve(exomiserVersion).resolve(EXOMISER_OUTPUT_OPTIONS_FILENAME).toFile(), getOutDir().resolve(EXOMISER_OUTPUT_OPTIONS_FILENAME).toFile()); } catch (IOException e) { throw new ToolException("Error copying Exomiser output options file", e); } // Build the docker command line to run Exomiser - StringBuilder sb = initCommandLine(); + String[] userAndGroup = FileUtils.getUserAndGroup(getOutDir(), true); + String dockerUser = userAndGroup[0] + ":" + userAndGroup[1]; + logger.info("Docker user: {}", dockerUser); + StringBuilder sb = initCommandLine(dockerUser); // Append mounts sb.append(" --mount type=bind,source=" + exomiserDataPath + ",target=/data") @@ -192,7 +208,8 @@ public void run() throws ToolException { sb.append(" --ped /jobdir/").append(pedigreeFile.getName()); } sb.append(" --vcf /jobdir/" + vcfPath.getFileName()) - .append(" --assembly hg38 --output /jobdir/").append(EXOMISER_OUTPUT_OPTIONS_FILENAME) + .append(" --assembly ").append(assembly) + .append(" --output /jobdir/").append(EXOMISER_OUTPUT_OPTIONS_FILENAME) .append(" --spring.config.location=/jobdir/").append(EXOMISER_PROPERTIES_TEMPLATE_FILENAME); // Execute command and redirect stdout and stderr to the files @@ -365,7 +382,7 @@ private Path getAnalysisDataPath(String analysisId) throws ToolException { } private Path getExomiserDataPath(Path openCgaHome) throws ToolException { - Path exomiserDataPath = openCgaHome.resolve("analysis/resources/exomiser"); + Path exomiserDataPath = openCgaHome.resolve("analysis/resources/" + ExomiserWrapperAnalysis.ID); if (!exomiserDataPath.toFile().exists()) { if (!exomiserDataPath.toFile().mkdirs()) { throw new ToolException("Error creating the Exomiser data directory"); @@ -374,33 +391,37 @@ private Path getExomiserDataPath(Path openCgaHome) throws ToolException { // Mutex management to avoid multiple downloadings at the same time // the first to come, download data, others have to wait for - File readyFile = exomiserDataPath.resolve("READY").toFile(); - File preparingFile = exomiserDataPath.resolve("PREPARING").toFile(); + String resource = getToolResource(ExomiserWrapperAnalysis.ID, exomiserVersion, PHENOTYPE_RESOURCE_KEY); + String resourceVersion = Paths.get(resource).getFileName().toString().split("[_]")[0]; + File readyFile = exomiserDataPath.resolve("READY-" + resourceVersion).toFile(); + File preparingFile = exomiserDataPath.resolve("PREPARING-" + resourceVersion).toFile(); // If all is ready, then return if (readyFile.exists()) { - logger.info("{}: Exomiser data is already downloaded, so Exomiser analysis is ready to be executed.", ID); + logger.info("Exomiser {} data {} is already downloaded, so Exomiser analysis is ready to be executed.", exomiserVersion, + resourceVersion); return exomiserDataPath; } // If it is preparing, then wait for ready and then return if (preparingFile.exists()) { long startTime = System.currentTimeMillis(); - logger.info("{}: Exomiser data is downloading, waiting for it...", ID); + logger.info("Exomiser {} data {} is downloading, waiting for it...", exomiserVersion, resourceVersion); while (!readyFile.exists()) { try { Thread.sleep(10000); } catch (InterruptedException e) { // Nothing to do here + preparingFile.delete(); throw new ToolException(e); } long elapsedTime = System.currentTimeMillis() - startTime; if (elapsedTime > 18000000) { - throw new ToolException("Unable to run the Exomiser analysis because of Exomiser data is not ready yet: maximum" - + " waiting time exceeded"); + throw new ToolException("Unable to run the Exomiser analysis because of Exomiser " + exomiserVersion + " data " + + resourceVersion + " is not ready yet: maximum waiting time exceeded"); } } - logger.info("{}: Exomiser data is now downloaded: Exomiser analysis is ready to be executed", ID); + logger.info("Exomiser {} data is now downloaded: Exomiser analysis is ready to be executed", exomiserVersion); return exomiserDataPath; } @@ -408,38 +429,49 @@ private Path getExomiserDataPath(Path openCgaHome) throws ToolException { try { preparingFile.createNewFile(); } catch (IOException e) { - throw new ToolException("Error creating the Exomiser data directory"); + preparingFile.delete(); + throw new ToolException("Error creating the Exomiser " + exomiserVersion + " data " + resourceVersion + " directory"); } - // Download and unzip files + // Download resources and unzip files try { - downloadAndUnzip(exomiserDataPath, "2109_hg38.zip"); - downloadAndUnzip(exomiserDataPath, "2109_phenotype.zip"); + downloadAndUnzip(exomiserDataPath, HG38_RESOURCE_KEY); + downloadAndUnzip(exomiserDataPath, PHENOTYPE_RESOURCE_KEY); } catch (ToolException e) { // If something wrong happened, the preparing file has to be deleted preparingFile.delete(); - throw new ToolException("Something wrong happened when preparing Exomiser data", e); + throw new ToolException("Something wrong happened when preparing Exomiser " + exomiserVersion + " data " + resourceVersion, e); } // Mutex management, signal exomiser data is ready try { readyFile.createNewFile(); } catch (IOException e) { - throw new ToolException("Error preparing Exomiser data", e); + throw new ToolException("Error preparing Exomiser " + exomiserVersion + " data " + resourceVersion, e); } preparingFile.delete(); return exomiserDataPath; } + private String getHg38DataVersion() throws ToolException { + String resource = getToolResource(ExomiserWrapperAnalysis.ID, exomiserVersion, HG38_RESOURCE_KEY); + return Paths.get(resource).getFileName().toString().split("_")[0]; + } + + private String getPhenotypeDataVersion() throws ToolException { + String resource = getToolResource(ExomiserWrapperAnalysis.ID, exomiserVersion, PHENOTYPE_RESOURCE_KEY); + return Paths.get(resource).getFileName().toString().split("_")[0]; + } + @Override - public String getDockerImageName() { - return DOCKER_IMAGE_NAME; + public String getDockerImageName() throws ToolException { + return getDockerImageName(ExomiserWrapperAnalysis.ID, exomiserVersion); } @Override - public String getDockerImageVersion() { - return DOCKER_IMAGE_VERSION; + public String getDockerImageVersion() throws ToolException { + return getDockerImageVersion(ExomiserWrapperAnalysis.ID, exomiserVersion); } public String getStudyId() { @@ -451,21 +483,38 @@ public ExomiserWrapperAnalysisExecutor setStudyId(String studyId) { return this; } - private void downloadAndUnzip(Path exomiserDataPath, String filename) throws ToolException { - URL url = null; - - // Download data - try { - url = new URL("http://resources.opencb.org/opencb/opencga/analysis/exomiser/" + filename); - logger.info("{}: Downloading Exomiser data: {} in {}", ID, url, exomiserDataPath); - ResourceUtils.downloadThirdParty(url, exomiserDataPath); - } catch (IOException e) { - throw new ToolException("Error downloading Exomiser data from " + url, e); + private void downloadAndUnzip(Path exomiserDataPath, String resourceKey) throws ToolException { + String filename; + String resource = getToolResource(ExomiserWrapperAnalysis.ID, exomiserVersion, resourceKey); + if (resource.startsWith("file://")) { + // Copy resouce + try { + Path sourcePath = Paths.get(resource); + filename = sourcePath.getFileName().toString(); + Files.copy(sourcePath, exomiserDataPath.resolve(filename)); + } catch (IOException e) { + throw new ToolException("Error copying Exomiser data from " + resource, e); + } + } else { + // Download resource + String url; + if (resource.startsWith("http://") || resource.startsWith("https://") || resource.startsWith("ftp://")) { + url = resource; + } else { + url = getConfiguration().getAnalysis().getResourceUrl() + resource; + } + logger.info("Downloading Exomiser data: {} in {}", url, exomiserDataPath); + try { + ResourceUtils.downloadThirdParty(new URL(url), exomiserDataPath); + filename = Paths.get(url).getFileName().toString(); + } catch (IOException e) { + throw new ToolException("Error downloading Exomiser data from " + url, e); + } } // Unzip try { - logger.info("{}: Decompressing Exomiser data: {}", ID, filename); + logger.info("Decompressing Exomiser {} data: {}", exomiserDataPath, filename); new Command("unzip -o -d " + exomiserDataPath + " " + exomiserDataPath + "/" + filename) .setOutputOutputStream(new DataOutputStream(new FileOutputStream(getOutDir().resolve("stdout_unzip_" + filename + ".txt").toFile()))) @@ -473,11 +522,11 @@ private void downloadAndUnzip(Path exomiserDataPath, String filename) throws Too + filename + ".txt").toFile()))) .run(); } catch (FileNotFoundException e) { - throw new ToolException("Error unzipping Exomiser data: " + filename, e); + throw new ToolException("Error unzipping Exomiser " + exomiserVersion + " data: " + filename, e); } // Free disk space - logger.info("{}: Deleting Exomiser data: {}", ID, filename); + logger.info("Deleting Exomiser data: {}", filename); exomiserDataPath.resolve(filename).toFile().delete(); } @@ -490,6 +539,15 @@ public ExomiserWrapperAnalysisExecutor setSampleId(String sampleId) { return this; } + public String getExomiserVersion() { + return exomiserVersion; + } + + public ExomiserWrapperAnalysisExecutor setExomiserVersion(String exomiserVersion) { + this.exomiserVersion = exomiserVersion; + return this; + } + public ClinicalAnalysis.Type getClinicalAnalysisType() { return clinicalAnalysisType; } diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/alignment/AlignmentAnalysisTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/alignment/AlignmentAnalysisTest.java index 8964c2bf603..9339ddf40e2 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/alignment/AlignmentAnalysisTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/alignment/AlignmentAnalysisTest.java @@ -39,7 +39,9 @@ import org.opencb.opencga.core.models.file.File; import org.opencb.opencga.core.models.file.FileLinkParams; import org.opencb.opencga.core.models.file.FileRelatedFile; -import org.opencb.opencga.core.models.user.Account; +import org.opencb.opencga.core.models.organizations.OrganizationCreateParams; +import org.opencb.opencga.core.models.organizations.OrganizationUpdateParams; +import org.opencb.opencga.core.models.user.User; import org.opencb.opencga.core.testclassification.duration.MediumTests; import org.opencb.opencga.storage.core.StorageEngineFactory; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; @@ -60,6 +62,7 @@ @Category(MediumTests.class) public class AlignmentAnalysisTest { + public static final String ORGANIZATION = "test"; public static final String USER = "user"; public static final String PASSWORD = TestParamConstants.PASSWORD; public static final String PROJECT = "project"; @@ -67,7 +70,7 @@ public class AlignmentAnalysisTest { public static final String PHENOTYPE_NAME = "myPhenotype"; public static final Phenotype PHENOTYPE = new Phenotype(PHENOTYPE_NAME, PHENOTYPE_NAME, "mySource") .setStatus(Phenotype.Status.OBSERVED); - public static final String DB_NAME = "opencga_test_" + USER + "_" + PROJECT; + public static final String DB_NAME = VariantStorageManager.buildDatabaseName("opencga_test", ORGANIZATION, PROJECT); private ToolRunner toolRunner; private static String father = "NA19661"; private static String mother = "NA19660"; @@ -211,7 +214,7 @@ public void setUp() throws Throwable { catalogManager = opencga.getCatalogManager(); variantStorageManager = new VariantStorageManager(catalogManager, opencga.getStorageEngineFactory()); toolRunner = new ToolRunner(opencga.getOpencgaHome().toString(), catalogManager, StorageEngineFactory.get(variantStorageManager.getStorageConfiguration())); - token = catalogManager.getUserManager().login("user", PASSWORD).getToken(); + token = catalogManager.getUserManager().login(ORGANIZATION, "user", PASSWORD).getToken(); } @AfterClass @@ -222,9 +225,13 @@ public static void afterClass() { opencga.after(); } - public void setUpCatalogManager() throws IOException, CatalogException { - catalogManager.getUserManager().create(USER, "User Name", "mail@ebi.ac.uk", PASSWORD, "", null, Account.AccountType.FULL, opencga.getAdminToken()); - token = catalogManager.getUserManager().login("user", PASSWORD).getToken(); + public void setUpCatalogManager() throws CatalogException { + catalogManager.getOrganizationManager().create(new OrganizationCreateParams().setId("test"), null, opencga.getAdminToken()); + catalogManager.getUserManager().create(new User().setId(USER).setName("User Name").setEmail("mail@ebi.ac.uk").setOrganization("test"), + PASSWORD, opencga.getAdminToken()); + catalogManager.getOrganizationManager().update("test", new OrganizationUpdateParams().setOwner(USER), null, opencga.getAdminToken()); + + token = catalogManager.getUserManager().login(ORGANIZATION, "user", PASSWORD).getToken(); String projectId = catalogManager.getProjectManager().create(PROJECT, "Project about some genomes", "", "Homo sapiens", null, "GRCh38", new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true), token).first().getId(); @@ -271,7 +278,7 @@ public void geneCoverageStatsTest() throws IOException, ToolException, CatalogEx String geneName = "BRCA2"; params.setGenes(Arrays.asList(geneName)); - toolRunner.execute(AlignmentGeneCoverageStatsAnalysis.class, params, new ObjectMap(), outdir, "coverage-job-id", token); + toolRunner.execute(AlignmentGeneCoverageStatsAnalysis.class, params, new ObjectMap(), outdir, "coverage-job-id", false, token); bamFile = catalogManager.getFileManager().link(STUDY, new FileLinkParams(bamFilename, "", "", "", null, null, null, null, null), false, token).first(); @@ -294,7 +301,7 @@ public void testNonReadOnlyAlignmentIndex() throws Exception { AlignmentIndexParams params = new AlignmentIndexParams(); params.setFileId(bamFile.getId()); Path alignmentIndexOutdir = Paths.get(opencga.createTmpOutdir("_alignment_index")); - toolRunner.execute(AlignmentIndexOperation.class, params, new ObjectMap(ParamConstants.STUDY_PARAM, STUDY), alignmentIndexOutdir, "jobId-non-readonly-coverage-index", token); + toolRunner.execute(AlignmentIndexOperation.class, params, new ObjectMap(ParamConstants.STUDY_PARAM, STUDY), alignmentIndexOutdir, "jobId-non-readonly-coverage-index", false, token); // Checking BAI file Path baiPath = nonReadOnlyDir.resolve(bamFilename + AlignmentConstants.BAI_EXTENSION); @@ -323,7 +330,7 @@ public void testReadOnlyAlignmentIndex() throws Exception { AlignmentIndexParams params = new AlignmentIndexParams(); params.setFileId(bamFile.getId()); Path alignmentIndexOutdir = Paths.get(opencga.createTmpOutdir("_alignment_index")); - toolRunner.execute(AlignmentIndexOperation.class, params, new ObjectMap(ParamConstants.STUDY_PARAM, STUDY), alignmentIndexOutdir, "jobId-readonly-coverage-index", token); + toolRunner.execute(AlignmentIndexOperation.class, params, new ObjectMap(ParamConstants.STUDY_PARAM, STUDY), alignmentIndexOutdir, "jobId-readonly-coverage-index", false, token); // Checking BAI file Path baiPath = alignmentIndexOutdir.resolve(bamFilename + AlignmentConstants.BAI_EXTENSION); @@ -351,7 +358,7 @@ public void testNonReadOnlyCoverageIndex() throws Exception { AlignmentIndexParams indexParams = new AlignmentIndexParams(); indexParams.setFileId(bamFile.getId()); Path alignmentIndexOutdir = Paths.get(opencga.createTmpOutdir("_alignment_index")); - toolRunner.execute(AlignmentIndexOperation.class, indexParams, new ObjectMap(ParamConstants.STUDY_PARAM, STUDY), alignmentIndexOutdir, "jobId-non-readonly-alignment-coverage-index", token); + toolRunner.execute(AlignmentIndexOperation.class, indexParams, new ObjectMap(ParamConstants.STUDY_PARAM, STUDY), alignmentIndexOutdir, "jobId-non-readonly-alignment-coverage-index", false, token); // Checking BAI file Path baiPath = nonReadOnlyDir.resolve(bamFilename + AlignmentConstants.BAI_EXTENSION); @@ -367,7 +374,7 @@ public void testNonReadOnlyCoverageIndex() throws Exception { coverageOarams.setBamFileId(bamFile.getId()); coverageOarams.setBaiFileId(baiFile.getId()); Path coverageIndexOutdir = Paths.get(opencga.createTmpOutdir("_coverage_index")); - toolRunner.execute(AlignmentCoverageAnalysis.class, coverageOarams, new ObjectMap(ParamConstants.STUDY_PARAM, STUDY), coverageIndexOutdir, "jobId-readonly-coverage-index", token); + toolRunner.execute(AlignmentCoverageAnalysis.class, coverageOarams, new ObjectMap(ParamConstants.STUDY_PARAM, STUDY), coverageIndexOutdir, "jobId-readonly-coverage-index", false, token); // Checking BW file Path bwPath = nonReadOnlyDir.resolve(bamFilename + AlignmentConstants.BIGWIG_EXTENSION); @@ -393,7 +400,7 @@ public void testReadOnlyCoverageIndex() throws Exception { AlignmentIndexParams indexParams = new AlignmentIndexParams(); indexParams.setFileId(bamFile.getId()); Path alignmentIndexOutdir = Paths.get(opencga.createTmpOutdir("_alignment_index")); - toolRunner.execute(AlignmentIndexOperation.class, indexParams, new ObjectMap(ParamConstants.STUDY_PARAM, STUDY), alignmentIndexOutdir, "jobId-readonly-coverage-index", token); + toolRunner.execute(AlignmentIndexOperation.class, indexParams, new ObjectMap(ParamConstants.STUDY_PARAM, STUDY), alignmentIndexOutdir, "jobId-readonly-coverage-index", false, token); // Checking BAI file Path baiPath = readOnlyDir.resolve(bamFilename + AlignmentConstants.BAI_EXTENSION); @@ -412,7 +419,7 @@ public void testReadOnlyCoverageIndex() throws Exception { coverageOarams.setBamFileId(bamFile.getId()); coverageOarams.setBaiFileId(baiFile.getId()); Path coverageIndexOutdir = Paths.get(opencga.createTmpOutdir("_coverage_index")); - toolRunner.execute(AlignmentCoverageAnalysis.class, coverageOarams, new ObjectMap(ParamConstants.STUDY_PARAM, STUDY), coverageIndexOutdir, "jobId-readonly-coverage-index", token); + toolRunner.execute(AlignmentCoverageAnalysis.class, coverageOarams, new ObjectMap(ParamConstants.STUDY_PARAM, STUDY), coverageIndexOutdir, "jobId-readonly-coverage-index", false, token); // Checking BW file Path bwPath = coverageIndexOutdir.resolve(bamFilename + AlignmentConstants.BIGWIG_EXTENSION); diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/ClinicalAnalysisUtilsTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/ClinicalAnalysisUtilsTest.java index f77a1025377..87a8d1d2a26 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/ClinicalAnalysisUtilsTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/ClinicalAnalysisUtilsTest.java @@ -24,6 +24,7 @@ import org.opencb.biodata.models.variant.avro.SequenceOntologyTerm; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.analysis.variant.OpenCGATestExternalResource; +import org.opencb.opencga.analysis.variant.manager.VariantOperationsTest; import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.AbstractClinicalManagerTest; @@ -51,25 +52,17 @@ public static AbstractClinicalManagerTest getClinicalTest(OpenCGATestExternalRes clinicalTest.catalogManagerResource = opencga.getCatalogManagerExternalResource(); clinicalTest.setUp(); - // Exomiser analysis - Path exomiserDataPath = opencga.getOpencgaHome().resolve("analysis/exomiser"); - Files.createDirectories(exomiserDataPath); - Path parent = Paths.get(ClinicalAnalysisUtilsTest.class.getClassLoader().getResource("pheno").getPath()).getParent(); - Files.copy(parent.resolve("exomiser/application.properties"), exomiserDataPath.resolve("application.properties"), - StandardCopyOption.REPLACE_EXISTING); - Files.copy(parent.resolve("exomiser/output.yml"), exomiserDataPath.resolve("output.yml"), - StandardCopyOption.REPLACE_EXISTING); - // Storage ObjectMap storageOptions = new ObjectMap() .append(VariantStorageOptions.ANNOTATE.key(), true) .append(VariantStorageOptions.STATS_CALCULATE.key(), false); - VariantStorageManager variantStorageManager = new VariantStorageManager(opencga.getCatalogManager(), opencga.getStorageEngineFactory()); + VariantStorageManager variantStorageManager = opencga.getVariantStorageManager(); Path outDir = Paths.get("target/test-data").resolve("junit_clinical_analysis_" + RandomStringUtils.randomAlphabetic(10)); Files.createDirectories(outDir); + VariantOperationsTest.dummyVariantSetup(variantStorageManager, clinicalTest.studyFqn, clinicalTest.token); variantStorageManager.index(clinicalTest.studyFqn, "family.vcf", outDir.toString(), storageOptions, clinicalTest.token); variantStorageManager.index(clinicalTest.studyFqn, "exomiser.vcf.gz", outDir.toString(), storageOptions, clinicalTest.token); variantStorageManager.index(clinicalTest.studyFqn, "HG004.1k.vcf.gz", outDir.toString(), storageOptions, clinicalTest.token); diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/ClinicalInterpretationAnalysisTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/ClinicalInterpretationAnalysisTest.java index 8a8de40962b..73235db4d6d 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/ClinicalInterpretationAnalysisTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/ClinicalInterpretationAnalysisTest.java @@ -44,9 +44,7 @@ import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.clinical.ClinicalAnalysis; -import org.opencb.opencga.core.models.clinical.ClinicalAnalysisUpdateParams; import org.opencb.opencga.core.models.clinical.InterpretationUpdateParams; -import org.opencb.opencga.core.models.individual.Individual; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.MediumTests; import org.opencb.opencga.core.tools.result.ExecutionResult; diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java index ba185297c28..46799bfab58 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java @@ -1,17 +1,15 @@ package org.opencb.opencga.analysis.clinical.exomiser; import org.apache.commons.lang3.StringUtils; -import org.junit.After; +import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; +import org.junit.experimental.categories.Category; import org.opencb.biodata.models.clinical.interpretation.ClinicalVariant; import org.opencb.biodata.models.variant.Variant; import org.opencb.biodata.models.variant.exceptions.NonStandardCompliantSampleField; import org.opencb.biodata.tools.variant.VariantNormalizer; -import org.junit.*; -import org.eclipse.jetty.util.Scanner; -import org.junit.experimental.categories.Category; import org.opencb.commons.datastore.core.Event; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.QueryOptions; @@ -33,14 +31,13 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.Optional; -import java.util.stream.Stream; import static com.mongodb.assertions.Assertions.assertFalse; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeThat; +import static org.junit.Assume.assumeTrue; @Category(MediumTests.class) public class ExomiserInterpretationAnalysisTest { @@ -79,8 +76,12 @@ public void testNormalization() throws NonStandardCompliantSampleField { } @Test - public void singleSingleExomiserAnalysis() throws IOException, CatalogException, ToolException { - assumeThat(Paths.get("/opt/opencga/analysis/resources/exomiser").toFile().exists(), is(true)); + public void singleExomiserAnalysis() throws IOException, CatalogException, ToolException { +// String exomiserVersion = "13.1"; +// String resourceVersion = "2109"; + String exomiserVersion = "14.0"; + String resourceVersion = "2402"; + assumeTrue(Paths.get("/opt/opencga/analysis/resources/exomiser/READY-" + resourceVersion).toFile().exists()); prepareExomiserData(); outDir = Paths.get(opencga.createTmpOutdir("_interpretation_analysis_single")); @@ -90,11 +91,14 @@ public void singleSingleExomiserAnalysis() throws IOException, CatalogException, ClinicalAnalysis clinicalAnalysis = caResult.getResults().get(0); assertEquals(0, clinicalAnalysis.getSecondaryInterpretations().size()); - ExomiserInterpretationAnalysis exomiser = new ExomiserInterpretationAnalysis(); + System.out.println("opencga.getOpencgaHome() = " + opencga.getOpencgaHome().toAbsolutePath()); + System.out.println("outDir = " + outDir); + ExomiserInterpretationAnalysis exomiser = new ExomiserInterpretationAnalysis(); exomiser.setUp(opencga.getOpencgaHome().toAbsolutePath().toString(), new ObjectMap(), outDir, clinicalTest.token); exomiser.setStudyId(clinicalTest.studyFqn) - .setClinicalAnalysisId(clinicalTest.CA_ID2); + .setClinicalAnalysisId(clinicalTest.CA_ID2) + .setExomiserVersion(exomiserVersion); ExecutionResult result = exomiser.start(); @@ -122,8 +126,12 @@ public void singleSingleExomiserAnalysis() throws IOException, CatalogException, } @Test - public void trioFamilyExomiserAnalysis() throws IOException, CatalogException, ToolException { - assumeThat(Paths.get("/opt/opencga/analysis/resources/exomiser").toFile().exists(), is(true)); + public void familyExomiserAnalysis() throws IOException, CatalogException, ToolException { +// String exomiserVersion = "13.1"; +// String resourceVersion = "2109"; + String exomiserVersion = "14.0"; + String resourceVersion = "2402"; + assumeTrue(Paths.get("/opt/opencga/analysis/resources/exomiser/READY-" + resourceVersion).toFile().exists()); prepareExomiserData(); outDir = Paths.get(opencga.createTmpOutdir("_interpretation_analysis_trio_family")); @@ -136,7 +144,8 @@ public void trioFamilyExomiserAnalysis() throws IOException, CatalogException, T exomiser.setUp(opencga.getOpencgaHome().toAbsolutePath().toString(), new ObjectMap(), outDir, clinicalTest.token); exomiser.setStudyId(clinicalTest.studyFqn) - .setClinicalAnalysisId(clinicalTest.CA_ID3); + .setClinicalAnalysisId(clinicalTest.CA_ID3) + .setExomiserVersion(exomiserVersion); ExecutionResult result = exomiser.start(); @@ -167,7 +176,11 @@ public void trioFamilyExomiserAnalysis() throws IOException, CatalogException, T @Test public void trioSingleExomiserAnalysis() throws IOException, CatalogException, ToolException { - assumeThat(Paths.get("/opt/opencga/analysis/resources/exomiser").toFile().exists(), is(true)); +// String exomiserVersion = "13.1"; +// String resourceVersion = "2109"; + String exomiserVersion = "14.0"; + String resourceVersion = "2402"; + assumeTrue(Paths.get("/opt/opencga/analysis/resources/exomiser/READY-" + resourceVersion).toFile().exists()); prepareExomiserData(); outDir = Paths.get(opencga.createTmpOutdir("_interpretation_analysis_trio_single")); @@ -180,7 +193,8 @@ public void trioSingleExomiserAnalysis() throws IOException, CatalogException, T exomiser.setUp(opencga.getOpencgaHome().toAbsolutePath().toString(), new ObjectMap(), outDir, clinicalTest.token); exomiser.setStudyId(clinicalTest.studyFqn) - .setClinicalAnalysisId(clinicalTest.CA_ID4); + .setClinicalAnalysisId(clinicalTest.CA_ID4) + .setExomiserVersion(exomiserVersion); ExecutionResult result = exomiser.start(); diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/family/FamilyAnalysisTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/family/FamilyAnalysisTest.java index 129797ea078..91ac3624471 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/family/FamilyAnalysisTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/family/FamilyAnalysisTest.java @@ -1,7 +1,10 @@ package org.opencb.opencga.analysis.family; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; +import org.hamcrest.CoreMatchers; +import org.hamcrest.MatcherAssert; import org.junit.*; +import org.junit.experimental.categories.Category; import org.junit.rules.ExpectedException; import org.opencb.biodata.models.clinical.Disorder; import org.opencb.biodata.models.clinical.Phenotype; @@ -27,10 +30,12 @@ import org.opencb.opencga.core.models.family.PedigreeGraphAnalysisParams; import org.opencb.opencga.core.models.individual.Individual; import org.opencb.opencga.core.models.individual.IndividualUpdateParams; +import org.opencb.opencga.core.models.organizations.OrganizationCreateParams; +import org.opencb.opencga.core.models.organizations.OrganizationUpdateParams; import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.sample.SampleReferenceParam; -import org.opencb.opencga.core.models.user.Account; import org.opencb.opencga.core.response.OpenCGAResult; +import org.opencb.opencga.core.testclassification.duration.MediumTests; import org.opencb.opencga.storage.core.StorageEngineFactory; import java.io.IOException; @@ -44,9 +49,10 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +@Category(MediumTests.class) public class FamilyAnalysisTest extends GenericTest { - public final static String STUDY = "user@1000G:phase1"; + public static final String USER = "user"; @Rule public ExpectedException thrown = ExpectedException.none(); @@ -58,6 +64,9 @@ public class FamilyAnalysisTest extends GenericTest { private static String opencgaToken; protected static String sessionIdUser; + private static String ORGANIZATION = "test"; + public final static String STUDY = ORGANIZATION + "@1000G:phase1"; + protected static Family family; protected static Family family2; protected static Family family3; @@ -75,29 +84,24 @@ public static void setUp() throws IOException, CatalogException { } public static void setUpCatalogManager(CatalogManager catalogManager) throws CatalogException { - opencgaToken = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); + opencgaToken = opencga.getAdminToken(); - catalogManager.getUserManager().create("user", "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, "", null, - Account.AccountType.FULL, opencgaToken); - sessionIdUser = catalogManager.getUserManager().login("user", TestParamConstants.PASSWORD).getToken(); + catalogManager.getOrganizationManager().create(new OrganizationCreateParams().setId(ORGANIZATION), QueryOptions.empty(), + opencga.getAdminToken()); + catalogManager.getUserManager().create(USER, USER, "my@email.org", TestParamConstants.PASSWORD, ORGANIZATION, 1000L, opencga.getAdminToken()); + catalogManager.getOrganizationManager().update(ORGANIZATION, new OrganizationUpdateParams().setAdmins(Collections.singletonList(USER)), + null, + opencga.getAdminToken()); + sessionIdUser = catalogManager.getUserManager().login(ORGANIZATION, USER, TestParamConstants.PASSWORD).getToken(); projectId = catalogManager.getProjectManager().create("1000G", "Project about some genomes", "", "Homo sapiens", null, "GRCh38", INCLUDE_RESULT, sessionIdUser).first().getId(); studyId = catalogManager.getStudyManager().create(projectId, "phase1", null, "Phase 1", "Done", null, null, null, null, null, sessionIdUser).first().getId(); - try { - family = createDummyFamily("Martinez-Martinez").first(); - } catch (CatalogException e) { - } - try { - family2 = createDummyFamily("Lopez-Lopez", Arrays.asList("father11-sample", "mother11-sample"), 1).first(); - } catch (CatalogException e) { - } - try { - family3 = createDummyFamily("Perez-Perez", Arrays.asList("father22-sample", "mother22-sample", "child222-sample"), 0).first(); - } catch (CatalogException e) { - } + family = createDummyFamily("Martinez-Martinez").first(); + family2 = createDummyFamily("Lopez-Lopez", Arrays.asList("father11-sample", "mother11-sample"), 1).first(); + family3 = createDummyFamily("Perez-Perez", Arrays.asList("father22-sample", "mother22-sample", "child222-sample"), 0).first(); } @After @@ -169,15 +173,17 @@ public void test2Member2GenerationFamilyTest() throws CatalogException { public void updateTest() throws CatalogException { FamilyUpdateParams updateParams = new FamilyUpdateParams(); + Family prevFamily = catalogManager.getFamilyManager().get(studyId, family.getId(), null, sessionIdUser).first(); + assertEquals(prevFamily.getPedigreeGraph().getBase64(), family.getPedigreeGraph().getBase64()); + QueryOptions queryOptions = new QueryOptions() .append(ParamConstants.FAMILY_UPDATE_ROLES_PARAM, true) .append(ParamConstants.INCLUDE_RESULT_PARAM, true); Family updatedFamily = catalogManager.getFamilyManager().update(studyId, family.getId(), updateParams, queryOptions, sessionIdUser) .first(); - PedigreeGraph pedigreeGraph = updatedFamily.getPedigreeGraph(); - assertTrue(pedigreeGraph.getBase64().startsWith("iVBORw0KGgoAAAANSUhEUgAAAeAAAAHg")); - assertTrue(pedigreeGraph.getBase64().endsWith("5UIf81hI8AAAAASUVORK5CYII=")); + assertEquals(prevFamily.getPedigreeGraph().getBase64(), updatedFamily.getPedigreeGraph().getBase64()); + assertEquals(prevFamily.getVersion() + 1, updatedFamily.getVersion()); } @Test @@ -196,11 +202,11 @@ public void testPedigreeGraphAnalysis() throws ToolException, IOException { params.setFamilyId(family.getId()); toolRunner.execute(PedigreeGraphAnalysis.class, params, new ObjectMap(ParamConstants.STUDY_PARAM, studyId), outDir, null, - sessionIdUser); + false, sessionIdUser); String b64Image = PedigreeGraphUtils.getB64Image(outDir); - assertTrue(b64Image.startsWith("iVBORw0KGgoAAAANSUhEUgAAAeAAAAHg")); - assertTrue(b64Image.endsWith("s3Rj5UIf81hI8AAAAASUVORK5CYII=")); + MatcherAssert.assertThat(b64Image, CoreMatchers.startsWith("iVBORw0KGgoAAAANSUhEUgAAAeAAAAHg")); + MatcherAssert.assertThat(b64Image, CoreMatchers.endsWith("s3Rj5UIf81hI8AAAAASUVORK5CYII=")); assertEquals(family.getPedigreeGraph().getBase64(), b64Image); } diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/file/PostLinkSampleAssociationTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/file/PostLinkSampleAssociationTest.java index 665ebfbfa50..2aad751f48c 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/file/PostLinkSampleAssociationTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/file/PostLinkSampleAssociationTest.java @@ -49,32 +49,32 @@ public void setUp() throws Exception { @Test public void linkTest() throws Exception { - fileManager.setFileSampleLinkThreshold(1); + fileManager.setFileSampleLinkThreshold(organizationId, 1); String vcfFile = opencga.getResourceUri("biofiles/variant-test-file.vcf.gz").toString(); - OpenCGAResult link = fileManager.link(studyFqn, new FileLinkParams(vcfFile, "", "", "", null, null, null, null, null), false, token); + OpenCGAResult link = fileManager.link(studyFqn, new FileLinkParams(vcfFile, "", "", "", null, null, null, null, null), false, ownerToken); assertEquals(0, link.first().getSampleIds().size()); assertEquals(FileStatus.MISSING_SAMPLES, link.first().getInternal().getStatus().getId()); assertFalse(link.first().getInternal().getMissingSamples().getNonExisting().isEmpty()); Query query = new Query(SampleDBAdaptor.QueryParams.FILE_IDS.key(), link.first().getId()); - OpenCGAResult search = catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, token); + OpenCGAResult search = catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, ownerToken); assertEquals(0, search.getNumResults()); query = new Query(SampleDBAdaptor.QueryParams.ID.key(), Arrays.asList("NA19600", "NA19660", "NA19661", "NA19685")); - search = catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, token); + search = catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, ownerToken); assertEquals(0, search.getNumResults()); Path outDir = Paths.get(opencga.createTmpOutdir("_postlink")); PostLinkSampleAssociation postLinkSampleAssociation = new PostLinkSampleAssociation(); - postLinkSampleAssociation.setUp(opencga.getOpencgaHome().toString(), null, outDir, token); + postLinkSampleAssociation.setUp(opencga.getOpencgaHome().toString(), null, outDir, ownerToken); postLinkSampleAssociation.setStudy(studyFqn); postLinkSampleAssociation.start(); System.out.println(link); - File file = fileManager.get(studyFqn, link.first().getId(), QueryOptions.empty(), token).first(); + File file = fileManager.get(studyFqn, link.first().getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(4, file.getSampleIds().size()); assertTrue(file.getSampleIds().containsAll(Arrays.asList("NA19600", "NA19660", "NA19661", "NA19685"))); @@ -83,19 +83,19 @@ public void linkTest() throws Exception { assertTrue(file.getInternal().getMissingSamples().getExisting().isEmpty()); query = new Query(SampleDBAdaptor.QueryParams.FILE_IDS.key(), file.getId()); - search = catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, token); + search = catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, ownerToken); assertEquals(4, search.getNumResults()); assertTrue(search.getResults().stream().map(Sample::getId).collect(Collectors.toSet()) .containsAll(Arrays.asList("NA19600", "NA19660", "NA19661", "NA19685"))); query = new Query(SampleDBAdaptor.QueryParams.FILE_IDS.key(), file.getUuid()); - search = catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, token); + search = catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, ownerToken); assertEquals(4, search.getNumResults()); assertTrue(search.getResults().stream().map(Sample::getId).collect(Collectors.toSet()) .containsAll(Arrays.asList("NA19600", "NA19660", "NA19661", "NA19685"))); query = new Query(SampleDBAdaptor.QueryParams.FILE_IDS.key(), file.getPath()); - search = catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, token); + search = catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, ownerToken); assertEquals(4, search.getNumResults()); assertTrue(search.getResults().stream().map(Sample::getId).collect(Collectors.toSet()) .containsAll(Arrays.asList("NA19600", "NA19660", "NA19661", "NA19685"))); @@ -103,12 +103,12 @@ public void linkTest() throws Exception { @Test public void postLinkPassingVirtualFile() throws Exception { - fileManager.setFileSampleLinkThreshold(1); + fileManager.setFileSampleLinkThreshold(organizationId, 1); String vcfFile = opencga.getResourceUri("biofiles/variant-test-file.vcf.gz").toString(); - OpenCGAResult link = fileManager.link(studyFqn, new FileLinkParams(vcfFile, "", "", "", null, "biofiles/virtual_file.vcf", null, null, null), false, token); + OpenCGAResult link = fileManager.link(studyFqn, new FileLinkParams(vcfFile, "", "", "", null, "biofiles/virtual_file.vcf", null, null, null), false, ownerToken); - OpenCGAResult virtualResult = fileManager.get(studyFqn, "virtual_file.vcf", QueryOptions.empty(), token); + OpenCGAResult virtualResult = fileManager.get(studyFqn, "virtual_file.vcf", QueryOptions.empty(), ownerToken); assertEquals(0, link.first().getSampleIds().size()); assertEquals(FileStatus.READY, link.first().getInternal().getStatus().getId()); @@ -119,11 +119,11 @@ public void postLinkPassingVirtualFile() throws Exception { assertFalse(virtualResult.first().getInternal().getMissingSamples().getNonExisting().isEmpty()); Query query = new Query(SampleDBAdaptor.QueryParams.FILE_IDS.key(), link.first().getId()); - OpenCGAResult search = catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, token); + OpenCGAResult search = catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, ownerToken); assertEquals(0, search.getNumResults()); query = new Query(SampleDBAdaptor.QueryParams.ID.key(), Arrays.asList("NA19600", "NA19660", "NA19661", "NA19685")); - search = catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, token); + search = catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, ownerToken); assertEquals(0, search.getNumResults()); Path outDir = Paths.get(opencga.createTmpOutdir("_postlink")); @@ -131,12 +131,12 @@ public void postLinkPassingVirtualFile() throws Exception { PostLinkSampleAssociation postLinkSampleAssociation = new PostLinkSampleAssociation(); postLinkSampleAssociation.setUp(opencga.getOpencgaHome().toString(), new ObjectMap(new PostLinkToolParams().setFiles(Collections.singletonList(virtualResult.first().getId())).toParams()), - outDir, token); + outDir, ownerToken); postLinkSampleAssociation.setStudy(studyFqn); postLinkSampleAssociation.start(); System.out.println(link); - File file = fileManager.get(studyFqn, virtualResult.first().getId(), QueryOptions.empty(), token).first(); + File file = fileManager.get(studyFqn, virtualResult.first().getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(4, file.getSampleIds().size()); assertTrue(file.getSampleIds().containsAll(Arrays.asList("NA19600", "NA19660", "NA19661", "NA19685"))); @@ -145,19 +145,19 @@ public void postLinkPassingVirtualFile() throws Exception { assertTrue(file.getInternal().getMissingSamples().getExisting().isEmpty()); query = new Query(SampleDBAdaptor.QueryParams.FILE_IDS.key(), file.getId()); - search = catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, token); + search = catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, ownerToken); assertEquals(4, search.getNumResults()); assertTrue(search.getResults().stream().map(Sample::getId).collect(Collectors.toSet()) .containsAll(Arrays.asList("NA19600", "NA19660", "NA19661", "NA19685"))); query = new Query(SampleDBAdaptor.QueryParams.FILE_IDS.key(), file.getUuid()); - search = catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, token); + search = catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, ownerToken); assertEquals(4, search.getNumResults()); assertTrue(search.getResults().stream().map(Sample::getId).collect(Collectors.toSet()) .containsAll(Arrays.asList("NA19600", "NA19660", "NA19661", "NA19685"))); query = new Query(SampleDBAdaptor.QueryParams.FILE_IDS.key(), file.getPath()); - search = catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, token); + search = catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, ownerToken); assertEquals(4, search.getNumResults()); assertTrue(search.getResults().stream().map(Sample::getId).collect(Collectors.toSet()) .containsAll(Arrays.asList("NA19600", "NA19660", "NA19661", "NA19685"))); @@ -165,12 +165,12 @@ public void postLinkPassingVirtualFile() throws Exception { @Test public void postLinkPassingMultipartFile() throws Exception { - fileManager.setFileSampleLinkThreshold(1); + fileManager.setFileSampleLinkThreshold(organizationId, 1); String vcfFile = opencga.getResourceUri("biofiles/variant-test-file.vcf.gz").toString(); - OpenCGAResult link = fileManager.link(studyFqn, new FileLinkParams(vcfFile, "", "", "", null, "biofiles/virtual_file.vcf", null, null, null), false, token); + OpenCGAResult link = fileManager.link(studyFqn, new FileLinkParams(vcfFile, "", "", "", null, "biofiles/virtual_file.vcf", null, null, null), false, ownerToken); - OpenCGAResult virtualResult = fileManager.get(studyFqn, "virtual_file.vcf", QueryOptions.empty(), token); + OpenCGAResult virtualResult = fileManager.get(studyFqn, "virtual_file.vcf", QueryOptions.empty(), ownerToken); assertEquals(0, link.first().getSampleIds().size()); assertEquals(FileStatus.READY, link.first().getInternal().getStatus().getId()); @@ -181,11 +181,11 @@ public void postLinkPassingMultipartFile() throws Exception { assertFalse(virtualResult.first().getInternal().getMissingSamples().getNonExisting().isEmpty()); Query query = new Query(SampleDBAdaptor.QueryParams.FILE_IDS.key(), link.first().getId()); - OpenCGAResult search = catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, token); + OpenCGAResult search = catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, ownerToken); assertEquals(0, search.getNumResults()); query = new Query(SampleDBAdaptor.QueryParams.ID.key(), Arrays.asList("NA19600", "NA19660", "NA19661", "NA19685")); - search = catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, token); + search = catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, ownerToken); assertEquals(0, search.getNumResults()); Path outDir = Paths.get(opencga.createTmpOutdir("_postlink")); @@ -193,12 +193,12 @@ public void postLinkPassingMultipartFile() throws Exception { PostLinkSampleAssociation postLinkSampleAssociation = new PostLinkSampleAssociation(); postLinkSampleAssociation.setUp(opencga.getOpencgaHome().toString(), new ObjectMap(new PostLinkToolParams().setFiles(Collections.singletonList(link.first().getId())).toParams()), - outDir, token); + outDir, ownerToken); postLinkSampleAssociation.setStudy(studyFqn); postLinkSampleAssociation.start(); System.out.println(link); - File file = fileManager.get(studyFqn, virtualResult.first().getId(), QueryOptions.empty(), token).first(); + File file = fileManager.get(studyFqn, virtualResult.first().getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(4, file.getSampleIds().size()); assertTrue(file.getSampleIds().containsAll(Arrays.asList("NA19600", "NA19660", "NA19661", "NA19685"))); @@ -207,19 +207,19 @@ public void postLinkPassingMultipartFile() throws Exception { assertTrue(file.getInternal().getMissingSamples().getExisting().isEmpty()); query = new Query(SampleDBAdaptor.QueryParams.FILE_IDS.key(), file.getId()); - search = catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, token); + search = catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, ownerToken); assertEquals(4, search.getNumResults()); assertTrue(search.getResults().stream().map(Sample::getId).collect(Collectors.toSet()) .containsAll(Arrays.asList("NA19600", "NA19660", "NA19661", "NA19685"))); query = new Query(SampleDBAdaptor.QueryParams.FILE_IDS.key(), file.getUuid()); - search = catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, token); + search = catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, ownerToken); assertEquals(4, search.getNumResults()); assertTrue(search.getResults().stream().map(Sample::getId).collect(Collectors.toSet()) .containsAll(Arrays.asList("NA19600", "NA19660", "NA19661", "NA19685"))); query = new Query(SampleDBAdaptor.QueryParams.FILE_IDS.key(), file.getPath()); - search = catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, token); + search = catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, ownerToken); assertEquals(4, search.getNumResults()); assertTrue(search.getResults().stream().map(Sample::getId).collect(Collectors.toSet()) .containsAll(Arrays.asList("NA19600", "NA19660", "NA19661", "NA19685"))); @@ -227,33 +227,33 @@ public void postLinkPassingMultipartFile() throws Exception { @Test public void linkTestFile() throws Exception { - fileManager.setFileSampleLinkThreshold(1); + fileManager.setFileSampleLinkThreshold(organizationId, 1); String vcfFile = opencga.getResourceUri("biofiles/variant-test-file.vcf.gz").toString(); - OpenCGAResult link = fileManager.link(studyFqn, new FileLinkParams(vcfFile, "", "", "", null, null, null, null, null), false, token); + OpenCGAResult link = fileManager.link(studyFqn, new FileLinkParams(vcfFile, "", "", "", null, null, null, null, null), false, ownerToken); assertEquals(0, link.first().getSampleIds().size()); assertEquals(FileStatus.MISSING_SAMPLES, link.first().getInternal().getStatus().getId()); assertFalse(link.first().getInternal().getMissingSamples().getNonExisting().isEmpty()); Query query = new Query(SampleDBAdaptor.QueryParams.FILE_IDS.key(), link.first().getId()); - OpenCGAResult search = catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, token); + OpenCGAResult search = catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, ownerToken); assertEquals(0, search.getNumResults()); query = new Query(SampleDBAdaptor.QueryParams.ID.key(), Arrays.asList("NA19600", "NA19660", "NA19661", "NA19685")); - search = catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, token); + search = catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, ownerToken); assertEquals(0, search.getNumResults()); Path outDir = Paths.get(opencga.createTmpOutdir("_postlink")); PostLinkSampleAssociation postLinkSampleAssociation = new PostLinkSampleAssociation(); postLinkSampleAssociation.setUp(opencga.getOpencgaHome().toString(), - new ObjectMap("files", Collections.singletonList("variant-test-file.vcf.gz")), outDir, token); + new ObjectMap("files", Collections.singletonList("variant-test-file.vcf.gz")), outDir, ownerToken); postLinkSampleAssociation.setStudy(studyFqn); postLinkSampleAssociation.start(); System.out.println(link); - File file = fileManager.get(studyFqn, link.first().getId(), QueryOptions.empty(), token).first(); + File file = fileManager.get(studyFqn, link.first().getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(4, file.getSampleIds().size()); assertTrue(file.getSampleIds().containsAll(Arrays.asList("NA19600", "NA19660", "NA19661", "NA19685"))); @@ -262,7 +262,7 @@ public void linkTestFile() throws Exception { assertTrue(file.getInternal().getMissingSamples().getExisting().isEmpty()); query = new Query(SampleDBAdaptor.QueryParams.FILE_IDS.key(), file.getId()); - search = catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, token); + search = catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, ownerToken); assertEquals(4, search.getNumResults()); assertTrue(search.getResults().stream().map(Sample::getId).collect(Collectors.toSet()) .containsAll(Arrays.asList("NA19600", "NA19660", "NA19661", "NA19685"))); diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/rga/RgaEngineTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/rga/RgaEngineTest.java index 8439c56e88b..9283ab82efb 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/rga/RgaEngineTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/rga/RgaEngineTest.java @@ -15,16 +15,17 @@ import org.opencb.opencga.catalog.managers.CatalogManager; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.config.storage.StorageConfiguration; -import org.opencb.opencga.core.models.analysis.knockout.*; +import org.opencb.opencga.core.models.analysis.knockout.KnockoutByIndividual; import org.opencb.opencga.core.testclassification.duration.MediumTests; import org.opencb.opencga.storage.core.StorageEngineFactory; +import org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageTest; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; @Category(MediumTests.class) public class RgaEngineTest { @@ -38,7 +39,10 @@ public class RgaEngineTest { private List knockoutByIndividualList = new ArrayList<>(2); - @Rule + @Rule(order = 0) + public HadoopVariantStorageTest.HadoopSolrSupport solrSupport = new HadoopVariantStorageTest.HadoopSolrSupport(); + + @Rule(order = 1) public RgaSolrExtenalResource solr = new RgaSolrExtenalResource(); @Before diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/rga/RgaManagerTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/rga/RgaManagerTest.java index 475d70523d3..74cc8339db8 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/rga/RgaManagerTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/rga/RgaManagerTest.java @@ -30,7 +30,6 @@ import org.opencb.opencga.core.models.file.FileLinkParams; import org.opencb.opencga.core.models.individual.*; import org.opencb.opencga.core.models.sample.*; -import org.opencb.opencga.core.models.user.Account; import org.opencb.opencga.core.models.variant.KnockoutAnalysisParams; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.MediumTests; @@ -50,6 +49,7 @@ @Ignore public class RgaManagerTest { + public static final String ORGANIZATION = "test"; public static final String OWNER = "owner"; public static final String USER = "user"; public static final String PASSWORD = TestParamConstants.PASSWORD; @@ -165,7 +165,7 @@ public void setUp() throws Throwable { KnockoutAnalysisParams params = new KnockoutAnalysisParams(); params.setSample(file.getSampleIds()); - toolRunner.execute(KnockoutAnalysis.class, params.toObjectMap(), outDir, null, ownerToken); + toolRunner.execute(KnockoutAnalysis.class, params.toObjectMap(), outDir, null, false, ownerToken); File file = catalogManager.getFileManager().link(STUDY, new FileLinkParams() @@ -183,8 +183,8 @@ public static void afterClass() { } public void setUpCatalogManager() throws CatalogException { - catalogManager.getUserManager().create(OWNER, "User Name", "mail@ebi.ac.uk", PASSWORD, "", null, Account.AccountType.FULL, opencga.getAdminToken()); - ownerToken = catalogManager.getUserManager().login(OWNER, PASSWORD).getToken(); + catalogManager.getUserManager().create(OWNER, "User Name", "mail@ebi.ac.uk", PASSWORD, ORGANIZATION, null, opencga.getAdminToken()); + ownerToken = catalogManager.getUserManager().login(ORGANIZATION, OWNER, PASSWORD).getToken(); String projectId = catalogManager.getProjectManager().create(PROJECT, "Project about some genomes", "", "Homo sapiens", null, "GRCh37", new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true), ownerToken).first().getId(); @@ -199,8 +199,8 @@ public void setUpCatalogManager() throws CatalogException { catalogManager.getSampleManager().create(STUDY, sample, null, ownerToken); } - catalogManager.getUserManager().create(USER, "Other Name", "mail2@ebi.ac.uk", PASSWORD, "", null, Account.AccountType.GUEST, opencga.getAdminToken()); - userToken = catalogManager.getUserManager().login(USER, PASSWORD).getToken(); + catalogManager.getUserManager().create(USER, "Other Name", "mail2@ebi.ac.uk", PASSWORD, ORGANIZATION, null, opencga.getAdminToken()); + userToken = catalogManager.getUserManager().login(ORGANIZATION, USER, PASSWORD).getToken(); } @Test diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java index 38782aef1f0..6fca2e7c2aa 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java @@ -22,14 +22,17 @@ import org.opencb.commons.datastore.mongodb.MongoDataStore; import org.opencb.commons.datastore.mongodb.MongoDataStoreManager; import org.opencb.opencga.analysis.StorageManager; +import org.opencb.opencga.analysis.clinical.exomiser.ExomiserInterpretationAnalysisTest; import org.opencb.opencga.analysis.tools.ToolRunner; import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; +import org.opencb.opencga.catalog.db.mongodb.MongoBackupUtils; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.CatalogManager; import org.opencb.opencga.catalog.managers.CatalogManagerExternalResource; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.config.storage.StorageConfiguration; import org.opencb.opencga.core.models.file.File; +import org.opencb.opencga.core.models.project.DataStore; import org.opencb.opencga.storage.core.StorageEngineFactory; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.variant.VariantStorageBaseTest; @@ -39,20 +42,20 @@ import org.opencb.opencga.storage.core.variant.solr.VariantSolrExternalResource; import org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageEngine; import org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageTest; +import org.reflections.Reflections; +import org.reflections.scanners.ResourcesScanner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.*; import java.net.URI; +import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.text.SimpleDateFormat; -import java.util.Arrays; -import java.util.Date; -import java.util.List; -import java.util.Map; +import java.util.*; /** * Created on 26/08/15 @@ -61,7 +64,7 @@ */ public class OpenCGATestExternalResource extends ExternalResource { - private CatalogManagerExternalResource catalogManagerExternalResource = new CatalogManagerExternalResource(); + private final CatalogManagerExternalResource catalogManagerExternalResource = new CatalogManagerExternalResource(); private Path opencgaHome; private String storageEngine; private boolean initiated = false; @@ -72,7 +75,8 @@ public class OpenCGATestExternalResource extends ExternalResource { private ToolRunner toolRunner; - public static HadoopVariantStorageTest.HadoopExternalResource hadoopExternalResource = new HadoopVariantStorageTest.HadoopExternalResource(); + public static HadoopVariantStorageTest.HadoopExternalResource hadoopExternalResource + = new HadoopVariantStorageTest.HadoopExternalResource(); public OpenCGATestExternalResource() { this(false); @@ -214,8 +218,6 @@ public Path isolateOpenCGA() throws IOException { Files.createDirectories(conf); Files.createDirectories(userHome); - catalogManagerExternalResource.getConfiguration().getAdmin().setSecretKey(null); - catalogManagerExternalResource.getConfiguration().getAdmin().setAlgorithm(null); catalogManagerExternalResource.getConfiguration().serialize( new FileOutputStream(conf.resolve("configuration.yml").toFile())); InputStream inputStream = StorageManager.class.getClassLoader().getResourceAsStream("storage-configuration.yml"); @@ -236,6 +238,9 @@ public Path isolateOpenCGA() throws IOException { StorageEngineFactory.configure(storageConfiguration); storageEngineFactory = StorageEngineFactory.get(storageConfiguration); + if (storageEngine.equals(DummyVariantStorageEngine.STORAGE_ENGINE_ID)) { + DummyVariantStorageEngine.configure(getStorageEngineFactory(), true); + } // inputStream = StorageEngine.class.getClassLoader().getResourceAsStream("client-configuration-test.yml"); // Files.copy(inputStream, conf.resolve("client-configuration.yml"), StandardCopyOption.REPLACE_EXISTING); @@ -263,11 +268,15 @@ public Path isolateOpenCGA() throws IOException { Files.copy(inputStream, analysisPath.resolve("ped.R"), StandardCopyOption.REPLACE_EXISTING); // Exomiser analysis files - analysisPath = Files.createDirectories(opencgaHome.resolve("analysis/exomiser")).toAbsolutePath(); + List exomiserVersions = Arrays.asList("13.1", "14.0"); List exomiserFiles = Arrays.asList("application.properties", "exomiser-analysis.yml", "output.yml"); - for (String exomiserFile : exomiserFiles) { - inputStream = new FileInputStream("../opencga-app/app/analysis/exomiser/" + exomiserFile); - Files.copy(inputStream, analysisPath.resolve(exomiserFile), StandardCopyOption.REPLACE_EXISTING); + for (String exomiserVersion : exomiserVersions) { + analysisPath = Files.createDirectories(opencgaHome.resolve("analysis/exomiser").resolve(exomiserVersion).toAbsolutePath()); + Path exomiserPath = Paths.get(ExomiserInterpretationAnalysisTest.class.getClassLoader().getResource("exomiser").getPath()); + for (String exomiserFile : exomiserFiles) { + String resource = exomiserVersion + "/" + exomiserFile; + Files.copy(exomiserPath.resolve(resource).toAbsolutePath(), analysisPath.resolve(exomiserFile), StandardCopyOption.REPLACE_EXISTING); + } } return opencgaHome; @@ -359,6 +368,34 @@ public String createTmpOutdir(String suffix) throws IOException { // return getCatalogManager().getJobManager().createJobOutDir(studyId, "I_tmp_" + date + sufix, sessionId).toString(); } + public void restore(URL resource) throws Exception { + if (resource.getProtocol().equals("jar")) { + Reflections reflections = new Reflections(resource.getPath().replace('/','.'), new ResourcesScanner()); + Set resources = reflections.getResources(x -> true); + for (String file : resources) { + catalogManagerExternalResource.getResourceUri(file.replace('.', '/')); + } + MongoBackupUtils.restore(getCatalogManager(), opencgaHome, opencgaHome + .resolve("resources") + .resolve(resource.getPath()) + .resolve("mongodb")); + } else { + MongoBackupUtils.restore(getCatalogManager(), opencgaHome, Paths.get(resource.toURI()).resolve("mongodb")); + } + catalogManagerExternalResource.resetCatalogManager(); + } + + public final VariantStorageEngine getVariantStorageEngineByProject(String projectFqn) throws Exception { + DataStore dataStore = getVariantStorageManager().getDataStoreByProjectId(projectFqn, getAdminToken()); + VariantStorageEngine variantStorageEngine = storageEngineFactory + .getVariantStorageEngine(dataStore.getStorageEngine(), dataStore.getDbName()); + if (dataStore.getOptions() != null) { + variantStorageEngine.getOptions().putAll(dataStore.getOptions()); + } + return variantStorageEngine; + } + + // private class StorageLocalExecutorManager extends LocalExecutorManager { // // public StorageLocalExecutorManager(String sessionId) { diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/VariantAnalysisTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/VariantAnalysisTest.java index 718d28fee1f..f9e9392be80 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/VariantAnalysisTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/VariantAnalysisTest.java @@ -39,10 +39,12 @@ import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.TestParamConstants; +import org.opencb.opencga.analysis.clinical.ClinicalAnalysisLoadTask; import org.opencb.opencga.analysis.tools.ToolRunner; import org.opencb.opencga.analysis.variant.gwas.GwasAnalysis; import org.opencb.opencga.analysis.variant.hrdetect.HRDetectAnalysis; import org.opencb.opencga.analysis.variant.knockout.KnockoutAnalysis; +import org.opencb.opencga.analysis.variant.manager.VariantOperationsTest; import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; import org.opencb.opencga.analysis.variant.mutationalSignature.MutationalSignatureAnalysis; import org.opencb.opencga.analysis.variant.operations.VariantIndexOperationTool; @@ -62,6 +64,8 @@ import org.opencb.opencga.core.config.storage.CellBaseConfiguration; import org.opencb.opencga.core.config.storage.StorageConfiguration; import org.opencb.opencga.core.exceptions.ToolException; +import org.opencb.opencga.core.models.clinical.ClinicalAnalysis; +import org.opencb.opencga.core.models.clinical.ClinicalAnalysisLoadParams; import org.opencb.opencga.core.models.cohort.Cohort; import org.opencb.opencga.core.models.cohort.CohortCreateParams; import org.opencb.opencga.core.models.cohort.CohortUpdateParams; @@ -72,13 +76,14 @@ import org.opencb.opencga.core.models.individual.IndividualInternal; import org.opencb.opencga.core.models.individual.Location; import org.opencb.opencga.core.models.operations.variant.VariantIndexParams; +import org.opencb.opencga.core.models.organizations.OrganizationCreateParams; +import org.opencb.opencga.core.models.organizations.OrganizationUpdateParams; import org.opencb.opencga.core.models.project.ProjectCreateParams; import org.opencb.opencga.core.models.project.ProjectOrganism; import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.sample.SampleQualityControl; import org.opencb.opencga.core.models.sample.SampleReferenceParam; import org.opencb.opencga.core.models.sample.SampleUpdateParams; -import org.opencb.opencga.core.models.user.Account; import org.opencb.opencga.core.models.variant.*; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.LongTests; @@ -95,10 +100,7 @@ import org.opencb.opencga.storage.hadoop.variant.VariantHbaseTestUtils; import org.opencb.opencga.storage.hadoop.variant.adaptors.VariantHadoopDBAdaptor; -import java.io.BufferedReader; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStreamReader; +import java.io.*; import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; @@ -114,6 +116,7 @@ @Category(LongTests.class) public class VariantAnalysisTest { + public static final String ORGANIZATION = "test"; public static final String USER = "user"; public static final String PASSWORD = TestParamConstants.PASSWORD; public static final String PROJECT = "project"; @@ -121,7 +124,7 @@ public class VariantAnalysisTest { public static final String PHENOTYPE_NAME = "myPhenotype"; public static final Phenotype PHENOTYPE = new Phenotype(PHENOTYPE_NAME, PHENOTYPE_NAME, "mySource") .setStatus(Phenotype.Status.OBSERVED); - public static final String DB_NAME = "opencga_test_" + USER + "_" + PROJECT; + public static final String DB_NAME = VariantStorageManager.buildDatabaseName("opencga_test", ORGANIZATION, PROJECT); private ToolRunner toolRunner; private static String father = "NA19661"; private static String mother = "NA19660"; @@ -167,8 +170,6 @@ public void setUp() throws Throwable { // System.setProperty("opencga.log.level", "INFO"); // Configurator.reconfigure(); if (!indexed) { - indexed = true; - opencga.after(); opencga.before(storageEngine); @@ -179,6 +180,9 @@ public void setUp() throws Throwable { setUpCatalogManager(); + VariantOperationsTest.dummyVariantSetup(variantStorageManager, STUDY, token); + VariantOperationsTest.dummyVariantSetup(variantStorageManager, CANCER_STUDY, token); + file = opencga.createFile(STUDY, "variant-test-file.vcf.gz", token); variantStorageManager.index(STUDY, file.getId(), opencga.createTmpOutdir("_index"), new ObjectMap(VariantStorageOptions.ANNOTATE.key(), true), token); @@ -250,6 +254,7 @@ public void setUp() throws Throwable { if (storageEngine.equals(HadoopVariantStorageEngine.STORAGE_ENGINE_ID)) { VariantHbaseTestUtils.printVariants(((VariantHadoopDBAdaptor) engine.getDBAdaptor()), Paths.get(opencga.createTmpOutdir("_hbase_print_variants")).toUri()); } + indexed = true; } // Reset engines opencga.getStorageEngineFactory().close(); @@ -257,7 +262,7 @@ public void setUp() throws Throwable { variantStorageManager = opencga.getVariantStorageManager(); variantStorageManager.getStorageConfiguration().setMode(StorageConfiguration.Mode.READ_ONLY); toolRunner = new ToolRunner(opencga.getOpencgaHome().toString(), catalogManager, StorageEngineFactory.get(variantStorageManager.getStorageConfiguration())); - token = catalogManager.getUserManager().login("user", PASSWORD).getToken(); + token = catalogManager.getUserManager().login(ORGANIZATION, "user", PASSWORD).getToken(); } @AfterClass @@ -269,11 +274,18 @@ public static void afterClass() { } public void setUpCatalogManager() throws IOException, CatalogException { - catalogManager.getUserManager().create(USER, "User Name", "mail@ebi.ac.uk", PASSWORD, "", null, Account.AccountType.FULL, opencga.getAdminToken()); - token = catalogManager.getUserManager().login("user", PASSWORD).getToken(); - - String projectId = catalogManager.getProjectManager().create(PROJECT, "Project about some genomes", "", "Homo sapiens", - null, "GRCh38", new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true), token).first().getId(); + catalogManager.getOrganizationManager().create(new OrganizationCreateParams().setId(ORGANIZATION), QueryOptions.empty(), opencga.getAdminToken()); + catalogManager.getUserManager().create(USER, "User Name", "mail@ebi.ac.uk", PASSWORD, ORGANIZATION, null, opencga.getAdminToken()); + catalogManager.getOrganizationManager().update(ORGANIZATION, new OrganizationUpdateParams().setAdmins(Collections.singletonList(USER)), + null, + opencga.getAdminToken()); + token = catalogManager.getUserManager().login(ORGANIZATION, "user", PASSWORD).getToken(); + + String projectId = catalogManager.getProjectManager().create(new ProjectCreateParams() + .setId(PROJECT) + .setDescription("Project about some genomes") + .setOrganism(new ProjectOrganism("hsapiens", "GRCh38")), + new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true), token).first().getId(); catalogManager.getStudyManager().create(projectId, STUDY, null, "Phase 1", "Done", null, null, null, null, null, token); // Create 10 samples not indexed @@ -306,7 +318,7 @@ public void testMalformedVcfFileIndex() throws Exception { try { toolRunner.execute(VariantIndexOperationTool.class, new VariantIndexParams().setFile(file.getId()).setAnnotate(true), - Paths.get(opencga.createTmpOutdir()), null, token); + Paths.get(opencga.createTmpOutdir()), null, false, token); } catch (ToolException e) { System.out.println(ExceptionUtils.prettyExceptionMessage(e, true, true)); } @@ -322,7 +334,7 @@ public void testVariantStats() throws Exception { VariantStatsAnalysis variantStatsAnalysis = new VariantStatsAnalysis() .setStudy(STUDY) .setSamples(samples.subList(1, 3)); - variantStatsAnalysis.setUp(opencga.getOpencgaHome().toString(), catalogManager, variantStorageManager, executorParams, outDir, "", token); + variantStatsAnalysis.setUp(opencga.getOpencgaHome().toString(), catalogManager, variantStorageManager, executorParams, outDir, "", false, token); ExecutionResult ar = variantStatsAnalysis.start(); checkExecutionResult(ar); @@ -347,7 +359,7 @@ public void testVariantStatsTwoCohorts() throws Exception { VariantStatsAnalysis variantStatsAnalysis = new VariantStatsAnalysis() .setStudy(STUDY) .setCohort(Arrays.asList("c1", "c2")); - variantStatsAnalysis.setUp(opencga.getOpencgaHome().toString(), catalogManager, variantStorageManager, executorParams, outDir, "", token); + variantStatsAnalysis.setUp(opencga.getOpencgaHome().toString(), catalogManager, variantStorageManager, executorParams, outDir, "", false, token); ExecutionResult ar = variantStatsAnalysis.start(); checkExecutionResult(ar); @@ -374,7 +386,7 @@ public void testVariantStatsWithFilter() throws Exception { VariantStatsAnalysisParams params = new VariantStatsAnalysisParams() .setSamples(samples.subList(1, 3)) .setRegion(region); - ExecutionResult ar = toolRunner.execute(VariantStatsAnalysis.class, STUDY, params, outDir, "", token); + ExecutionResult ar = toolRunner.execute(VariantStatsAnalysis.class, STUDY, params, outDir, "", false, token); checkExecutionResult(ar); MutableInt count = new MutableInt(); @@ -472,7 +484,7 @@ private ExecutionResult sampleVariantStats(String region, String indexId, boolea params.getVariantQuery() .appendQuery(query) .setRegion(region); - ExecutionResult result = toolRunner.execute(SampleVariantStatsAnalysis.class, STUDY, params, outDir, null, token); + ExecutionResult result = toolRunner.execute(SampleVariantStatsAnalysis.class, STUDY, params, outDir, null, false, token); if (nothingToDo) { assertEquals("All samples stats indexed. Nothing to do!", result.getEvents().get(0).getMessage()); @@ -508,7 +520,7 @@ public void testCohortStats() throws Exception { CohortVariantStatsAnalysis analysis = new CohortVariantStatsAnalysis(); Path outDir = Paths.get(opencga.createTmpOutdir("_cohort_stats")); System.out.println("output = " + outDir.toAbsolutePath()); - analysis.setUp(opencga.getOpencgaHome().toString(), catalogManager, variantStorageManager, executorParams, outDir, "", token); + analysis.setUp(opencga.getOpencgaHome().toString(), catalogManager, variantStorageManager, executorParams, outDir, "", false, token); List samples = file.getSampleIds(); analysis.setStudy(STUDY) .setSamplesQuery(new Query(SampleDBAdaptor.QueryParams.ID.key(), samples.subList(0, 3))); @@ -525,7 +537,7 @@ public void testCohortStatsIndex() throws Exception { .setIndex(true); ExecutionResult result = toolRunner.execute(CohortVariantStatsAnalysis.class, STUDY, toolParams, - outDir, null, token); + outDir, null, false, token); checkExecutionResult(result, storageEngine.equals(HadoopVariantStorageEngine.STORAGE_ENGINE_ID)); Cohort cohort = catalogManager.getCohortManager().get(STUDY, StudyEntry.DEFAULT_COHORT, new QueryOptions(), token).first(); @@ -549,7 +561,7 @@ public void testCohortStatsIndex() throws Exception { outDir = Paths.get(opencga.createTmpOutdir("_cohort_stats_index_2")); System.out.println("output = " + outDir.toAbsolutePath()); - result = toolRunner.execute(CohortVariantStatsAnalysis.class, STUDY, toolParams, outDir, null, token); + result = toolRunner.execute(CohortVariantStatsAnalysis.class, STUDY, toolParams, outDir, null, false, token); checkExecutionResult(result, storageEngine.equals(HadoopVariantStorageEngine.STORAGE_ENGINE_ID)); @@ -572,7 +584,7 @@ public void testExport() throws Exception { variantExportParams.setOutputFileName("chr22.vcf"); toolRunner.execute(VariantExportTool.class, - variantExportParams.toObjectMap().append(ParamConstants.STUDY_PARAM, STUDY), outDir, null, token); + variantExportParams.toObjectMap().append(ParamConstants.STUDY_PARAM, STUDY), outDir, null, false, token); assertTrue(outDir.resolve(variantExportParams.getOutputFileName() + ".gz").toFile().exists()); } @@ -587,7 +599,7 @@ public void testExportVep() throws Exception { variantExportParams.setOutputFileName("chr1-5-22"); variantExportParams.setOutputFileFormat(VariantWriterFactory.VariantOutputFormat.ENSEMBL_VEP.name()); toolRunner.execute(VariantExportTool.class, - variantExportParams.toObjectMap().append(ParamConstants.STUDY_PARAM, STUDY), outDir, null, token); + variantExportParams.toObjectMap().append(ParamConstants.STUDY_PARAM, STUDY), outDir, null, false, token); } @Test @@ -607,7 +619,7 @@ public void testExportTped() throws Exception { variantExportParams.setInclude("id,studies.samples"); toolRunner.execute(VariantExportTool.class, - variantExportParams.toObjectMap().append(ParamConstants.STUDY_PARAM, STUDY), outDir, null, token); + variantExportParams.toObjectMap().append(ParamConstants.STUDY_PARAM, STUDY), outDir, null, false, token); System.out.println(outDir); Path tped = outDir.resolve(variantExportParams.getOutputFileName() + ".tped"); @@ -628,7 +640,7 @@ public void testGwas() throws Exception { GwasAnalysis analysis = new GwasAnalysis(); Path outDir = Paths.get(opencga.createTmpOutdir("_gwas")); System.out.println("output = " + outDir.toAbsolutePath()); - analysis.setUp(opencga.getOpencgaHome().toString(), catalogManager, variantStorageManager, executorParams, outDir, "", token); + analysis.setUp(opencga.getOpencgaHome().toString(), catalogManager, variantStorageManager, executorParams, outDir, "", false, token); List samples = file.getSampleIds(); analysis.setStudy(STUDY) .setCaseCohortSamplesQuery(new Query(SampleDBAdaptor.QueryParams.ID.key(), samples.subList(0, 2))) @@ -642,7 +654,7 @@ public void testGwasByPhenotype() throws Exception { GwasAnalysis analysis = new GwasAnalysis(); Path outDir = Paths.get(opencga.createTmpOutdir("_gwas_phenotype")); System.out.println("output = " + outDir.toAbsolutePath()); - analysis.setUp(opencga.getOpencgaHome().toString(), catalogManager, variantStorageManager, executorParams, outDir, "", token); + analysis.setUp(opencga.getOpencgaHome().toString(), catalogManager, variantStorageManager, executorParams, outDir, "", false, token); analysis.setStudy(STUDY) .setPhenotype(PHENOTYPE_NAME); @@ -657,7 +669,7 @@ public void testKnockoutGenes() throws Exception { params.setSample(file.getSampleIds()); ExecutionResult er = toolRunner.execute(KnockoutAnalysis.class, - params.toObjectMap().append(ParamConstants.STUDY_PARAM, STUDY), outDir, null, token); + params.toObjectMap().append(ParamConstants.STUDY_PARAM, STUDY), outDir, null, false, token); checkExecutionResult(er, false); } @@ -672,7 +684,7 @@ public void testKnockoutGenesSpecificGenes() throws Exception { ExecutionResult er = toolRunner.execute(KnockoutAnalysis.class, params.toObjectMap() .append(ParamConstants.STUDY_PARAM, STUDY) - .append("executionMethod", "byGene"), outDir, null, token); + .append("executionMethod", "byGene"), outDir, null, false, token); checkExecutionResult(er, false); assertEquals(4, er.getAttributes().get("otherGenesCount")); assertEquals(3, er.getAttributes().get("proteinCodingGenesCount")); @@ -688,7 +700,7 @@ public void testKnockoutGenesSpecificGenesAndBiotypeProteinCoding() throws Excep params.setBiotype(VariantAnnotationConstants.PROTEIN_CODING); ExecutionResult er = toolRunner.execute(KnockoutAnalysis.class, - params.toObjectMap().append(ParamConstants.STUDY_PARAM, STUDY), outDir, null, token); + params.toObjectMap().append(ParamConstants.STUDY_PARAM, STUDY), outDir, null, false, token); checkExecutionResult(er, false); assertEquals(0, er.getAttributes().get("otherGenesCount")); assertEquals(3, er.getAttributes().get("proteinCodingGenesCount")); @@ -704,7 +716,7 @@ public void testKnockoutGenesSpecificGenesAndBiotypeNMD() throws Exception { params.setBiotype("nonsense_mediated_decay"); ExecutionResult er = toolRunner.execute(KnockoutAnalysis.class, - params.toObjectMap().append(ParamConstants.STUDY_PARAM, STUDY), outDir, null, token); + params.toObjectMap().append(ParamConstants.STUDY_PARAM, STUDY), outDir, null, false, token); checkExecutionResult(er, false); assertEquals(3, er.getAttributes().get("otherGenesCount")); // MIR1909 only has miRNA biotype assertEquals(0, er.getAttributes().get("proteinCodingGenesCount")); @@ -741,7 +753,7 @@ public void testKnockoutGenesByBiotype() throws Exception { // + "," + "TR_V_gene"); ExecutionResult er = toolRunner.execute(KnockoutAnalysis.class, - params.toObjectMap().append(ParamConstants.STUDY_PARAM, STUDY), outDir, null, token); + params.toObjectMap().append(ParamConstants.STUDY_PARAM, STUDY), outDir, null, false, token); checkExecutionResult(er, false); } @@ -753,7 +765,7 @@ public void testSampleMultiVariantFilterAnalysis() throws Exception { params.setQuery("(biotype=protein_coding AND ct=missense_variant AND gene=BRCA2) OR (gene=BTN3A2)"); ExecutionResult er = toolRunner.execute(SampleEligibilityAnalysis.class, - params.toObjectMap().append(ParamConstants.STUDY_PARAM, STUDY), outDir, null, token); + params.toObjectMap().append(ParamConstants.STUDY_PARAM, STUDY), outDir, null, false, token); // checkExecutionResult(er, false); } @@ -784,7 +796,7 @@ public void testMutationalSignatureFittingSNV() throws Exception { params.setSkip("catalogue"); toolRunner.execute(MutationalSignatureAnalysis.class, params, new ObjectMap(ParamConstants.STUDY_PARAM, CANCER_STUDY), - outDir, null, token); + outDir, null, false, token); java.io.File catalogueFile = outDir.resolve(MutationalSignatureAnalysis.SIGNATURE_COEFFS_FILENAME).toFile(); byte[] bytes = Files.readAllBytes(catalogueFile.toPath()); @@ -844,7 +856,7 @@ public void testMutationalSignatureCatalogueSV() throws Exception { params.setSkip("fitting"); toolRunner.execute(MutationalSignatureAnalysis.class, params, new ObjectMap(ParamConstants.STUDY_PARAM, CANCER_STUDY), - outDir, null, token); + outDir, null, false, token); java.io.File catalogueFile = outDir.resolve(MutationalSignatureAnalysis.CATALOGUES_FILENAME_DEFAULT).toFile(); byte[] bytes = Files.readAllBytes(catalogueFile.toPath()); @@ -904,7 +916,7 @@ public void testMutationalSignatureFittingSV() throws Exception { params.setSkip("catalogue"); toolRunner.execute(MutationalSignatureAnalysis.class, params, new ObjectMap(ParamConstants.STUDY_PARAM, CANCER_STUDY), - outDir, null, token); + outDir, null, false, token); java.io.File catalogueFile = outDir.resolve(MutationalSignatureAnalysis.SIGNATURE_COEFFS_FILENAME).toFile(); byte[] bytes = Files.readAllBytes(catalogueFile.toPath()); @@ -954,7 +966,7 @@ public void testHRDetect() throws Exception { params.setSkip("catalogue"); toolRunner.execute(MutationalSignatureAnalysis.class, params, new ObjectMap(ParamConstants.STUDY_PARAM, CANCER_STUDY), - snvFittingOutDir, null, token); + snvFittingOutDir, null, false, token); java.io.File snvSignatureFittingFile = snvFittingOutDir.resolve(MutationalSignatureAnalysis.MUTATIONAL_SIGNATURE_FITTING_DATA_MODEL_FILENAME).toFile(); assertTrue(snvSignatureFittingFile.exists()); @@ -986,7 +998,7 @@ public void testHRDetect() throws Exception { params.setSkip("catalogue"); toolRunner.execute(MutationalSignatureAnalysis.class, params, new ObjectMap(ParamConstants.STUDY_PARAM, CANCER_STUDY), - svFittingOutDir, null, token); + svFittingOutDir, null, false, token); java.io.File svSignatureFittingFile = svFittingOutDir.resolve(MutationalSignatureAnalysis.MUTATIONAL_SIGNATURE_FITTING_DATA_MODEL_FILENAME).toFile(); assertTrue(svSignatureFittingFile.exists()); @@ -1003,9 +1015,10 @@ public void testHRDetect() throws Exception { hrdParams.setIndelQuery("{\"sample\": \"" + cancer_sample + "\", \"type\": \"" + VariantType.INDEL + "\"}"); hrdParams.setBootstrap(true); - toolRunner.execute(HRDetectAnalysis.class, hrdParams, new ObjectMap(ParamConstants.STUDY_PARAM, CANCER_STUDY), hrdetectOutDir, null, token); + toolRunner.execute(HRDetectAnalysis.class, hrdParams, new ObjectMap(ParamConstants.STUDY_PARAM, CANCER_STUDY), hrdetectOutDir, null, false, token); java.io.File hrDetectFile = hrdetectOutDir.resolve(HRDetectAnalysis.HRDETECT_SCORES_FILENAME_DEFAULT).toFile(); + assertTrue("File missing : " + hrDetectFile, hrDetectFile.exists()); byte[] bytes = Files.readAllBytes(hrDetectFile.toPath()); System.out.println(new String(bytes)); assertTrue(hrDetectFile.exists()); @@ -1055,15 +1068,56 @@ public void testHRDetectParseResults() throws Exception { @Test public void testPedigreeGraph() throws CatalogException { - String base64 = "iVBORw0KGgoAAAANSUhEUgAAAeAAAAHgCAMAAABKCk6nAAAC6FBMVEUAAAABAQECAgIEBAQGBgYHBwcICAgJCQkKCgoLCwsMDAwNDQ0ODg4PDw8QEBARERESEhITExMUFBQVFRUWFhYXFxcYGBgZGRkaGhobGxsdHR0eHh4fHx8gICAhISEiIiIjIyMkJCQlJSUmJiYnJycoKCgpKSkqKiorKyssLCwtLS0uLi4vLy8wMDAxMTEyMjIzMzM0NDQ1NTU2NjY3Nzc4ODg5OTk6Ojo7Ozs8PDw9PT0+Pj4/Pz9AQEBBQUFCQkJDQ0NERERFRUVGRkZHR0dISEhJSUlKSkpLS0tMTExNTU1OTk5PT09QUFBRUVFSUlJTU1NUVFRVVVVWVlZXV1dYWFhZWVlaWlpbW1tcXFxdXV1eXl5fX19gYGBhYWFiYmJjY2NkZGRlZWVmZmZnZ2doaGhpaWlqampra2tsbGxtbW1ubm5vb29wcHBxcXFycnJzc3N0dHR1dXV2dnZ3d3d4eHh5eXl6enp7e3t9fX1+fn5/f3+AgICBgYGCgoKDg4OEhISFhYWGhoaHh4eIiIiJiYmKioqMjIyNjY2Ojo6Pj4+QkJCRkZGSkpKTk5OUlJSVlZWWlpaXl5eYmJiZmZmampqbm5ucnJydnZ2fn5+goKChoaGioqKjo6OkpKSlpaWmpqanp6eoqKipqamqqqqrq6usrKytra2urq6vr6+wsLCxsbGysrKzs7O0tLS1tbW2tra3t7e4uLi5ubm6urq7u7u8vLy+vr6/v7/AwMDBwcHCwsLDw8PExMTFxcXGxsbHx8fIyMjJycnKysrLy8vMzMzNzc3Ozs7Pz8/Q0NDR0dHS0tLT09PU1NTV1dXW1tbX19fY2NjZ2dna2trb29vc3Nzd3d3e3t7f39/g4ODh4eHi4uLj4+Pk5OTl5eXm5ubn5+fo6Ojp6enq6urr6+vs7Ozu7u7v7+/w8PDx8fHy8vLz8/P09PT19fX29vb39/f4+Pj5+fn6+vr7+/v8/Pz9/f3+/v7////lDE73AAARGklEQVR4nO3dfVwUdQLH8T2tDpXQ8ta0uB4OKq87PTpleRAhwMTKhxDsOLWEyDTS4MzrLDMf8PLSUznTyisVNbvOh9My6y6VUCyP81mUTE4BxUpFediF3783M7uzu7LKMTuLs/vt+3m9YoYZ2Pn99s0CvwXMJBh0JqMHwNo3AoNHYPAIDB6BwSMweAQGj8DgERg8AoNHYPAIDB6BwSMweAQGj8DgERg8AoNHYPAIDB6BwSMweAQGj8DgERg8AoNHYPAIDB6BwSMweAQGj8DgERg8AoNHYPAIDB6BwSMweAQGj8DgERg8AoNHYPAIDB6BwSMweAQGj8DgERg8AoNHYPAIDB6BwSMweAQGj8DgERg8HcA176u9s9S5e9p3QwuQmsu3zfv4iM3oYVwrHcBrf7PUUeKv1b3sAt8NLSA6PDFyzKwfz8mKGrvH6KFcPT3ATszJ6Z7HfhBdem5okbQJkf7bN/o354weztUisI7ODlinbEOUl/+MPG7kYK4Rgb2vbsBu+44dWByLrDFuMNeKwN6Xt8qx4wAWn6df602Ni8BeV5mo7qnAImOvMUNpJQJ73fxCdc8JXDTZmKG0EoG9bvgZdc8JbIszZiit5GPgxUPnGtD4GUZcNcx5TziBBTzwX3O2GdCwZUZctW+zOusfELAxn6Jz9hlx1dHl6p4T+HLi1d/UwAjsde8uVPecwOtnGjGQViOw19VaGhx7TuDECiMG0moE9r6l0xw7IS0P+FEE1tHot+xbB/DmwY3GjKO1CKwj67jJtfJWAW6c81itMcNoNQLrqrD/vFMKcM2blkVNRo2itXQA/+1XIx2F3aHuRS713dDannHAomHl49EpNz0a/cjSi4aNodV0ANvK1crKnLtW3w2t7RkILHex8nw73Or3aybEm0x39BqYsPo7HTcD8Ut3BgO3R6UZJqW+64pSX7/1iU+9viEC+2NnRpscdbcNK1v45oafZnj72yIE9sM+NTmLO5xms1x+snj6T7d6d1sE9r/ecfmanhq7a0X+qYdFXMX9+V7dmAbg3l3OCbE6Qt6NC6mXXi7o0/EZ+bUjCZ3uWivvvBt2Q1iJa6ueb/euC7Dm6V9ID+q1xIsLvefma0obLAZ8l7t1y+++j8mY482wtQB3m+KY4YkOXd+XNh9szJBnaA2feulfnaX7eJN545k9J1xbx/n27/oAa51+VvzZ7Z23a75Okbuv6f5PNr/4bUxzUsWswuLbtd+YNuCZXarsM5wR9cIQ5dBEeYYHOlwWIjVXiL7L7W+obh3n27/rA6xx+o2dPhciM1PrZS5YrgA2NyednrmmJLPOYk0teqBa+7C1AK8bk2Of4d0FX3ZUrqXMcF+HOmmGiaL+R/k9e+TUObcCDFjj9MtM0up4UaTWy/zuCl/ToN1P10VZRxwsKDg6whb+nPZhawIu71whz3BHxxoR/oZ8SJlh493TGnYGRYlyU0x1xS9ecW4FGrC26e81NUtfT3trvMrpK31NTw0/tHjJkVRb1OWsojWvJR7TPGxNwCI7S55hVor0aeqX8iE74IGEblGZaaLStF6Itx90bgUasLbpe/cIntYCePhIyXbcF6tnVQ0SsefSRmketjbgik6zIsTl4E5mc4ipVLgDJswT4tYNdlh1CwesafqNQTukDwaNX4ObLS2A7yleNfuUbDtly9bc87GW5v9/G1emDVjkdIsQhV1PVlVVxeYKa934rDqrEMWVla/dVivElNia031edW3V8+3e9QLWNv3MpG+LgzV+41vawtfUS8R9l/fxx3nnY5sHncxfMe4zrcPWCFwVFCEG5cqvrTVblU8nU4V4OeSmxEPSofqsYPOketdWPd/uXTdgTdO/kBZ0m9Z1cEFL4MekFXBsc/LJOatKnqzv1xj9ktZh85ksv2pCS+Cn5BVwybh6i3XkgTcXlic+rfUWCeyT8qRHcui1/gQ8z/OzmONtPc481hL4EWUFfHDJ4uMjbJEXn9mWovWCBPZJ8l367tlWTrbI8bbqGfVZz+bwlsCh0gr4iGRb+3TR+zOqk8QdTRovSGCfdBVDOetVTrp/16mecTzr2fx0mAewtErKlFbA1cliQM3vN30aMq5J2wUJrLuD/YKHZNo/Y07vcXPYbnEsMeTetUKY5/a913myemT30Dccx4TytuoZe/KKq3nCnFEtgVML7SvgFzdvm3ze0pSYMv+pJk0XJLDebGH5tq03Kvf3l6Fnxdf/td33csP2znuFOaW+WT3ZbMmr+ybsH8ox+Z1C9zjfzZ4MfPZt8QePb7KUFfAL0irp4W/++F7p4Cli+SlNFySw3nbfYhMiRbm/S7tvaxCipKv0iTIzT5i3uk6WBkvbhWOVY3Khe5zvZs/xnMm6lsDJjhXwyn+Pre/XMOql1VovSGC9/b2v9OI5+2fMdx4Mzji7/ufSgZmjhHmf6+SGG8PDw+8crhyTC93jfDd7DuCalsA9T85etUdeAaftX/bn8qFDq7VekMB6291LejFSXbXUPPK88wG133Vyr9n+HKN8TE56BKvvZk991vPhFsA/U1fAw5siL47fnKz5ggTWm+2ev4myIOX+PlBkbRiVZ7v31cadXb5S7m/1ZFPklIu2g7uVY299pHwNVt9NftX1pO6aFsBP2FfAtdk7100/M3DuSs0XJLDu9kdGp45V7u9dfbrcmvqtOJpwc3ih49GqnqzOMIf036ocGzRNefCpZ+RXXU/q2qKuBB4rr4DXyivghJqX1lgaNV+QwH7WxiuB4+wr4KmbPn3+giXnQ+23R2B/a/QVwGZpBfyJvAJO+eb1vCe8uDkC+1sXY92Beysr4LnvlY6pfyDam7+QIbDfdcr9h/5jpBWwvEoatf+VUK/++QAC+1+n49yA5RXw0j+XD1t3/1GvbozAftjliU7gqIXlI6QV8Oik8Ze8uy0C+2VfJDqAu8sr4GcnhO7w9pYI7Kd9MU4BvrfwvvE9n9jp/e0Q2G9r2r9qdua4aa/v0/VPQxAYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAwegcGDAB6i488r0YMA5iP42hEYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAwegcEjMHgEBg8CeHCR0SPw3yCA+Qi+dgQGj8DgERg8AoNHYPAIDB6BwSMweAQGj8DgERg8AoNHYPAIDB6BwSMweAQGj8DgERg8AoNHYPAIDB6BwSMweAQGj8DgERg8AoNHYPAIDB6BwSMweAQGj8DgERg8AoNHYPAIDB6BwSMweAQGj8DgERg8AoNHYPAIDB6BwSMweAQGj8DgERg8AoNHYPAIDB6BwSMweAQGj8DgERg8AoNHYPAIDB6BwSMweAQGj8DgERg8AoNHYPAIDJ5PgOvLyg1tzBZjr3/KF3diO+UT4AWx2YaWPMbY69/pizuxnfIJ8NoCX9xK4DbQ6AG0EoF9EIHBIzB4BAaPwOARGDwCg0dg8AgMHoHBIzB4BAaPwOARGDwCg0dg8AgMHoHBIzB4BAaPwOARGDwCg0dg8LCBazfnJ/SfteF7HwwmsPqT8y8buj6UHZM1dMST8dljjR6UR3qBD42Jn/mP9MSP85PS9vpkQIFTjMnZX9Oz/xKx4VdLsnoYPSiP9AE3zRj8b2kzOV16cXhEbqNPhhQoDXQBT3xtS+Lx6M+mvTjA6EF5pAvYOmpes7xVgIV4+5E6H4woYHIDXrAp+VjMF5OmfnKL0YPySBdw7mL71gEsVvvfl6B2zA14zaNH++3KlB7GMUYPyiM9wLvSHDsqsHh2s87hBFJuwCOOWkqfnL8x+WhPowflkR7gYV87dpzANQn6RhNQuQEf+PV/Ri5a/ejRqAeNHpRHOoAvJKl7TmAxskLfcAIpN2DL4ceXrxx2xFIabvSgPNIBvP0P6p4L+I0PdY0moHIDPpqy4s1RBywH06CAP1yg7rmAVy7TN5xAyg148IeLM/bLD+N+Rg/KIx3An8xU91zASwr1DSeQcgP+aG72l1FlKSsKoJ7oOJ2q7rmAx/9H33ACKTfguc+WRB+THsa/jTN6UB7p+S56wAXHjhO4sX+zzvEEUG7AU7cPOB6/cW52CdYTHSumO3acwIvm6xxOIOUG/M+44/Hbpk8oibYYPSiP9AA3Ddpl31GBD8U26B5Q4OQGnHg8dmfui9LD+HajB+WRrqcqq6NKlK0D+GDk17rHE0C5AZfHFD8/VX4Y/9LoQXmk76dJVcl/lB+zCrCtIO6ET4YUKLkBR+0dN3Oz/DD+udGD8kjnz4Ntf4l8ZXfd5PSGr2ZbXv8hfX4WVwDvGzt/Y/LxmOJn/e8fNdT9Gx3WLZOS7r/noYnr630xnEAqPVHtJw/cN7BHQmj/vmHxRg/KI4h/TphdOwKDR2DwCAwegcHTANy7yzkhVkfIu3Eh8vfMC/p0fEZ+7UhCp7vWyjvvht0QViLEhfSgXkuEa4tRm6d/ML5zj7wmIYZI66ebDRywPS3A3aY4ZniiQ9f3pc0HGzPkGVrDp176V+d9Qmwybzyz54QQWfFnt3fe7tpi1Obp9828VBa6TAJeVldn/NpRC/DMLlX2Gc6IemGIcmiiPMMDHS4LkZorzWy5crCx0+dCZGY6tyC1dfoiRJp21iQJeLlhY3VLC/C6MTn2Gd5d8GXHavmQMsN9HeqkGSaK+h/l9+yRUyfKTOeFWBTp3ILU1umLGVmXjt35kQR8++0PfWbokOU0AZd3rpBnuKNjjQh/Qz6kzLDx7mkNO4OiRLkpprriF6+IvaZmId7r7dyC1Nbpiz3hJtNz0onNXx3Ov9Hw/6OTJmCRnSXPMCtF+jBVfm6izFAcSOgWlZkmKk3rhXj7QdxHcNumf+mW2XUVFsevM6W8atyA7WkDrug0K0JcDu5kNoeYSoU6Q7mEeULcukGZYWPQDuleyHRuQWrr9I+ZLkof19H248NeNmi0zrQBi5xuEaKw68mqqqrYXGGtG59VZxWiuLLytdtqhZgSW3O6j/Qhm5n0bXHwdtcWo7ZO3/qTuY2V0RNE7arT5966YbfRw9YIXBUUIQblyq+tNVunKb+OJMTLITclHpIO1WcFmydJC4MLaUG3KetgxxajNk9/V1SX7hnfiYuxNwf1Nf7XxPlMFngEBo/A4BEYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAze/wCMbFD0pB/BRAAAAABJRU5ErkJggg=="; + String base64 = "iVBORw0KGgoAAAANSUhEUgAAAeAAAAHgCAMAAABKCk6nAAAC7lBMVEUAAAABAQECAgIEBAQGBgYHBwcICAgJCQkKCgoLCwsMDAwNDQ0ODg4PDw8QEBARERESEhITExMUFBQVFRUWFhYXFxcYGBgZGRkaGhobGxsdHR0eHh4fHx8gICAhISEiIiIjIyMkJCQlJSUmJiYnJycoKCgpKSkqKiorKyssLCwtLS0uLi4vLy8wMDAxMTEyMjIzMzM0NDQ1NTU2NjY3Nzc4ODg5OTk6Ojo7Ozs8PDw9PT0+Pj4/Pz9AQEBBQUFCQkJDQ0NERERFRUVGRkZHR0dISEhJSUlKSkpLS0tMTExNTU1OTk5PT09QUFBRUVFSUlJTU1NUVFRVVVVWVlZXV1dYWFhZWVlaWlpbW1tcXFxdXV1eXl5fX19gYGBhYWFiYmJjY2NkZGRlZWVmZmZnZ2doaGhpaWlqampra2tsbGxtbW1ubm5vb29wcHBxcXFycnJzc3N0dHR1dXV2dnZ3d3d4eHh5eXl6enp7e3t8fHx9fX1+fn5/f3+AgICBgYGCgoKDg4OEhISFhYWGhoaHh4eIiIiJiYmKioqLi4uMjIyNjY2Ojo6Pj4+QkJCRkZGSkpKTk5OUlJSVlZWXl5eYmJiZmZmampqbm5ucnJydnZ2fn5+goKChoaGioqKjo6OkpKSlpaWmpqanp6eoqKipqamqqqqrq6usrKytra2urq6vr6+wsLCxsbGysrKzs7O0tLS1tbW2tra3t7e4uLi5ubm6urq7u7u8vLy9vb2+vr6/v7/AwMDBwcHCwsLDw8PExMTFxcXGxsbHx8fIyMjJycnKysrLy8vMzMzNzc3Ozs7Pz8/Q0NDR0dHS0tLT09PU1NTV1dXW1tbX19fY2NjZ2dna2trb29vc3Nzd3d3e3t7f39/g4ODh4eHi4uLj4+Pk5OTl5eXm5ubn5+fo6Ojp6enq6urr6+vs7Ozu7u7v7+/w8PDx8fHy8vLz8/P09PT19fX29vb39/f4+Pj5+fn6+vr7+/v8/Pz9/f3+/v7///8t19oUAAARGElEQVR4nO3dfVxUdaLH8VmtLiKi6Y5psW270Grb6tK2DI9CoAYuPiyixgqroHJVSiVzXa+laeqW65qmadlaiaJtW5qV2e7mA6FSyuJDiHJLUkCxVkGeBvj9d885M2dQhLmcmYNn+Pr9vF5xDufAnN9v3s7Ab0AzCQadyegBsI6NwOARGDwCg0dg8AgMHoHBIzB4BAaPwOARGDwCg0dg8AgMHoHBIzB4BAaPwOARGDwCg0dg8AgMHoHBIzB4BAaPwOARGDwCg0dg8AgMHoHBIzB4BAaPwOARGDwCg0dg8AgMHoHBIzB4BAaPwOARGDwCg0dg8AgMHoHBIzB4BAaPwOARGDwCg0dg8AgMHoHBIzB4BAbPDeCK7WqbNjh2L+g3tE5SU/HelR8XNhg9jLZyAzj7iQ32on+l7k1dp9/QOkVfzQhKXvpfy9KCU/KMHkrruQPswJw17uZjt0XXZo7MkTa+0n8FE5+4bPRwWovAbnQpfIey9VXe/ivorJGDaSMCu15N+GHbjg1YnAmqMG4wbUVg18vcYt+xA4t949r6UOMisMuVRqt7KrBIOmrMUJxEYJf7S5a65wDOmWXMUJxEYJcbfVHdcwA3RBgzFCfpDPzKyBUGlL7YiKv6O+4JB7CAB/5rxl4DGrXRiKsOblJnfRsBG/MUnVFgxFUnFqt7DuDq6NY/1MAI7HKb16h7DuD3lhgxEKcR2OWqLHX2PQdwdIkRA3EagV1vwwL7jm/LAx4Ugd1o4mu2rR149+P1xozDWQR2I+vkWVXyVgGuXxZfZcwwnEZgt8r69crzCnDFq5a1jUaNwlluAP/tl2Pt+d+n7gVt0G9o7c84YFH39m9DYu/6TciIDZWGjcFpbgA3FKsVFTl2rfoNrf0ZCCxXWXqlA271P9umR5pM9/UfErX1ezduBuKX7gwG7ojyk0xKg3fkJLzYe8KnLt8QgT2xixNN9vo0jCpa8+r7P0py9bdFCOyBfWpyFPFVYoOl+ve5z/1oj2u3RWDPa1Ozr2lSyqG3lp8fLiJKBix36cY0AA/sflmIrYHyboRvrfR29aCu0+T3CqO6/Thb3tnsf4f/keater7DuyXAmqd/dZxX//UuXOjN63xNiY+L8O/n7Pnw6f+EJi1zZdhagHvNtc/w6y49t0ubd3YmyTO0Bsy79pm3dB/vMu+8mPd189Z+vuO7NcBap58WeWm/937N18m53tc04JPdz3wX2hRTsjQr917tN6YNeEn3MtsMFwfPjlMOzZBneKJLtRAJc4QY/IbtA9Wt/XzHd2uANU6/vts+IVJTtV7mquUGYHNTzIUl246k1lisCTk/L9c+bC3AO5IzbDN8YN0XXZVrKTMs6FIjzTBa1P5geb++GTWOrQAD1jj9IpO0Ol4bpPUyT9/gaxp2eEpNsHXMyXXrTo9pCJipfdiagIu9S+QZHuhaIQJWyYeUGdY/sKDuoFewKDaFlpc8/KxjK9CAtU3/qKlJ+no6UONVLtzoa5o0+tQr6wsTGoKr03K2PR99RvOwNQGLqWnyDNNipaepX8iHbIAnonoFpyaKUtN7Qrz+iGMr0IC1Td+1R/CCFsCjx0q2kz/furRsmAi7nDhe87C1AZd0Wxooqn26mc2+pnxxPWDUSiF6v2+DVbdwwJqmX+91QPrDoPFrcJOlBfBPcre8cF62nfvhnjlXwixN//9t3Jg2YJHRK1Bk9TxXVlYWNkdYa9LTaqxC5JaWPn9PlRBzwyouDFrUvFXPd3i3Cljb9FNjvsv10fiNb34LX1N/EfF95scfZ14Jaxp2bvlbk/+pddgagcu8AsWwOfJ72War8nQyT4iFvndFn5IO1ab5mJ+qbd6q5zu8WwasafpXE73u0boOXtcSOF5aAYc1DT23bMuR39c+Wh8yX+uw+UqWRzW9JfAkeQV8ZHKtxTr2xKtriqOnaL1FAutSpvRI9mvrr4Bn3vwsZv/Ym87EtwQeoayAT65/5eyYhqDKaXtjtV6QwLok36WbLzk52SL7x6pn1Fc9mwJaAvtJK+BCybZqSs72xeUx4r5GjRcksC61YihnbeXk9d91qmfsr3o2TfG/CVhaJaVKK+DyoSK84g+7PvWd3KjtggR2u5OP+sSl2p4xn+vbw/+wOBPt+2C2EOYVgx90nCwf28dvlf2YUD5WPWNLXnE1TV82viVwQpZtBfzM7r2zrlgao2P/MqlR0wUJ7G4N/ssb9typ3N9f+F0S//ttw88W1u33PirMsbVN6skmS2bNN/4fKMfkT/LLc3yaLRn40uvijzd9k6WsgGdLq6Th3/zpzfzH54o3zmu6IIHd7fDdDULEKvd3fp+9dUIc6Sk9UaZmCvOe5pP5PtJ2TYpyTM4vz/FptuyvmexoCTzUvgJ++1hK7aN14+dv1XpBArvb3wdLb2banjE3PeKTdOm9h6QDS8YLc0HzyffvDAgIuH+0ckzOL8/xabbswBUtgfude2FLnrwCTjy+8eXikSPLtV6QwO52uL/0Zqy6aqkY8aTjAXW8+eRRs+01RvmYnPQIVj/Nlvqq5/AWwD9VV8CjG4Mq03cP1XxBArtbw0/+Joq8lPv7RI61bnxmw4OL6g92/1K5v9WTjUFzKxtOHlaOvfaR8jVY/TT53eYXdbe1AJ5gWwFXTT2447mLQ1a8rfmCBHa740EhCSnK/X1oUPfeCd+J01E9ArLsj1b1ZHmS2ffXe5RjwxYoDz71jPxu84u6DcE3AqfIK+BseQUcVTF/m6Ve8wUJ7GHtvBE4wrYCnrfr0yevWjLe1X57BPa0Jt4AbJZWwJ/IK+DYb17MnODCzRHY06oMux54oLICXvFmfnLtz0Nc+RsyBPa4zl//Q/9kaQUsr5LGH3/Wz6V/PoDAnteFiOuA5RXwhpeLR+0YcNqlGyOwB1Y9wwEcvKZ4jLQCnhiTfs212yKwR/Z5tB24j7wC/u/pfgdcvSUCe2ifT1aAH8z6WXq/CQddvx0Ce2yNx7e8kDp5wYsFbv3TEAQGj8DgERg8AoNHYPAIDB6BwSMweAQGj8DgERg8AoNHYPAIDB6BwSMweAQGj8DgERg8AoNHYPAIDB6BwSMweAQGj8DgERg8AoNHYPAggOPc+OuV6EEA8xHcdgQGj8DgERg8AoNHYPAIDB6BwSMweAQGj8DgERg8AoNHYPAIDB6BwSMweAQGj8DgERg8AoNHYPAIDB6BwSMweAQGj8DgERg8AoNHYPAIDB6BwSMweAQGj8DgERg8AoNHYPAIDB6BwSMweAQGj8DgERg8AoNHYPAIDB6BwSMweAQGj8DgERg8AoNHYPAIDB6BwSMweAQGj8DgERg8AoNHYPAIDB6BwSMweAQGj8DgERg8AoNHYPAIDB6BwSMweAQGj8DgERg8AoNHYPAIDB6BwSMweAQGj8DgERg8AoNHYPAIDB6BwSMweAQGj8DgERg8AoNHYPAIDB6BwYMAfjzH6BF4bhDAfAS3HYHBIzB4BAaPwOARGDwCg0dg8AgMHoHBIzB4BAaPwOARGDwCg0dg8AgMHoHBIzB4BAaPwOARGDwCg0dg8AgMHoHBIzB4BAaPwOARGDwCg0dg8AgMHoHBIzB4BAaPwOARGDwCg0dg8AgMHoHBIzB4BAaPwOARGDwCg0dg8AgMHoHBIzB4BAaPwOARGDwCg0dg8AgMHoHBIzB4BAaPwOARGDwCg0dg8AgMHoHBIzB4BAaPwOARGDwCg6cLcG1RsaElf2js9c/rcSd2ULoArw6bamhDk429/v163IkdlC7A2ev0uJXO2xCjB+AkAusQgcEjMHgEBo/A4BEYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwQIFPOX6zP3jw8PFJQ6eETh2R+KV+Q+skFW76Y+8/vHqsyehxtJEbwNkmRwPD3304K37WqqBnVug3tM6QdWNoyqYc79y30i0rq40eTKvpA3zvacu/U17eHlPo/z+6jawzVBj+UpW08ZX+q3vdkmv0cFpLH+BYy6mxm7LiT1teuq2+Gh8KOatsfZW3F6N3GTmYNtIH+O6iMX99bWSh5d+hT+k3NI/v26BLth0bsLgWmW/cYNpKH+DfxO1YP+GE9DBOv50ewaPV7yjtwOJceKNRY2kzfYDNO1/6XYHlqzGb02frNzRPryBJ3VOBxfwPjBmKk/QBjlwxLS/4TOw760PX6jc0T2/hJ+qeA/h0sjFDcZJOy6QZh0PODv/7i2kvLdRvaJ7e8Cp1zwEswgwZibP0AX5oX/jZqD0rph3yX6bf0Dy9CMeebyvHPCV9gCPCiof849mnDwSt+bN+Q/P0Iq3qXjNwuCEjcZY+wH3PhuXMnrcv4syAefoNzdObcVTdcwBXxBszFCfpAxwfemjmoo/Ci4csvI2WSR/PV/ccwOs3GDMUJ+n0Qkf+pD9/ECM9jOOe1G9onl5DiP11DgdwTVClUYNpM32A45Jfzo45Iz2Mx95Gj2Dx2Uj76xoqcHqWYWNpM51+2KC8EH1s0qpnMvUbmue3NqVO2dqAm+Z74jcg+gCHbBx1/FcFE9dk/3K1fkPrBG0eckzeKMBnR3jkj0r1AQ584oTlVMKmLfGrn9dvaJ2hs+Pit3wrAV98d0LcMaMH02puAP/r0Wh7P+3xWL/QAQ+F93us7+30NVjp3JpxkT+M/O2fCo0eSBtB/HPCrO0IDB6BwSMweAQGTwPwwO6XhdgaKO9G+NZKb1cP6jpNfq8wqtuPs+Wdzf53+B8R4mSkd9/MRiHipAVUjw4YszG1e/pXx3n1Xy+at8amBbjXXPsMv+7Sc7u0eWdnkjxDa8C8a595Fwixy7zzYt7XQgxOvVbkt1EC3lhTU9tB4771tXv6aZGX9nvvb94amxbgJd3LbDNcHDw7Tjk0Q57hiS7VQiTMkWDfsH2g7z5pdk9JwG/oPlwDa+/067tJ009NdWwNTgvwjuQM2wwfWPdF13L5kDLDgi410gyjRe0PlvfrmyHtL067dub+jyTge+997J8dMmojau/0i0xXhFgb5NganCbgYu8SeYYHulaIgFXyIWWG9Q8sqDvoFSyKTaHlJQ8/K0RegMk0Uzqx+8uvlt8J8780au/0j5qahHhzoGNrcJqAxdQ0eYZpsdKj9BfyIWWG4kRUr+DURFFqek+I1x8R1+5+oabEssT2SbGL9B+zMbV3+p35ESxKui0NFNU+3cxmX5P8W/y2GcpFrRSi9/vKDM+YKqWZhdiOj4L5Ncv2Tr/e64D0hyDVsTU4bcAio1egyOp5rqysLGyOsNakp9VYhcgtLX3+nioh5oZVXBi0SFh/uKK+NGS6qNpy4fJrdxzuuLHf2to7fZEa812uz/7mrbFpBC7zChTD5sjvZZutC+SfFM4TYqHvXdGnpEO1aT7mp6R10aHg7n2SvheVYT28Br/bQeO+9bV7+lcTve5R1sH2rbHxlSzwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAwegcEjMHgEBo/A4BEYPAKDR2DwCAwegcEjMHgEBo/A4BEYvP8D3o1BCHWO4+EAAAAASUVORK5CYII="; - OpenCGAResult results = catalogManager.getFamilyManager().search(STUDY, new Query("id", "f1"), QueryOptions.empty(), token); + OpenCGAResult results = catalogManager.getFamilyManager().get(STUDY, "f1", QueryOptions.empty(), token); Family family = results.first(); - assertTrue(family.getPedigreeGraph() != null); + assertNotNull(family.getPedigreeGraph()); assertEquals(base64, family.getPedigreeGraph().getBase64()); } + @Test + public void testClinicalAnalysisLoading() throws IOException, ToolException, CatalogException { + String fileStr = "clinical_analyses.json.gz"; + File file; + try (InputStream stream = getClass().getResourceAsStream("/biofiles/" + fileStr)) { + file = catalogManager.getFileManager().upload(CANCER_STUDY, stream, new File().setPath("biofiles/" + fileStr), false, true, false, token).first(); + } + + System.out.println("file ID = " + file.getId()); + System.out.println("file name = " + file.getName()); + + // Run clinical analysis load task + Path loadingOutDir = Paths.get(opencga.createTmpOutdir("_clinical_analysis_outdir")); + System.out.println("Clinical analysis load task out dir = " + loadingOutDir); + + ClinicalAnalysisLoadParams params = new ClinicalAnalysisLoadParams(); + params.setFile(file.getId()); + + toolRunner.execute(ClinicalAnalysisLoadTask.class, params, new ObjectMap(ParamConstants.STUDY_PARAM, + CANCER_STUDY), loadingOutDir, null, false, token); + + String ca1Id = "SAP-45016-1"; + String ca2Id = "OPA-6607-1"; + + Query query = new Query(); + OpenCGAResult result = catalogManager.getClinicalAnalysisManager().search(CANCER_STUDY, query, QueryOptions.empty(), + token); + Assert.assertTrue(result.getResults().stream().map(ca -> ca.getId()).collect(Collectors.toList()).contains(ca1Id)); + Assert.assertTrue(result.getResults().stream().map(ca -> ca.getId()).collect(Collectors.toList()).contains(ca2Id)); + + query.put("id", ca1Id); + ClinicalAnalysis clinicalAnalysis = catalogManager.getClinicalAnalysisManager().search(CANCER_STUDY, query, QueryOptions.empty(), + token).first(); + Assert.assertEquals(ca1Id, clinicalAnalysis.getId()); + + query.put("id", ca2Id); + clinicalAnalysis = catalogManager.getClinicalAnalysisManager().search(CANCER_STUDY, query, QueryOptions.empty(), + token).first(); + Assert.assertEquals(ca2Id, clinicalAnalysis.getId()); + } + @Test public void testCellbaseConfigure() throws Exception { String project = "Project_test_cellbase_configure"; diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantCatalogQueryUtilsTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantCatalogQueryUtilsTest.java index b38101ef933..25563473655 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantCatalogQueryUtilsTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantCatalogQueryUtilsTest.java @@ -41,6 +41,7 @@ import org.opencb.opencga.catalog.managers.CatalogManager; import org.opencb.opencga.catalog.managers.CatalogManagerExternalResource; import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.cohort.Cohort; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.common.InternalStatus; @@ -52,10 +53,11 @@ import org.opencb.opencga.core.models.individual.Individual; import org.opencb.opencga.core.models.individual.IndividualInternal; import org.opencb.opencga.core.models.individual.Location; +import org.opencb.opencga.core.models.organizations.OrganizationCreateParams; +import org.opencb.opencga.core.models.organizations.OrganizationUpdateParams; import org.opencb.opencga.core.models.panel.Panel; +import org.opencb.opencga.core.models.project.Project; import org.opencb.opencga.core.models.sample.Sample; -import org.opencb.opencga.core.models.user.Account; -import org.opencb.opencga.core.models.user.User; import org.opencb.opencga.core.testclassification.duration.MediumTests; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; @@ -88,7 +90,7 @@ @Category(MediumTests.class) public class VariantCatalogQueryUtilsTest { - @ClassRule + @ClassRule(order = -1) public static CatalogManagerExternalResource catalogManagerExternalResource = new CatalogManagerExternalResource(); private static String assembly; @@ -96,6 +98,7 @@ public class VariantCatalogQueryUtilsTest { public ExpectedException thrown = ExpectedException.none(); private static CatalogManager catalog; + private static final String ORGANIZATION = "test"; private static String sessionId; private static VariantCatalogQueryUtils queryUtils; private static List samples = new ArrayList<>(); @@ -109,18 +112,26 @@ public class VariantCatalogQueryUtilsTest { private static Panel myPanelWithRegions; private static CellBaseUtils cellBaseUtils; + private static final QueryOptions INCLUDE_RESULT = new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true); + @BeforeClass public static void setUp() throws Exception { catalog = catalogManagerExternalResource.getCatalogManager(); - User user = catalog.getUserManager().create("user", "user", "my@email.org", TestParamConstants.PASSWORD, "ACME", 1000L, Account.AccountType.FULL, catalogManagerExternalResource.getAdminToken()).first(); + catalog.getOrganizationManager().create(new OrganizationCreateParams().setId(ORGANIZATION), QueryOptions.empty(), + catalogManagerExternalResource.getAdminToken()); + catalog.getUserManager().create("user", "user", "my@email.org", TestParamConstants.PASSWORD, ORGANIZATION, 1000L, + catalogManagerExternalResource.getAdminToken()).first(); + catalog.getOrganizationManager().update(ORGANIZATION, new OrganizationUpdateParams().setAdmins(Collections.singletonList("user")), + null, + catalogManagerExternalResource.getAdminToken()); - sessionId = catalog.getUserManager().login("user", TestParamConstants.PASSWORD).getToken(); + sessionId = catalog.getUserManager().login(ORGANIZATION, "user", TestParamConstants.PASSWORD).getToken(); assembly = "GRCh38"; - catalog.getProjectManager().create("p1", "p1", "", "hsapiens", "Homo Sapiens", assembly, null, sessionId); - catalog.getStudyManager().create("p1", "s1", "s1", "s1", null, null, null, null, null, null, sessionId); - catalog.getStudyManager().create("p1", "s2", "s2", "s2", null, null, null, null, null, null, sessionId); - catalog.getStudyManager().create("p1", "s3", "s3", "s3", null, null, null, null, null, null, sessionId); + Project project = catalog.getProjectManager().create("p1", "p1", "", "hsapiens", "Homo Sapiens", assembly, INCLUDE_RESULT, sessionId).first(); + catalog.getStudyManager().create(project.getFqn(), "s1", "s1", "s1", null, null, null, null, null, null, sessionId); + catalog.getStudyManager().create(project.getFqn(), "s2", "s2", "s2", null, null, null, null, null, null, sessionId); + catalog.getStudyManager().create(project.getFqn(), "s3", "s3", "s3", null, null, null, null, null, null, sessionId); file1 = createFile("data/file1.vcf"); file2 = createFile("data/file2.vcf"); @@ -238,7 +249,7 @@ public static File createFile(String path, boolean indexed) throws CatalogExcept true, sessionId).first(); if (indexed) { int release = catalog.getProjectManager().get("p1", null, sessionId).first().getCurrentRelease(); - catalog.getFileManager().updateFileInternalVariantIndex(file, new FileInternalVariantIndex() + catalog.getFileManager().updateFileInternalVariantIndex("s1", file, new FileInternalVariantIndex() .setStatus(new VariantIndexStatus(InternalStatus.READY)) .setRelease(release), sessionId); } @@ -352,10 +363,10 @@ public void cohortStatsWrongRelease() throws Exception { @Test public void parseQuery() throws Exception { - assertEquals("user@p1:s1", parseValue(STUDY, "s1")); - assertEquals("user@p1:s1,user@p1:s2", parseValue(STUDY, "s1,s2")); - assertEquals("!user@p1:s1,user@p1:s2", parseValue(STUDY, "!s1,s2")); - assertEquals("user@p1:s2;!user@p1:s1;user@p1:s3", parseValue(STUDY, "user@p1:s2;!s1;p1:s3")); + assertEquals("test@p1:s1", parseValue(STUDY, "s1")); + assertEquals("test@p1:s1,test@p1:s2", parseValue(STUDY, "s1,s2")); + assertEquals("!test@p1:s1,test@p1:s2", parseValue(STUDY, "!s1,s2")); + assertEquals("test@p1:s2;!test@p1:s1;test@p1:s3", parseValue(STUDY, "test@p1:s2;!s1;p1:s3")); assertEquals(file1.getName(), parseValue("s1", FILE, file1.getName())); assertEquals(file1.getName(), parseValue("s1", FILE, file1.getId())); @@ -375,24 +386,24 @@ public void parseQuery() throws Exception { assertEquals("c1", parseValue("s1", COHORT, "s1:c1")); assertEquals("c1>0.1", parseValue("s1", STATS_MAF, "s1:c1>0.1")); - assertEquals("c1", parseValue("s1", COHORT, "user@p1:s1:c1")); - assertEquals("c1>0.1", parseValue("s1", STATS_MAF, "user@p1:s1:c1>0.1")); + assertEquals("c1", parseValue("s1", COHORT, "test@p1:s1:c1")); + assertEquals("c1>0.1", parseValue("s1", STATS_MAF, "test@p1:s1:c1>0.1")); - assertEquals("user@p1:s1:ALL;user@p1:s2:ALL", parseValue("s1,s2", COHORT, "s1:ALL;s2:ALL")); - assertEquals("user@p1:s1:ALL>0.1;user@p1:s2:ALL>0.1", parseValue("s1,s2", STATS_MAF, "s1:ALL>0.1;s2:ALL>0.1")); + assertEquals("test@p1:s1:ALL;test@p1:s2:ALL", parseValue("s1,s2", COHORT, "s1:ALL;s2:ALL")); + assertEquals("test@p1:s1:ALL>0.1;test@p1:s2:ALL>0.1", parseValue("s1,s2", STATS_MAF, "s1:ALL>0.1;s2:ALL>0.1")); } @Test public void queryBySavedFilter() throws Exception { - String userId = catalog.getUserManager().getUserId(sessionId); + String userId = new JwtPayload(sessionId).getUserId(); catalog.getUserManager().addFilter(userId, "myFilter", "", Enums.Resource.VARIANT, new Query("key1", "value1").append("key2", "value2"), new QueryOptions(), sessionId); Query query = queryUtils.parseQuery(new Query(STUDY.key(), "s1").append(SAVED_FILTER.key(), "myFilter"), null, sessionId); - assertEquals(new Query().append(STUDY.key(), "user@p1:s1").append(SAVED_FILTER.key(), "myFilter").append("key1", "value1").append("key2", "value2"), query); + assertEquals(new Query().append(STUDY.key(), "test@p1:s1").append(SAVED_FILTER.key(), "myFilter").append("key1", "value1").append("key2", "value2"), query); query = queryUtils.parseQuery(new Query(STUDY.key(), "s1").append(SAVED_FILTER.key(), "myFilter").append("key1", "otherValue"), null, sessionId); - assertEquals(new Query().append(STUDY.key(), "user@p1:s1").append(SAVED_FILTER.key(), "myFilter").append("key1", "otherValue").append("key2", "value2"), query); + assertEquals(new Query().append(STUDY.key(), "test@p1:s1").append(SAVED_FILTER.key(), "myFilter").append("key1", "otherValue").append("key2", "value2"), query); } @Test @@ -548,7 +559,7 @@ public void queryByFamilyWithoutStudy() throws CatalogException { @Test public void queryByFamilyNotFound() throws CatalogException { - CatalogException e = new CatalogException("Missing families: asdf not found"); + CatalogException e = new CatalogException("Missing families: 'asdf' not found"); thrown.expectMessage(e.getMessage()); thrown.expect(e.getClass()); queryUtils.parseQuery(new Query(STUDY.key(), "s1").append(FAMILY.key(), "asdf").append(FAMILY_DISORDER.key(), "asdf"), null, sessionId); @@ -841,11 +852,11 @@ public void queryByPanelNotFound() throws Exception { @Test public void getAnyStudy() throws Exception { - assertEquals("user@p1:s1", queryUtils.getAnyStudy(new Query(PROJECT.key(), "p1"), sessionId)); - assertEquals("user@p2:p2s2", queryUtils.getAnyStudy(new Query(PROJECT.key(), "p2"), sessionId)); - assertEquals("user@p2:p2s2", queryUtils.getAnyStudy(new Query(STUDY.key(), "p2s2"), sessionId)); - assertEquals("user@p2:p2s2", queryUtils.getAnyStudy(new Query(INCLUDE_STUDY.key(), "p2s2"), sessionId)); - assertEquals("user@p2:p2s2", queryUtils.getAnyStudy(new Query(STUDY.key(), "p2s2").append(INCLUDE_STUDY.key(), "all"), sessionId)); + assertEquals("test@p1:s1", queryUtils.getAnyStudy(new Query(PROJECT.key(), "p1"), sessionId)); + assertEquals("test@p2:p2s2", queryUtils.getAnyStudy(new Query(PROJECT.key(), "p2"), sessionId)); + assertEquals("test@p2:p2s2", queryUtils.getAnyStudy(new Query(STUDY.key(), "p2s2"), sessionId)); + assertEquals("test@p2:p2s2", queryUtils.getAnyStudy(new Query(INCLUDE_STUDY.key(), "p2s2"), sessionId)); + assertEquals("test@p2:p2s2", queryUtils.getAnyStudy(new Query(STUDY.key(), "p2s2").append(INCLUDE_STUDY.key(), "all"), sessionId)); thrown.expect(CatalogException.class); thrown.expectMessage("Multiple projects"); queryUtils.getAnyStudy(new Query(), sessionId); diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantManagerFetchTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantManagerFetchTest.java index 4caca0d7c20..2002b8a42ba 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantManagerFetchTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantManagerFetchTest.java @@ -26,6 +26,7 @@ import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.analysis.variant.manager.operations.AbstractVariantOperationManagerTest; +import org.opencb.opencga.analysis.variant.stats.VariantStatsAnalysis; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.api.ParamConstants; @@ -56,9 +57,17 @@ @Category(MediumTests.class) public class VariantManagerFetchTest extends AbstractVariantOperationManagerTest { + private String anonToken; + @Before public void setUp() throws Exception { indexFile(getSmallFile(), new QueryOptions(), outputId); + + String dummyStudy = catalogManager.getStudyManager().create(projectId, "s_dummy", "s_dummy", "s_dummy", "", null, null, + null, null, null, sessionId).first().getId(); + catalogManager.getStudyManager().updateGroup(dummyStudy, ParamConstants.MEMBERS_GROUP, ParamUtils.BasicUpdateAction.ADD, + new GroupUpdateParams(Collections.singletonList(ParamConstants.ANONYMOUS_USER_ID)), sessionId); + anonToken = catalogManager.getUserManager().loginAnonymous(ORGANIZATION).getToken(); } @Override @@ -142,12 +151,12 @@ public void testQueryAnonymousOnlyAggregated() throws Exception { // Only Aggregated studies catalogManager.getStudyManager().updateGroup(studyFqn, "@members", ParamUtils.BasicUpdateAction.ADD, new GroupUpdateParams(Collections.singletonList("*")), sessionId); - catalogManager.getStudyManager().updateAcl(Collections.singletonList(studyFqn), "*", + catalogManager.getStudyManager().updateAcl(studyFqn, "*", new StudyAclParams(StudyPermissions.Permissions.VIEW_AGGREGATED_VARIANTS.name(), null), ADD, sessionId); - Query query = new Query(VariantQueryParam.STUDY.key(), userId + "@p1:s1"); - DataResult result = variantManager.get(query, new QueryOptions(), null); + Query query = new Query(VariantQueryParam.STUDY.key(), ORGANIZATION + "@p1:s1"); + DataResult result = variantManager.get(query, new QueryOptions(), anonToken); Assert.assertNotEquals(0, result.getNumResults()); for (Variant variant : result.getResults()) { Assert.assertEquals(0, variant.getStudies().get(0).getSamples().size()); @@ -158,7 +167,7 @@ public void testQueryAnonymousOnlyAggregated() throws Exception { public void testQueryAnonymousViewSampleVariants() throws Exception { catalogManager.getStudyManager().updateGroup(studyFqn, "@members", ParamUtils.BasicUpdateAction.ADD, new GroupUpdateParams(Collections.singletonList("*")), sessionId); - catalogManager.getStudyManager().updateAcl(Collections.singletonList(studyFqn), "*", + catalogManager.getStudyManager().updateAcl(studyFqn, "*", new StudyAclParams(StudyPermissions.Permissions.VIEW_AGGREGATED_VARIANTS.name(), null), ADD, sessionId); catalogManager.getSampleManager().updateAcl(studyFqn, Arrays.asList("NA19600"), "*", @@ -168,8 +177,8 @@ public void testQueryAnonymousViewSampleVariants() throws Exception { new SampleAclParams().setPermissions(SamplePermissions.VIEW_VARIANTS.name()), // ViewVariants without VIEW should be enough ADD, sessionId); - Query query = new Query(VariantQueryParam.STUDY.key(), userId + "@p1:s1").append(VariantQueryParam.INCLUDE_SAMPLE.key(), ParamConstants.ALL); - DataResult result = variantManager.get(query, new QueryOptions(), null); + Query query = new Query(VariantQueryParam.STUDY.key(), ORGANIZATION + "@p1:s1").append(VariantQueryParam.INCLUDE_SAMPLE.key(), ParamConstants.ALL); + DataResult result = variantManager.get(query, new QueryOptions(), anonToken); Assert.assertNotEquals(0, result.getNumResults()); for (Variant variant : result.getResults()) { Assert.assertEquals(1, variant.getStudies().get(0).getSamples().size()); @@ -186,17 +195,17 @@ public void testQueryAnonymousViewSampleVariantsWithoutAggregatedVariants() thro .setPermissions(SamplePermissions.VIEW + "," + SamplePermissions.VIEW_VARIANTS), ADD, sessionId); - Query query = new Query(VariantQueryParam.STUDY.key(), userId + "@p1:s1").append(VariantQueryParam.INCLUDE_SAMPLE.key(), ParamConstants.ALL); - DataResult result = variantManager.get(query, new QueryOptions(), null); + Query query = new Query(VariantQueryParam.STUDY.key(), ORGANIZATION + "@p1:s1").append(VariantQueryParam.INCLUDE_SAMPLE.key(), ParamConstants.ALL); + DataResult result = variantManager.get(query, new QueryOptions(), anonToken); Assert.assertNotEquals(0, result.getNumResults()); for (Variant variant : result.getResults()) { Assert.assertEquals(2, variant.getStudies().get(0).getSamples().size()); } - query = new Query(VariantQueryParam.STUDY.key(), userId + "@p1:s1") + query = new Query(VariantQueryParam.STUDY.key(), ORGANIZATION + "@p1:s1") .append(VariantQueryParam.SAMPLE.key(), "NA19600") .append(VariantQueryParam.INCLUDE_SAMPLE.key(), ParamConstants.ALL); - result = variantManager.get(query, new QueryOptions(), null); + result = variantManager.get(query, new QueryOptions(), anonToken); Assert.assertNotEquals(0, result.getNumResults()); for (Variant variant : result.getResults()) { Assert.assertEquals(2, variant.getStudies().get(0).getSamples().size()); @@ -205,7 +214,7 @@ public void testQueryAnonymousViewSampleVariantsWithoutAggregatedVariants() thro @Test public void testQueryAnonymousViewSampleVariantsWithoutAggregatedVariantsFail1() throws Exception { - catalogManager.getStudyManager().updateAcl(Collections.singletonList(studyFqn), "*", + catalogManager.getStudyManager().updateAcl(studyFqn, "*", new StudyAclParams(StudyPermissions.Permissions.VIEW_SAMPLES.name(), null), ADD, sessionId); // Only 2 samples catalogManager.getSampleManager().updateAcl(studyFqn, Arrays.asList("NA19600", "NA19660"), "*", @@ -214,18 +223,18 @@ public void testQueryAnonymousViewSampleVariantsWithoutAggregatedVariantsFail1() ADD, sessionId); // Filter sample "NA19601" is unauthorized, even if the result is not returned - Query query = new Query(VariantQueryParam.STUDY.key(), userId + "@p1:s1") + Query query = new Query(VariantQueryParam.STUDY.key(), ORGANIZATION + "@p1:s1") .append(VariantQueryParam.INCLUDE_SAMPLE.key(), "NA19600,NA19660") .append(VariantQueryParam.SAMPLE.key(), "NA19661"); thrown.expectMessage("'sample'"); thrown.expect(CatalogAuthorizationException.class); - variantManager.get(query, new QueryOptions(), null); + variantManager.get(query, new QueryOptions(), anonToken); } @Test public void testQueryAnonymousViewSampleVariantsWithoutAggregatedVariantsFail2() throws Exception { - catalogManager.getStudyManager().updateAcl(Collections.singletonList(studyFqn), "*", + catalogManager.getStudyManager().updateAcl(studyFqn, "*", new StudyAclParams(StudyPermissions.Permissions.VIEW_SAMPLES.name(), null), ADD, sessionId); // Only 2 samples catalogManager.getSampleManager().updateAcl(studyFqn, Arrays.asList("NA19600", "NA19660"), "*", @@ -234,25 +243,25 @@ public void testQueryAnonymousViewSampleVariantsWithoutAggregatedVariantsFail2() ADD, sessionId); // Include sample "NA19601" is unauthorized - Query query = new Query(VariantQueryParam.STUDY.key(), userId + "@p1:s1") + Query query = new Query(VariantQueryParam.STUDY.key(), ORGANIZATION + "@p1:s1") .append(VariantQueryParam.INCLUDE_SAMPLE.key(), "NA19600,NA19660,NA19601"); thrown.expectMessage("'includeSample'"); thrown.expect(CatalogAuthorizationException.class); - variantManager.get(query, new QueryOptions(), null); + variantManager.get(query, new QueryOptions(), anonToken); } @Test public void testQueryAnonymousViewSampleVariantsStudyLevel() throws Exception { // All samples - catalogManager.getStudyManager().updateAcl(Collections.singletonList(studyFqn), "*", + catalogManager.getStudyManager().updateAcl(studyFqn, "*", new StudyAclParams() // VIEW_SAMPLE_VARIANTS without VIEW_SAMPLES should be enough .setPermissions(StudyPermissions.Permissions.VIEW_SAMPLE_VARIANTS.name()), ADD, sessionId); - Query query = new Query(VariantQueryParam.STUDY.key(), userId + "@p1:s1").append(VariantQueryParam.INCLUDE_SAMPLE.key(), ParamConstants.ALL); - DataResult result = variantManager.get(query, new QueryOptions(), null); + Query query = new Query(VariantQueryParam.STUDY.key(), ORGANIZATION + "@p1:s1").append(VariantQueryParam.INCLUDE_SAMPLE.key(), ParamConstants.ALL); + DataResult result = variantManager.get(query, new QueryOptions(), anonToken); Assert.assertNotEquals(0, result.getNumResults()); for (Variant variant : result.getResults()) { Assert.assertEquals(4, variant.getStudies().get(0).getSamples().size()); @@ -265,7 +274,7 @@ public void testQueryAnonymousWithoutPermissions() throws Exception { CatalogAuthorizationException expected = CatalogAuthorizationException.denyAny(ParamConstants.ANONYMOUS_USER_ID, "view", "study"); thrown.expectMessage(expected.getMessage()); thrown.expect(expected.getClass()); - variantManager.get(query, new QueryOptions(), null); + variantManager.get(query, new QueryOptions(), anonToken); } @Test @@ -276,14 +285,14 @@ public void testQueryAnonymousInMembersWithoutPermissions() throws Exception { Query query = new Query(VariantQueryParam.STUDY.key(), studyId); thrown.expectMessage("Permission denied"); thrown.expect(CatalogAuthorizationException.class); - variantManager.get(query, new QueryOptions(), null); + variantManager.get(query, new QueryOptions(), anonToken); } @Test public void testQueryAnonymousOneStudyPermissions() throws Exception { - catalogManager.getStudyManager().updateGroup(studyFqn, "@members", ParamUtils.BasicUpdateAction.ADD, - new GroupUpdateParams(Collections.singletonList("*")), sessionId); - catalogManager.getStudyManager().updateAcl(Collections.singletonList(studyFqn), "*", + catalogManager.getStudyManager().updateGroup(studyFqn, ParamConstants.MEMBERS_GROUP, ParamUtils.BasicUpdateAction.ADD, + new GroupUpdateParams(Collections.singletonList(ParamConstants.ANONYMOUS_USER_ID)), sessionId); + catalogManager.getStudyManager().updateAcl(studyFqn, "*", new StudyAclParams() .setPermissions(StudyPermissions.Permissions.VIEW_AGGREGATED_VARIANTS.name()), ADD, sessionId); @@ -292,13 +301,13 @@ public void testQueryAnonymousOneStudyPermissions() throws Exception { indexFile(file, new QueryOptions(), outputId2); Query query = new Query(); - DataResult result = variantManager.get(query, new QueryOptions(QueryOptions.EXCLUDE, VariantField.STUDIES.name()), null); + DataResult result = variantManager.get(query, new QueryOptions(QueryOptions.EXCLUDE, VariantField.STUDIES.name()), anonToken); Assert.assertNotEquals(0, result.getNumResults()); for (Variant variant : result.getResults()) { Assert.assertEquals(0, variant.getStudies().size()); } - result = variantManager.get(query, new QueryOptions(), null); + result = variantManager.get(query, new QueryOptions(), anonToken); Assert.assertNotEquals(0, result.getNumResults()); for (Variant variant : result.getResults()) { Assert.assertEquals(1, variant.getStudies().size()); @@ -309,7 +318,7 @@ public void testQueryAnonymousOneStudyPermissions() throws Exception { public void testQueryAnonymousOneStudyPermissionsIncludeBoth() throws Exception { catalogManager.getStudyManager().updateGroup(studyFqn, "@members", ParamUtils.BasicUpdateAction.ADD, new GroupUpdateParams(Collections.singletonList("*")), sessionId); - catalogManager.getStudyManager().updateAcl(Collections.singletonList(studyFqn), "*", + catalogManager.getStudyManager().updateAcl(studyFqn, "*", new StudyAclParams() .setPermissions(StudyPermissions.Permissions.VIEW_AGGREGATED_VARIANTS.name()), ADD, sessionId); @@ -317,7 +326,7 @@ public void testQueryAnonymousOneStudyPermissionsIncludeBoth() throws Exception File file = create(studyId2, getResourceUri("variant-test-file.vcf.gz")); indexFile(file, new QueryOptions(), outputId2); - DataResult result = variantManager.get(new Query(), new QueryOptions(QueryOptions.EXCLUDE, VariantField.STUDIES.name()), null); + DataResult result = variantManager.get(new Query(), new QueryOptions(QueryOptions.EXCLUDE, VariantField.STUDIES.name()), anonToken); Assert.assertNotEquals(0, result.getNumResults()); for (Variant variant : result.getResults()) { Assert.assertEquals(0, variant.getStudies().size()); @@ -326,14 +335,14 @@ public void testQueryAnonymousOneStudyPermissionsIncludeBoth() throws Exception CatalogAuthorizationException expected = CatalogAuthorizationException.denyAny(ParamConstants.ANONYMOUS_USER_ID, "view", "study"); thrown.expectMessage(expected.getMessage()); thrown.expect(expected.getClass()); - variantManager.get(new Query(VariantQueryParam.INCLUDE_STUDY.key(), studyId + "," + studyId2), new QueryOptions(), null); + variantManager.get(new Query(VariantQueryParam.INCLUDE_STUDY.key(), studyId + "," + studyId2), new QueryOptions(), anonToken); } @Test public void testQueryAnonymousTwoStudiesPermissions() throws Exception { catalogManager.getStudyManager().updateGroup(studyFqn, "@members", ParamUtils.BasicUpdateAction.ADD, new GroupUpdateParams(Collections.singletonList("*")), sessionId); - catalogManager.getStudyManager().updateAcl(Collections.singletonList(studyFqn), "*", + catalogManager.getStudyManager().updateAcl(studyFqn, "*", new StudyAclParams() .setPermissions(StudyPermissions.Permissions.VIEW_AGGREGATED_VARIANTS.name()), ADD, sessionId); @@ -344,19 +353,19 @@ public void testQueryAnonymousTwoStudiesPermissions() throws Exception { catalogManager.getStudyManager().updateGroup(studyId2, "@members", ParamUtils.BasicUpdateAction.ADD, new GroupUpdateParams(Collections.singletonList("*")), sessionId); - catalogManager.getStudyManager().updateAcl(Collections.singletonList(studyId2), "*", + catalogManager.getStudyManager().updateAcl(studyId2, "*", new StudyAclParams() .setPermissions(StudyPermissions.Permissions.VIEW_AGGREGATED_VARIANTS.name()), ADD, sessionId); Query query = new Query(); - DataResult result = variantManager.get(query, new QueryOptions(QueryOptions.EXCLUDE, VariantField.STUDIES.name()), null); + DataResult result = variantManager.get(query, new QueryOptions(QueryOptions.EXCLUDE, VariantField.STUDIES.name()), anonToken); Assert.assertNotEquals(0, result.getNumResults()); for (Variant variant : result.getResults()) { Assert.assertEquals(0, variant.getStudies().size()); } - result = variantManager.get(query, new QueryOptions(), null); + result = variantManager.get(query, new QueryOptions(), anonToken); Assert.assertNotEquals(0, result.getNumResults()); for (Variant variant : result.getResults()) { Assert.assertEquals(2, variant.getStudies().size()); diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java index 406103ce71e..ef6b79bd153 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java @@ -17,6 +17,7 @@ package org.opencb.opencga.analysis.variant.manager; import org.hamcrest.CoreMatchers; +import org.hamcrest.MatcherAssert; import org.junit.*; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -32,6 +33,7 @@ import org.opencb.opencga.analysis.variant.OpenCGATestExternalResource; import org.opencb.opencga.analysis.variant.gwas.GwasAnalysis; import org.opencb.opencga.analysis.variant.operations.*; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.CatalogManager; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.JacksonUtils; @@ -47,19 +49,20 @@ import org.opencb.opencga.core.models.file.File; import org.opencb.opencga.core.models.individual.*; import org.opencb.opencga.core.models.job.Job; -import org.opencb.opencga.core.models.operations.variant.VariantAnnotationIndexParams; -import org.opencb.opencga.core.models.operations.variant.VariantSecondaryAnnotationIndexParams; -import org.opencb.opencga.core.models.operations.variant.VariantSecondarySampleIndexParams; +import org.opencb.opencga.core.models.operations.variant.*; +import org.opencb.opencga.core.models.organizations.OrganizationCreateParams; +import org.opencb.opencga.core.models.organizations.OrganizationUpdateParams; import org.opencb.opencga.core.models.project.ProjectCreateParams; import org.opencb.opencga.core.models.project.ProjectOrganism; import org.opencb.opencga.core.models.sample.*; -import org.opencb.opencga.core.models.user.Account; -import org.opencb.opencga.core.models.operations.variant.VariantIndexParams; -import org.opencb.opencga.core.models.operations.variant.VariantStorageMetadataSynchronizeParams; +import org.opencb.opencga.core.models.study.VariantSetupResult; +import org.opencb.opencga.core.models.variant.VariantSetupParams; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.LongTests; import org.opencb.opencga.core.tools.result.ExecutionResult; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; +import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; +import org.opencb.opencga.storage.core.metadata.models.SampleMetadata; import org.opencb.opencga.storage.core.metadata.models.VariantScoreMetadata; import org.opencb.opencga.storage.core.utils.CellBaseUtils; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; @@ -70,6 +73,7 @@ import org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageTest; import org.opencb.opencga.storage.hadoop.variant.VariantHbaseTestUtils; import org.opencb.opencga.storage.hadoop.variant.adaptors.VariantHadoopDBAdaptor; +import org.slf4j.LoggerFactory; import java.nio.file.Path; import java.nio.file.Paths; @@ -78,22 +82,23 @@ import static org.hamcrest.CoreMatchers.anyOf; import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.fail; +import static org.junit.Assert.*; @RunWith(Parameterized.class) @Category(LongTests.class) public class VariantOperationsTest { + public static final String ORGANIZATION = "test"; public static final String USER = "user"; public static final String PASSWORD = TestParamConstants.PASSWORD; public static final String PROJECT = "project"; + public static final String PROJECT_FQN = ORGANIZATION + '@' + PROJECT; public static final String STUDY = "study"; + public static final String STUDY_FQN = PROJECT_FQN + ':' + STUDY; public static final String PHENOTYPE_NAME = "myPhenotype"; public static final Phenotype PHENOTYPE = new Phenotype(PHENOTYPE_NAME, PHENOTYPE_NAME, "mySource") .setStatus(Phenotype.Status.OBSERVED); - public static final String DB_NAME = "opencga_test_" + USER + "_" + PROJECT; + public static final String DB_NAME = VariantStorageManager.buildDatabaseName("opencga_test", ORGANIZATION, PROJECT); private static String father = "NA19661"; private static String mother = "NA19660"; private static String son = "NA19685"; @@ -125,7 +130,6 @@ public VariantOperationsTest(String storageEngine) { public static OpenCGATestExternalResource opencga = new OpenCGATestExternalResource(); public static HadoopVariantStorageTest.HadoopExternalResource hadoopExternalResource; - @ClassRule public static VariantSolrExternalResource solrExternalResource = new VariantSolrExternalResource(); private static String storageEngine; @@ -148,7 +152,7 @@ public void setUp() throws Throwable { // catalogManager = opencga.getCatalogManager(); // variantStorageManager = new VariantStorageManager(catalogManager, opencga.getStorageEngineFactory()); // toolRunner = new ToolRunner(opencga.getOpencgaHome().toString(), catalogManager, StorageEngineFactory.get(variantStorageManager.getStorageConfiguration())); - token = catalogManager.getUserManager().login("user", PASSWORD).getToken(); + token = catalogManager.getUserManager().login(ORGANIZATION, "user", PASSWORD).getToken(); } @After @@ -160,8 +164,8 @@ public void tearDown() { if (storageEngine.equals(HadoopVariantStorageEngine.STORAGE_ENGINE_ID)) { VariantHbaseTestUtils.printVariants(((VariantHadoopDBAdaptor) engine.getDBAdaptor()), Paths.get(opencga.createTmpOutdir("_hbase_print_variants_AFTER")).toUri()); } - } catch (Exception ignore) { - ignore.printStackTrace(); + } catch (Exception e) { + LoggerFactory.getLogger(getClass()).error("Ignoring exception printing variants", e); } hadoopExternalResource.after(); @@ -169,6 +173,13 @@ public void tearDown() { } } + @BeforeClass + public static void beforeClass() throws Exception { + if (HadoopVariantStorageTest.HadoopSolrSupport.isSolrTestingAvailable()) { + solrExternalResource.before(); + } + } + @AfterClass public static void afterClass() { opencga.after(); @@ -176,6 +187,9 @@ public static void afterClass() { hadoopExternalResource.after(); hadoopExternalResource = null; } + if (HadoopVariantStorageTest.HadoopSolrSupport.isSolrTestingAvailable()) { + solrExternalResource.after(); + } } private void loadDataset() throws Throwable { @@ -193,7 +207,11 @@ private void loadDataset() throws Throwable { } catalogManager = opencga.getCatalogManager(); - variantStorageManager = opencga.getVariantStorageManager(solrExternalResource); + if (HadoopVariantStorageTest.HadoopSolrSupport.isSolrTestingAvailable()) { + variantStorageManager = opencga.getVariantStorageManager(solrExternalResource); + } else { + variantStorageManager = opencga.getVariantStorageManager(); + } toolRunner = new ToolRunner(opencga.getOpencgaHome().toString(), catalogManager, variantStorageManager); opencga.clearStorageDB(DB_NAME); @@ -209,9 +227,12 @@ private void loadDataset() throws Throwable { } setUpCatalogManager(); + if (HadoopVariantStorageTest.HadoopSolrSupport.isSolrTestingAvailable()) { solrExternalResource.configure(variantStorageManager.getVariantStorageEngine(STUDY, token)); - solrExternalResource.configure(variantStorageManager.getVariantStorageEngineForStudyOperation(STUDY, new ObjectMap(), token)); + solrExternalResource.configure(variantStorageManager.getVariantStorageEngineForStudyOperation(STUDY, new ObjectMap(), token)); + } + dummyVariantSetup(variantStorageManager, STUDY, token); file = opencga.createFile(STUDY, "variant-test-file.vcf.gz", token); // variantStorageManager.index(STUDY, file.getId(), opencga.createTmpOutdir("_index"), new ObjectMap(VariantStorageOptions.ANNOTATE.key(), true), token); @@ -220,10 +241,10 @@ private void loadDataset() throws Throwable { .setFile(file.getId()) .setAnnotate(false) .setLoadHomRef(YesNoAuto.YES.name()), - Paths.get(opencga.createTmpOutdir("_index")), "index", token); + Paths.get(opencga.createTmpOutdir("_index")), "index", false, token); toolRunner.execute(VariantAnnotationIndexOperationTool.class, STUDY, new VariantAnnotationIndexParams(), - Paths.get(opencga.createTmpOutdir("_annotation-index")), "index", token); + Paths.get(opencga.createTmpOutdir("_annotation-index")), "index", false, token); for (int i = 0; i < file.getSampleIds().size(); i++) { if (i % 2 == 0) { @@ -274,9 +295,23 @@ private void loadDataset() throws Throwable { } } + public static void dummyVariantSetup(VariantStorageManager variantStorageManager, String study, String token) + throws CatalogException, StorageEngineException { + variantStorageManager.variantSetup(study, new VariantSetupParams() + .setAverageFileSize("100B") + .setExpectedFiles(5) + .setExpectedSamples(5) + .setVariantsPerSample(1000), token); + } + public void setUpCatalogManager() throws Exception { - catalogManager.getUserManager().create(USER, "User Name", "mail@ebi.ac.uk", PASSWORD, "", null, Account.AccountType.FULL, opencga.getAdminToken()); - token = catalogManager.getUserManager().login("user", PASSWORD).getToken(); + catalogManager.getOrganizationManager().create(new OrganizationCreateParams().setId(ORGANIZATION), QueryOptions.empty(), + opencga.getAdminToken()); + catalogManager.getUserManager().create(USER, "User Name", "mail@ebi.ac.uk", PASSWORD, ORGANIZATION, null, opencga.getAdminToken()); + catalogManager.getOrganizationManager().update(ORGANIZATION, new OrganizationUpdateParams().setAdmins(Collections.singletonList("user")), + null, + opencga.getAdminToken()); + token = catalogManager.getUserManager().login(ORGANIZATION, "user", PASSWORD).getToken(); String projectId = catalogManager.getProjectManager().create(PROJECT, "Project about some genomes", "", "Homo sapiens", null, "GRCh38", new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true), token).first().getId(); @@ -293,6 +328,84 @@ public void setUpCatalogManager() throws Exception { } + @Test + public void testSetup() throws Exception { + String study2 = "study2"; + String study2fqn = catalogManager.getStudyManager() + .create(PROJECT, study2, null, "Phase 1", "Done", null, null, null, null, null, token) + .first().getFqn(); + File file = opencga.createFile(study2, "variant-test-file.vcf.gz", token); + + try { + toolRunner.execute(VariantIndexOperationTool.class, study2, + new VariantIndexParams() + .setFile(file.getId()) + .setAnnotate(false) + .setLoadHomRef(YesNoAuto.YES.name()), + Paths.get(opencga.createTmpOutdir("_index")), "index", false, token); + fail("Should have thrown an exception"); + } catch (ToolException e) { + MatcherAssert.assertThat(e.getCause().getMessage(), CoreMatchers.containsString("The variant storage has not been setup for study")); + } + + try { + VariantSetupParams setupParams = new VariantSetupParams() + .setFileType(VariantSetupParams.FileType.GENOME_VCF) + .setDataDistribution(VariantSetupParams.DataDistribution.MULTIPLE_SAMPLES_PER_FILE) + .setExpectedFiles(20) + .setExpectedSamples(100) + .setNormalizeExtensions(Arrays.asList("VS", "SV")); + variantStorageManager.variantSetup(study2, setupParams, token); + fail("should have failed"); + } catch (Exception e) { + System.err.println(e.getMessage()); + MatcherAssert.assertThat(e.getMessage(), CoreMatchers.containsString("Unsupported normalize extensions")); + } + + try { + VariantSetupParams setupParams = new VariantSetupParams() + .setFileType(VariantSetupParams.FileType.GENOME_VCF) + .setDataDistribution(VariantSetupParams.DataDistribution.MULTIPLE_SAMPLES_PER_FILE) + .setExpectedSamples(100) + .setNormalizeExtensions(Arrays.asList("VS", "SV")); + variantStorageManager.variantSetup(study2, setupParams, token); + fail("should have failed"); + } catch (Exception e) { + System.err.println(e.getMessage()); + MatcherAssert.assertThat(e.getMessage(), CoreMatchers.containsString("Missing expectedFiles")); + } + + VariantSetupParams setupParams = new VariantSetupParams() + .setFileType(VariantSetupParams.FileType.GENOME_VCF) + .setDataDistribution(VariantSetupParams.DataDistribution.MULTIPLE_FILES_PER_SAMPLE) + .setExpectedFiles(20) + .setAverageSamplesPerFile(2.5f) + .setExpectedSamples(10) + .setNormalizeExtensions(Arrays.asList("SV", "VAF")); + VariantSetupResult result = variantStorageManager.variantSetup(study2, setupParams, token); + assertEquals(VariantSetupResult.Status.READY, result.getStatus()); + + toolRunner.execute(VariantIndexOperationTool.class, study2, + new VariantIndexParams() + .setFile(file.getId()) + .setLoadHomRef(YesNoAuto.YES.name()), + Paths.get(opencga.createTmpOutdir("_index")), "index", false, token); + + VariantStorageMetadataManager metadataManager = opencga.getVariantStorageEngineByProject(PROJECT_FQN).getMetadataManager(); + int studyId = metadataManager.getStudyId(study2fqn); + int sampleId = metadataManager.getSampleId(studyId, "NA19600"); + SampleMetadata sampleMetadata = metadataManager.getSampleMetadata(studyId, sampleId); + assertEquals(VariantStorageEngine.SplitData.MULTI, sampleMetadata.getSplitData()); + + try { + variantStorageManager.variantSetup(STUDY, setupParams, token); + fail("Should fail"); + } catch (Exception e) { + MatcherAssert.assertThat(e.getMessage(), CoreMatchers.containsString("Unable to execute variant-setup on study")); + MatcherAssert.assertThat(e.getMessage(), CoreMatchers.containsString("It already has indexed files.")); + } + } + @Test public void testVariantFileReload() throws Exception { try { @@ -300,7 +413,7 @@ public void testVariantFileReload() throws Exception { new VariantIndexParams() .setForceReload(false) .setFile(file.getId()), - Paths.get(opencga.createTmpOutdir()), "index_reload", token); + Paths.get(opencga.createTmpOutdir()), "index_reload", false, token); fail("Should have thrown an exception"); } catch (ToolException e) { assertEquals(StorageEngineException.class, e.getCause().getClass()); @@ -311,13 +424,13 @@ public void testVariantFileReload() throws Exception { new VariantIndexParams() .setForceReload(true) .setFile(file.getId()), - Paths.get(opencga.createTmpOutdir()), "index_reload", token); + Paths.get(opencga.createTmpOutdir()), "index_reload", false, token); } @Test public void testVariantSecondaryAnnotationIndex() throws Exception { - + Assume.assumeTrue(HadoopVariantStorageTest.HadoopSolrSupport.isSolrTestingAvailable()); for (String sample : samples) { SampleInternalVariantSecondaryAnnotationIndex index = catalogManager.getSampleManager().get(STUDY, sample, new QueryOptions(), token).first().getInternal().getVariant().getSecondaryAnnotationIndex(); assertEquals(IndexStatus.NONE, index.getStatus().getId()); @@ -326,7 +439,7 @@ public void testVariantSecondaryAnnotationIndex() throws Exception { toolRunner.execute(VariantSecondaryAnnotationIndexOperationTool.class, STUDY, new VariantSecondaryAnnotationIndexParams(), - Paths.get(opencga.createTmpOutdir()), "annotation_index", token); + Paths.get(opencga.createTmpOutdir()), "annotation_index", false, token); for (String sample : samples) { SampleInternalVariantSecondaryAnnotationIndex index = catalogManager.getSampleManager().get(STUDY, sample, new QueryOptions(), token).first().getInternal().getVariant().getSecondaryAnnotationIndex(); @@ -354,7 +467,7 @@ public void testVariantSecondarySampleIndex() throws Exception { new VariantSecondarySampleIndexParams() .setFamilyIndex(true) .setSample(Arrays.asList(mother)), - Paths.get(opencga.createTmpOutdir()), "index", token); + Paths.get(opencga.createTmpOutdir()), "index", false, token); fail("Expected to fail"); } catch (ToolException e) { assertEquals("Exception from step 'familyIndex'", e.getMessage()); @@ -366,7 +479,7 @@ public void testVariantSecondarySampleIndex() throws Exception { new VariantSecondarySampleIndexParams() .setFamilyIndex(true) .setSample(Arrays.asList(ParamConstants.ALL)), - Paths.get(opencga.createTmpOutdir()), "index", token); + Paths.get(opencga.createTmpOutdir()), "index", false, token); assertEquals(0, result.getEvents().size()); for (String sample : samples) { @@ -384,8 +497,8 @@ public void testVariantSecondarySampleIndex() throws Exception { // Initially nothing should change, even after running a manual synchronization toolRunner.execute(VariantStorageMetadataSynchronizeOperationTool.class, - new VariantStorageMetadataSynchronizeParams().setStudy(STUDY), - Paths.get(opencga.createTmpOutdir()), "", catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken()); + new VariantStorageMetadataSynchronizeParams().setStudy(STUDY_FQN), + Paths.get(opencga.createTmpOutdir()), "", false, catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken()); for (String sample : samples) { SampleInternalVariantSecondarySampleIndex sampleIndex = catalogManager.getSampleManager().get(STUDY, sample, new QueryOptions(), token) @@ -403,7 +516,8 @@ public void testVariantSecondarySampleIndex() throws Exception { // Everything should look the same, but with newer version for (String sample : samples) { - SampleInternalVariantSecondarySampleIndex sampleIndex = catalogManager.getSampleManager().get(STUDY, sample, new QueryOptions(), token).first().getInternal().getVariant().getSecondarySampleIndex(); + SampleInternalVariantSecondarySampleIndex sampleIndex = catalogManager.getSampleManager() + .get(STUDY, sample, new QueryOptions(), token).first().getInternal().getVariant().getSecondarySampleIndex(); assertEquals(IndexStatus.READY, sampleIndex.getStatus().getId()); if (sample.equals(daughter) || sample.equals(son)) { assertEquals(sample, IndexStatus.READY, sampleIndex.getFamilyStatus().getId()); @@ -456,7 +570,7 @@ public void testVariantSecondarySampleIndexPartialFamily() throws Exception { new VariantSecondarySampleIndexParams() .setFamilyIndex(true) .setSample(Arrays.asList(daughter)), - Paths.get(opencga.createTmpOutdir()), "index", token); + Paths.get(opencga.createTmpOutdir()), "index", false, token); for (String sample : samples) { SampleInternalVariantSecondarySampleIndex sampleIndex = catalogManager.getSampleManager().get(STUDY, sample, new QueryOptions(), token).first().getInternal().getVariant().getSecondarySampleIndex(); @@ -479,7 +593,7 @@ public void testGwasIndex() throws Exception { GwasAnalysis analysis = new GwasAnalysis(); Path outDir = Paths.get(opencga.createTmpOutdir("_gwas_index")); System.out.println("output = " + outDir.toAbsolutePath()); - analysis.setUp(opencga.getOpencgaHome().toString(), catalogManager, variantStorageManager, executorParams, outDir, "", token); + analysis.setUp(opencga.getOpencgaHome().toString(), catalogManager, variantStorageManager, executorParams, outDir, "", false, token); List samples = catalogManager.getSampleManager().get(STUDY, file.getSampleIds().subList(0, 2), QueryOptions.empty(), token).getResults(); catalogManager.getCohortManager().create(STUDY, new Cohort().setId("CASE").setSamples(samples), new QueryOptions(), token); diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManagerTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManagerTest.java index 0a5ac63293d..4aeedde871f 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManagerTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManagerTest.java @@ -17,12 +17,15 @@ package org.opencb.opencga.analysis.variant.manager; import org.apache.commons.lang3.RandomStringUtils; +import org.hamcrest.CoreMatchers; +import org.hamcrest.MatcherAssert; import org.junit.Test; import org.junit.experimental.categories.Category; import org.opencb.biodata.models.variant.metadata.Aggregation; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.analysis.variant.manager.operations.AbstractVariantOperationManagerTest; +import org.opencb.opencga.analysis.variant.stats.VariantStatsAnalysis; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.core.config.storage.SampleIndexConfiguration; import org.opencb.opencga.core.models.file.File; @@ -64,6 +67,18 @@ public void testConfigure() throws CatalogException, StorageEngineException { variantManager.configureStudy(studyFqn, expectedStudyConfiguration1, sessionId); variantManager.configureStudy(studyId2, expectedStudyConfiguration2, sessionId); + + try { + Study study = catalogManager.getStudyManager().create(projectId, "s_no_setup", "s_no_setup", "s_no_setup", + "Study 1", null, null, null, Collections.singletonMap(VariantStatsAnalysis.STATS_AGGREGATION_CATALOG, getAggregation()), null, sessionId) + .first(); + // Variant setup mandatory for configuring study + variantManager.configureStudy(study.getFqn(), expectedStudyConfiguration1, sessionId); + fail("Expect exception. Study not setup"); + } catch (Exception e) { + MatcherAssert.assertThat(e.getMessage(), CoreMatchers.containsString("The variant storage has not been setup for study")); + } + ObjectMap configuration = variantManager.getDataStoreByProjectId(projectId, sessionId).getOptions(); assertEquals(expectedConfiguration, configuration); diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/AbstractVariantOperationManagerTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/AbstractVariantOperationManagerTest.java index b9f597c9b3d..73838816fb3 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/AbstractVariantOperationManagerTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/AbstractVariantOperationManagerTest.java @@ -25,7 +25,9 @@ import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.test.GenericTest; +import org.opencb.opencga.TestParamConstants; import org.opencb.opencga.analysis.variant.OpenCGATestExternalResource; +import org.opencb.opencga.analysis.variant.manager.VariantOperationsTest; import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; import org.opencb.opencga.analysis.variant.operations.OperationTool; import org.opencb.opencga.analysis.variant.stats.VariantStatsAnalysis; @@ -43,10 +45,10 @@ import org.opencb.opencga.core.models.file.FileLinkParams; import org.opencb.opencga.core.models.file.FileRelatedFile; import org.opencb.opencga.core.models.file.VariantIndexStatus; +import org.opencb.opencga.core.models.organizations.OrganizationCreateParams; +import org.opencb.opencga.core.models.organizations.OrganizationUpdateParams; import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.models.user.Account; -import org.opencb.opencga.core.models.user.User; import org.opencb.opencga.core.tools.result.ExecutionResultManager; import org.opencb.opencga.storage.core.StorageEngineFactory; import org.opencb.opencga.storage.core.StoragePipelineResult; @@ -86,16 +88,16 @@ */ public abstract class AbstractVariantOperationManagerTest extends GenericTest { - private String JOB_STATUS_FILE = "status.json"; - private String OUT_LOG_EXTENSION = ".out"; + private String OUT_LOG_EXTENSION = ".log"; private String ERR_LOG_EXTENSION = ".err"; protected CatalogManager catalogManager; protected String sessionId; - protected final String userId = "user"; + protected final String USER = "user"; + protected String ORGANIZATION = "test"; protected String projectId; protected String studyId; protected String studyFqn; @@ -133,6 +135,15 @@ public abstract class AbstractVariantOperationManagerTest extends GenericTest { @Before public final void setUpAbstract() throws Exception { catalogManager = opencga.getCatalogManager(); + + catalogManager.getOrganizationManager().create(new OrganizationCreateParams().setId(ORGANIZATION), QueryOptions.empty(), + opencga.getAdminToken()); + catalogManager.getUserManager().create(USER, USER, "my@email.org", TestParamConstants.PASSWORD, ORGANIZATION, 1000L, opencga.getAdminToken()); + catalogManager.getOrganizationManager().update(ORGANIZATION, new OrganizationUpdateParams().setAdmins(Collections.singletonList(USER)), + null, + opencga.getAdminToken()); + sessionId = catalogManager.getUserManager().login(ORGANIZATION, USER, TestParamConstants.PASSWORD).getToken(); + DummyVariantStorageEngine.configure(opencga.getStorageEngineFactory(), true); variantManager = opencga.getVariantStorageManager(); @@ -142,8 +153,6 @@ public final void setUpAbstract() throws Exception { // Policies policies = new Policies(); // policies.setUserCreation(Policies.UserCreation.ALWAYS); - User user = catalogManager.getUserManager().create(userId, "User", "user@email.org", "userACME1.", "ACME", null, Account.AccountType.FULL, opencga.getAdminToken()).first(); - sessionId = catalogManager.getUserManager().login(userId, "userACME1.").getToken(); projectId = "p1"; catalogManager.getProjectManager().create(projectId, projectId, "Project 1", "Homo sapiens", null, "GRCh38", new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true), sessionId); @@ -161,6 +170,10 @@ public final void setUpAbstract() throws Exception { true, null, QueryOptions.empty(), sessionId).first().getId(); files = Arrays.asList(new File[5]); + + + VariantOperationsTest.dummyVariantSetup(variantManager, studyFqn, sessionId); + VariantOperationsTest.dummyVariantSetup(variantManager, studyId2, sessionId); } @After @@ -189,7 +202,11 @@ protected File getSmallFile() throws IOException, CatalogException { } protected File create(String resourceName) throws IOException, CatalogException { - return create(studyId, getResourceUri(resourceName)); + return create(resourceName, "data/vcfs/"); + } + + protected File create(String resourceName, String path) throws IOException, CatalogException { + return create(studyId, getResourceUri(resourceName), path); } protected File create(String studyId, URI uri) throws IOException, CatalogException { @@ -225,7 +242,7 @@ protected File transformFile(File inputFile, QueryOptions queryOptions, String o queryOptions.append(OperationTool.KEEP_INTERMEDIATE_FILES, true); boolean calculateStats = queryOptions.getBoolean(VariantStorageOptions.STATS_CALCULATE.key()); - Study study = catalogManager.getFileManager().getStudy(inputFile, sessionId); + Study study = catalogManager.getFileManager().getStudy(ORGANIZATION, inputFile, sessionId); String studyId = study.getId(); //Default cohort should not be modified @@ -277,7 +294,7 @@ protected List loadFiles(List files, List exp queryOptions.append(OperationTool.KEEP_INTERMEDIATE_FILES, true); boolean calculateStats = queryOptions.getBoolean(VariantStorageOptions.STATS_CALCULATE.key()); - String studyId = catalogManager.getFileManager().getStudy(files.get(0), sessionId).getId(); + String studyId = catalogManager.getFileManager().getStudy(ORGANIZATION, files.get(0), sessionId).getId(); List fileIds = files.stream().map(File::getId).collect(Collectors.toList()); String outdir = opencga.createTmpOutdir(studyId, "_LOAD_", sessionId); @@ -311,7 +328,7 @@ protected List indexFiles(List files, List ex queryOptions.append(VariantFileIndexerOperationManager.LOAD, true); boolean calculateStats = queryOptions.getBoolean(VariantStorageOptions.STATS_CALCULATE.key()); - String studyId = catalogManager.getFileManager().getStudy(files.get(0), sessionId).getId(); + String studyId = catalogManager.getFileManager().getStudy(ORGANIZATION, files.get(0), sessionId).getId(); // queryOptions.append(StorageOperation.CATALOG_PATH, outputId); queryOptions.append(OperationTool.KEEP_INTERMEDIATE_FILES, true); @@ -320,10 +337,26 @@ protected List indexFiles(List files, List ex List fileIds = files.stream().map(File::getPath).collect(Collectors.toList()); List etlResults; + Exception etlException = null; try { etlResults = variantManager.index(studyId, fileIds, tmpOutdir, queryOptions, sessionId); + } catch (Exception e) { + etlResults = null; + etlException = e; } finally { - copyResults(Paths.get(tmpOutdir), studyId, outputId, sessionId); + try { + copyResults(Paths.get(tmpOutdir), studyId, outputId, sessionId); + } catch (Exception e) { + logger.error("Error copying results", e); + if (etlException != null) { + etlException.addSuppressed(e); + } else { + throw e; + } + } + if (etlException != null) { + throw etlException; + } } assertEquals(expectedLoadedFiles.size(), etlResults.size()); @@ -378,18 +411,17 @@ protected List copyResults(Path tmpOutdirPath, String study, String catalo try { logger.info("Scanning files from {} to move to {}", tmpOutdirPath, outDir.getUri()); // Avoid copy the job.status file! - Predicate fileStatusFilter = uri -> !uri.getPath().endsWith(JOB_STATUS_FILE) - && !ExecutionResultManager.isExecutionResultFile(uri.getPath()) + Predicate fileStatusFilter = uri -> !ExecutionResultManager.isExecutionResultFile(uri.getPath()) && !uri.getPath().endsWith(OUT_LOG_EXTENSION) && !uri.getPath().endsWith(ERR_LOG_EXTENSION); - files = fileScanner.scan(outDir, tmpOutdirPath.toUri(), FileScanner.FileScannerPolicy.DELETE, false, true, fileStatusFilter, + files = fileScanner.scan(ORGANIZATION, outDir, tmpOutdirPath.toUri(), FileScanner.FileScannerPolicy.DELETE, false, true, fileStatusFilter, sessionId); System.out.println("files = " + files); // TODO: Check whether we want to store the logs as well. At this point, we are also storing them. // Do not execute checksum for log files! They may not be closed yet fileStatusFilter = uri -> uri.getPath().endsWith(OUT_LOG_EXTENSION) || uri.getPath().endsWith(ERR_LOG_EXTENSION); - files.addAll(fileScanner.scan(outDir, tmpOutdirPath.toUri(), FileScanner.FileScannerPolicy.DELETE, false, false, + files.addAll(fileScanner.scan(ORGANIZATION, outDir, tmpOutdirPath.toUri(), FileScanner.FileScannerPolicy.DELETE, false, false, fileStatusFilter, sessionId)); System.out.println("files2 = " + files); @@ -425,7 +457,7 @@ protected void checkEtlResults(String studyId, List etlRe protected DummyVariantStorageEngine mockVariantStorageManager() { DummyVariantStorageEngine vsm = spy(new DummyVariantStorageEngine()); - String dbName = VariantStorageManager.buildDatabaseName(catalogManager.getConfiguration().getDatabasePrefix(), userId, projectId); + String dbName = VariantStorageManager.buildDatabaseName(catalogManager.getConfiguration().getDatabasePrefix(), ORGANIZATION, projectId); vsm.setConfiguration(opencga.getStorageConfiguration(), DummyVariantStorageEngine.STORAGE_ENGINE_ID, dbName); StorageEngineFactory.get(opencga.getStorageConfiguration()).registerVariantStorageEngine(vsm, dbName, null); StorageEngineFactory.get(opencga.getStorageConfiguration()).registerVariantStorageEngine(vsm, dbName, studyFqn); diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/PlatinumFileIndexerTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/PlatinumFileIndexerTest.java index f62d332470b..c343ceff241 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/PlatinumFileIndexerTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/PlatinumFileIndexerTest.java @@ -105,7 +105,7 @@ public void testIndexFamily() throws Exception { params.setFamily(true); ExecutionResult er = toolRunner.execute(VariantIndexOperationTool.class, params.toObjectMap() - .append(ParamConstants.STUDY_PARAM, studyId), outDir, null, sessionId); + .append(ParamConstants.STUDY_PARAM, studyId), outDir, null, false, sessionId); assertEquals(1, er.getSteps().size()); assertEquals("variant-index", er.getSteps().get(0).getId()); @@ -212,7 +212,7 @@ public void testLauncher() throws CatalogException, IOException, ToolException { VariantFileIndexJobLauncherParams params = new VariantFileIndexJobLauncherParams().setDirectory("data/vcfs"); List tags = Arrays.asList("tag1", "tag2"); Job job = catalogManager.getJobManager().submit(studyFqn, VariantFileIndexJobLauncherTool.ID, Enums.Priority.HIGH, - params.toParams(STUDY_PARAM, studyFqn), null, null, null, tags, sessionId).first(); + params.toParams(STUDY_PARAM, studyFqn), null, null, null, tags, null, null, false, sessionId).first(); ExecutionResult result = toolRunner.execute(job, Paths.get(opencga.createTmpOutdir(studyId, "_LOAD_", sessionId)), sessionId); List tagsFromResult = result.getAttributes().getAsStringList(VariantFileIndexJobLauncherTool.JOB_TAGS_ATTRIBUTE); @@ -231,7 +231,7 @@ public void testLauncher() throws CatalogException, IOException, ToolException { //// Execute again, no new jobs should be submitted tags = Arrays.asList("tag10", "tag20"); job = catalogManager.getJobManager().submit(studyFqn, VariantFileIndexJobLauncherTool.ID, Enums.Priority.HIGH, - params.toParams(STUDY_PARAM, studyFqn), null, null, null, tags, sessionId).first(); + params.toParams(STUDY_PARAM, studyFqn), null, null, null, tags, null, null, false, sessionId).first(); result = toolRunner.execute(job, Paths.get(opencga.createTmpOutdir(studyId, "_LOAD_", sessionId)), sessionId); tagsFromResult = result.getAttributes().getAsStringList(VariantFileIndexJobLauncherTool.JOB_TAGS_ATTRIBUTE); diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/RemoveVariantsTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/RemoveVariantsTest.java index 0473d55a031..e5e190328b5 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/RemoveVariantsTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/RemoveVariantsTest.java @@ -32,7 +32,9 @@ import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.study.Study; import org.opencb.opencga.core.testclassification.duration.MediumTests; +import org.opencb.opencga.storage.core.variant.VariantStorageOptions; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -44,6 +46,7 @@ import static org.hamcrest.CoreMatchers.anyOf; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.*; +import static org.opencb.opencga.storage.core.variant.VariantStorageBaseTest.getResourceUri; /** * Created on 10/07/17. @@ -83,6 +86,42 @@ public void testLoadAndRemoveOneWithOtherLoaded() throws Exception { testLoadAndRemoveOne(); } + @Test + public void testLoadAndRemoveForce() throws Exception { + File file77 = create("platinum/1K.end.platinum-genomes-vcf-NA12877_S1.genome.vcf.gz"); + File file78 = create("platinum/1K.end.platinum-genomes-vcf-NA12878_S1.genome.vcf.gz"); + File file79 = create("platinum/1K.end.platinum-genomes-vcf-NA12879_S1.genome.vcf.gz"); + indexFile(file77, new QueryOptions(), outputId); + indexFile(file78, new QueryOptions(), outputId); + + removeFile(file77, new QueryOptions()); + + try { + removeFile(file77, new QueryOptions()); + } catch (Exception e) { + assertTrue(e.getMessage(), e.getMessage().contains("Unable to remove variants from file")); + } + removeFile(file77, new QueryOptions(VariantStorageOptions.FORCE.key(), true)); + + try { + removeFile(file79, new QueryOptions(VariantStorageOptions.FORCE.key(), true)); + } catch (Exception e) { + assertTrue(e.getMessage(), e.getMessage().contains("File not found in storage.")); + } + Path file77Path = Paths.get(file77.getUri()); + Path otherDir = file77Path.getParent().resolve("other_dir"); + Files.createDirectory(otherDir); + Path otherFile = Files.copy(file77Path, otherDir.resolve(file77Path.getFileName())); + File file77_2 = create(studyFqn, otherFile.toUri(), "other_dir"); + + try { + removeFile(studyFqn, Collections.singletonList(file77_2.getPath()), new QueryOptions(VariantStorageOptions.FORCE.key(), true)); + } catch (Exception e) { + assertTrue(e.getMessage(), e.getMessage().contains("Unable to remove variants from file")); + assertTrue(e.getMessage(), e.getMessage().contains("Instead, found file with same name but different path")); + } + } + @Test public void testLoadAndRemoveMany() throws Exception { @@ -124,11 +163,16 @@ private void removeFile(File file, QueryOptions options) throws Exception { private void removeFile(List files, QueryOptions options) throws Exception { List fileIds = files.stream().map(File::getId).collect(Collectors.toList()); - Study study = catalogManager.getFileManager().getStudy(files.get(0), sessionId); + Study study = catalogManager.getFileManager().getStudy(ORGANIZATION, files.get(0), sessionId); String studyId = study.getFqn(); + removeFile(studyId, fileIds, options); + } + + private void removeFile(String studyId, List fileIds, QueryOptions options) throws Exception { + Path outdir = Paths.get(opencga.createTmpOutdir(studyId, "_REMOVE_", sessionId)); - variantManager.removeFile(studyId, fileIds, new QueryOptions(), outdir.toUri(), sessionId); + variantManager.removeFile(studyId, fileIds, new QueryOptions(options), outdir.toUri(), sessionId); // assertEquals(files.size(), removedFiles.size()); Cohort all = catalogManager.getCohortManager().search(studyId, new Query(CohortDBAdaptor.QueryParams.ID.key(), diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/StatsVariantStorageTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/StatsVariantStorageTest.java index 09bcacc2ff7..cfe9deeac5f 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/StatsVariantStorageTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/StatsVariantStorageTest.java @@ -212,7 +212,7 @@ public void calculateStats(VariantStatsIndexParams params, QueryOptions options) options.put(ParamConstants.STUDY_PARAM, studyId); ToolRunner toolRunner = new ToolRunner(null, catalogManager, opencga.getStorageEngineFactory()); - toolRunner.execute(VariantStatsIndexOperationTool.class, params, options, Paths.get(tmpOutdir), null, sessionId); + toolRunner.execute(VariantStatsIndexOperationTool.class, params, options, Paths.get(tmpOutdir), null, false, sessionId); } @Test diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/VariantFileIndexerOperationManagerTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/VariantFileIndexerOperationManagerTest.java index a4fe3f97944..ef373cf1cd7 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/VariantFileIndexerOperationManagerTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/VariantFileIndexerOperationManagerTest.java @@ -182,7 +182,7 @@ public void testDeleteIndexedFile() throws Exception { File inputFile = getFile(0); indexFile(inputFile, queryOptions, outputId); - Study study = catalogManager.getFileManager().getStudy(inputFile, sessionId); + Study study = catalogManager.getFileManager().getStudy(ORGANIZATION, inputFile, sessionId); thrown.expect(CatalogException.class); thrown.expectMessage("Could not unlink file '" + inputFile.getId() + "'"); @@ -319,7 +319,7 @@ public void testTransformTransformingFiles() throws Exception { .append(VariantFileIndexerOperationManager.SKIP_INDEXED_FILES, true); List files = Arrays.asList(getFile(0), getFile(1)); - catalogManager.getFileManager().updateFileInternalVariantIndex(getFile(1), + catalogManager.getFileManager().updateFileInternalVariantIndex(studyFqn, getFile(1), FileInternalVariantIndex.init().setStatus(new VariantIndexStatus(VariantIndexStatus.TRANSFORMING)), sessionId); // Expect both files to be loaded @@ -333,7 +333,7 @@ public void testResumeTransformTransformingFiles() throws Exception { .append(VariantStorageOptions.RESUME.key(), true); List files = Arrays.asList(getFile(0), getFile(1)); - catalogManager.getFileManager().updateFileInternalVariantIndex(getFile(1), new FileInternalVariantIndex(new VariantIndexStatus(VariantIndexStatus.TRANSFORMING), 0, null), sessionId); + catalogManager.getFileManager().updateFileInternalVariantIndex(studyFqn, getFile(1), new FileInternalVariantIndex(new VariantIndexStatus(VariantIndexStatus.TRANSFORMING), 0, null), sessionId); // Expect only the first file to be loaded indexFiles(files, files, queryOptions, outputId); @@ -415,7 +415,7 @@ public void testIndexByStepsExternallyTransformed() throws Exception { outdir, queryOptions, sessionId); File transformFile = null; - create(studyId2, catalogManager.getFileManager().getUri(getFile(0))); + create(studyId2, catalogManager.getFileManager().getUri(studyFqn, getFile(0))); for (java.io.File file : Paths.get(UriUtils.createUri(outdir)).toFile().listFiles()) { File f = create(studyId2, file.toURI()); if (VariantReaderUtils.isTransformedVariants(file.toString())) { @@ -443,7 +443,7 @@ public void testIndexMalformed() throws Exception { ExecutionResult er = toolRunner.execute(VariantIndexOperationTool.class, params.toObjectMap() .append(ParamConstants.STUDY_PARAM, studyId) .append(VariantStorageOptions.TRANSFORM_FAIL_ON_MALFORMED_VARIANT.key(), false) - , outDir, null, sessionId); + , outDir, null, false, sessionId); assertEquals(Event.Type.WARNING, er.getEvents().get(0).getType()); assertThat(er.getEvents().get(0).getMessage(), CoreMatchers.containsString("Found malformed variants")); @@ -461,7 +461,7 @@ public void testIndexDuplicated() throws Exception { ExecutionResult er = toolRunner.execute(VariantIndexOperationTool.class, params.toObjectMap() .append(ParamConstants.STUDY_PARAM, studyId) .append(VariantStorageOptions.TRANSFORM_FAIL_ON_MALFORMED_VARIANT.key(), false) - , outDir, null, sessionId); + , outDir, null, false, sessionId); assertEquals(Event.Type.WARNING, er.getEvents().get(0).getType()); assertThat(er.getEvents().get(0).getMessage(), CoreMatchers.containsString("Found duplicated variants")); @@ -482,7 +482,7 @@ public void testIndexWeirdFileName() throws Exception { ExecutionResult er = toolRunner.execute(VariantIndexOperationTool.class, params.toObjectMap() .append(ParamConstants.STUDY_PARAM, studyId) - , outDir, null, sessionId); + , outDir, null, false, sessionId); } @Override diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/VariantVirtualFileIndexTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/VariantVirtualFileIndexTest.java index efa7e27fad7..fcc6abed280 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/VariantVirtualFileIndexTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/VariantVirtualFileIndexTest.java @@ -72,15 +72,15 @@ public void testIndexVirtual() throws Exception { toolRunner.execute(VariantIndexOperationTool.class, new VariantIndexParams() .setFile("chr20.variant-test-file.vcf.gz") .setLoadSplitData("REGION") - , new ObjectMap(ParamConstants.STUDY_PARAM, studyId), Paths.get(opencga.createTmpOutdir()), null, sessionId); + , new ObjectMap(ParamConstants.STUDY_PARAM, studyId), Paths.get(opencga.createTmpOutdir()), null, false, sessionId); toolRunner.execute(VariantIndexOperationTool.class, new VariantIndexParams() .setFile("chr21.variant-test-file.vcf.gz") .setLoadSplitData("REGION") - , new ObjectMap(ParamConstants.STUDY_PARAM, studyId), Paths.get(opencga.createTmpOutdir()), null, sessionId); + , new ObjectMap(ParamConstants.STUDY_PARAM, studyId), Paths.get(opencga.createTmpOutdir()), null, false, sessionId); toolRunner.execute(VariantIndexOperationTool.class, new VariantIndexParams() .setFile("chr22.variant-test-file.vcf.gz") .setLoadSplitData("REGION") - , new ObjectMap(ParamConstants.STUDY_PARAM, studyId), Paths.get(opencga.createTmpOutdir()), null, sessionId); + , new ObjectMap(ParamConstants.STUDY_PARAM, studyId), Paths.get(opencga.createTmpOutdir()), null, false, sessionId); File file = catalogManager.getFileManager().get(studyId, "chr20.variant-test-file.vcf.gz", null, sessionId).first(); assertTrue(FileUtils.isPartial(file)); diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/metadata/CatalogStorageMetadataSynchronizerTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/metadata/CatalogStorageMetadataSynchronizerTest.java index fad5fb53ded..8833884c5a4 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/metadata/CatalogStorageMetadataSynchronizerTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/metadata/CatalogStorageMetadataSynchronizerTest.java @@ -35,10 +35,11 @@ import org.opencb.opencga.core.models.common.IndexStatus; import org.opencb.opencga.core.models.common.InternalStatus; import org.opencb.opencga.core.models.file.*; +import org.opencb.opencga.core.models.organizations.OrganizationCreateParams; +import org.opencb.opencga.core.models.organizations.OrganizationUpdateParams; import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.sample.SampleReferenceParam; import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.models.user.Account; import org.opencb.opencga.core.testclassification.duration.MediumTests; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; @@ -72,6 +73,7 @@ public class CatalogStorageMetadataSynchronizerTest { private static final QueryOptions INCLUDE_RESULT = new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true); static private CatalogManager catalogManager; + static private final String ORGANIZATION = "test"; static private String sessionId; static private String projectId; static private String studyId; @@ -90,9 +92,15 @@ public static void beforeClass() throws Exception { catalogManager = catalogManagerExternalResource.getCatalogManager(); - catalogManager.getUserManager().create(userId, "User", "user@email.org", TestParamConstants.PASSWORD, "ACME", null, Account.AccountType.FULL, catalogManagerExternalResource.getAdminToken()).first(); + catalogManager.getOrganizationManager().create(new OrganizationCreateParams().setId(ORGANIZATION), QueryOptions.empty(), + catalogManagerExternalResource.getAdminToken()); + catalogManager.getUserManager().create("user", "user", "my@email.org", TestParamConstants.PASSWORD, ORGANIZATION, + 1000L, catalogManagerExternalResource.getAdminToken()); + catalogManager.getOrganizationManager().update(ORGANIZATION, new OrganizationUpdateParams().setAdmins(Collections.singletonList("user")), + null, + catalogManagerExternalResource.getAdminToken()); + sessionId = catalogManager.getUserManager().login(ORGANIZATION, "user", TestParamConstants.PASSWORD).getToken(); - sessionId = catalogManager.getUserManager().login(userId, TestParamConstants.PASSWORD).getToken(); projectId = catalogManager.getProjectManager().create("p1", "p1", "Project 1", "Homo sapiens", null, "GRCh38", INCLUDE_RESULT, sessionId).first().getId(); Study study = catalogManager.getStudyManager().create(projectId, "s1", null, "s1", "Study " + "1", null, null, @@ -149,7 +157,7 @@ public static File create(String resourceName, boolean indexed) throws IOExcepti .setPath("data/vcfs/"); file = catalogManager.getFileManager().link(studyId, params, true, sessionId).first(); if (indexed) { - catalogManager.getFileManager().updateFileInternalVariantIndex(file, + catalogManager.getFileManager().updateFileInternalVariantIndex(studyId, file, FileInternalVariantIndex.init().setStatus(new VariantIndexStatus(InternalStatus.READY)), sessionId); indexedFiles.add(file.getName()); List samples = catalogManager.getCohortManager().getSamples(studyId, cohortId, sessionId).getResults().stream().map(Sample::getId).collect(Collectors.toList()); diff --git a/opencga-analysis/src/test/resources/exomiser/application.properties b/opencga-analysis/src/test/resources/exomiser/13.1/application.properties similarity index 95% rename from opencga-analysis/src/test/resources/exomiser/application.properties rename to opencga-analysis/src/test/resources/exomiser/13.1/application.properties index e722c2fd89b..1775e32d14c 100644 --- a/opencga-analysis/src/test/resources/exomiser/application.properties +++ b/opencga-analysis/src/test/resources/exomiser/13.1/application.properties @@ -40,6 +40,6 @@ exomiser.data-directory=/data exomiser.hg38.data-version=2109 #exomiser.hg19.remm-path=${exomiser.data-directory}/remm/ReMM.v${remm.version}.hg19.tsv.gz #exomiser.hg19.variant-white-list-path=${exomiser.hg19.data-version}_hg19_clinvar_whitelist.tsv.gz -exomiser.hg38.variant-white-list-path=${exomiser.hg38.data-version}_hg38_clinvar_whitelist.tsv.gz +exomiser.hg38.variant-white-list-path=2109_hg38_clinvar_whitelist.tsv.gz exomiser.phenotype.data-version=2109 logging.file.name=/jobdir/exomiser.log diff --git a/opencga-app/app/analysis/exomiser/exomiser-analysis.yml b/opencga-analysis/src/test/resources/exomiser/13.1/exomiser-analysis.yml similarity index 100% rename from opencga-app/app/analysis/exomiser/exomiser-analysis.yml rename to opencga-analysis/src/test/resources/exomiser/13.1/exomiser-analysis.yml diff --git a/opencga-analysis/src/test/resources/exomiser/output.yml b/opencga-analysis/src/test/resources/exomiser/13.1/output.yml similarity index 88% rename from opencga-analysis/src/test/resources/exomiser/output.yml rename to opencga-analysis/src/test/resources/exomiser/13.1/output.yml index 6a70c778641..3203c545356 100644 --- a/opencga-analysis/src/test/resources/exomiser/output.yml +++ b/opencga-analysis/src/test/resources/exomiser/13.1/output.yml @@ -1,8 +1,8 @@ --- -#outputContributingVariantsOnly: true +outputContributingVariantsOnly: true #numGenes options: 0 = all or specify a limit e.g. 500 for the first 500 results -#numGenes: 20 -#minExomiserGeneScore: 0.7 +numGenes: 20 +minExomiserGeneScore: 0.0 #outputPrefix options: specify the path/filename without an extension and this will be added # according to the outputFormats option. If unspecified this will default to the following: # {exomiserDir}/results/input-vcf-name-exomiser-results.html diff --git a/opencga-analysis/src/test/resources/exomiser/14.0/application.properties b/opencga-analysis/src/test/resources/exomiser/14.0/application.properties new file mode 100644 index 00000000000..d1658370370 --- /dev/null +++ b/opencga-analysis/src/test/resources/exomiser/14.0/application.properties @@ -0,0 +1,47 @@ +# +# The Exomiser - A tool to annotate and prioritize genomic variants +# +# Copyright (c) 2016-2021 Queen Mary University of London. +# Copyright (c) 2012-2016 Charité Universitätsmedizin Berlin and Genome Research Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# + +## exomiser root data directory ## +# root path where data is to be downloaded and worked on it is assumed that all the files required by exomiser listed +# in this properties file will be found in the data directory, unless specifically overridden here. +#exomiser.data-directory=/exomiser-data +exomiser.data-directory=/data + +#remm.version=0.3.1.post1 +#cadd.version=1.4 +#exomiser.hg38.data-version=1811 +#exomiser.hg38.data-version=2109 +#exomiser.hg38.remm-path=${exomiser.data-directory}/remm/ReMM.v${remm.version}.hg38.tsv.gz +#exomiser.hg38.variant-white-list-path=${exomiser.hg38.data-version}_hg38_clinvar_whitelist.tsv.gz +#exomiser.phenotype.data-version=2109 +#logging.file.name=/exomiser-data/logs/exomiser.log + + +#remm.version=0.3.1.post1 +#cadd.version=1.4 +#exomiser.hg19.data-version=1811 +exomiser.hg38.data-version=2402 +#exomiser.hg19.remm-path=${exomiser.data-directory}/remm/ReMM.v${remm.version}.hg19.tsv.gz +#exomiser.hg19.variant-white-list-path=${exomiser.hg19.data-version}_hg19_clinvar_whitelist.tsv.gz +#exomiser.hg38.variant-white-list-path=${exomiser.hg38.data-version}_hg38_clinvar_whitelist.tsv.gz +exomiser.phenotype.data-version=2402 +exomiser.hg38.clin-var-data-version=2402 +exomiser.hg38.use-clinvar-white-list=true +logging.file.name=/jobdir/exomiser.log diff --git a/opencga-analysis/src/test/resources/exomiser/14.0/exomiser-analysis.yml b/opencga-analysis/src/test/resources/exomiser/14.0/exomiser-analysis.yml new file mode 100644 index 00000000000..e3d064ba07b --- /dev/null +++ b/opencga-analysis/src/test/resources/exomiser/14.0/exomiser-analysis.yml @@ -0,0 +1,85 @@ +--- +analysisMode: PASS_ONLY +inheritanceModes: { + AUTOSOMAL_DOMINANT: 0.1, + AUTOSOMAL_RECESSIVE_COMP_HET: 2.0, + AUTOSOMAL_RECESSIVE_HOM_ALT: 0.1, + X_DOMINANT: 0.1, + X_RECESSIVE_COMP_HET: 2.0, + X_RECESSIVE_HOM_ALT: 0.1, + MITOCHONDRIAL: 0.2 +} +frequencySources: [ + UK10K, + + GNOMAD_E_AFR, + GNOMAD_E_AMR, + # GNOMAD_E_ASJ, + GNOMAD_E_EAS, + # GNOMAD_E_FIN, + GNOMAD_E_NFE, + # GNOMAD_E_OTH, + GNOMAD_E_SAS, + + GNOMAD_G_AFR, + GNOMAD_G_AMR, + # GNOMAD_G_ASJ, + GNOMAD_G_EAS, + # GNOMAD_G_FIN, + GNOMAD_G_NFE, + # GNOMAD_G_OTH, + GNOMAD_G_SAS +] +# Possible pathogenicitySources: (POLYPHEN, MUTATION_TASTER, SIFT), (REVEL, MVP), CADD, REMM +# REMM is trained on non-coding regulatory regions +# *WARNING* if you enable CADD or REMM ensure that you have downloaded and installed the CADD/REMM tabix files +# and updated their location in the application.properties. Exomiser will not run without this. +pathogenicitySources: [ REVEL, MVP ] +#this is the standard exomiser order. +#all steps are optional +steps: [ + #hiPhivePrioritiser: {}, + #priorityScoreFilter: {priorityType: HIPHIVE_PRIORITY, minPriorityScore: 0.500}, + #intervalFilter: {interval: 'chr10:123256200-123256300'}, + # or for multiple intervals: + #intervalFilter: {intervals: ['chr10:123256200-123256300', 'chr10:123256290-123256350']}, + # or using a BED file - NOTE this should be 0-based, Exomiser otherwise uses 1-based coordinates in line with VCF + #intervalFilter: {bed: /full/path/to/bed_file.bed}, + #genePanelFilter: {geneSymbols: ['FGFR1','FGFR2']}, + #geneBlacklistFilter: { }, + failedVariantFilter: { }, + #qualityFilter: {minQuality: 50.0}, + variantEffectFilter: { + remove: [ + FIVE_PRIME_UTR_EXON_VARIANT, + FIVE_PRIME_UTR_INTRON_VARIANT, + THREE_PRIME_UTR_EXON_VARIANT, + THREE_PRIME_UTR_INTRON_VARIANT, + NON_CODING_TRANSCRIPT_EXON_VARIANT, + UPSTREAM_GENE_VARIANT, + INTERGENIC_VARIANT, + REGULATORY_REGION_VARIANT, + CODING_TRANSCRIPT_INTRON_VARIANT, + NON_CODING_TRANSCRIPT_INTRON_VARIANT, + DOWNSTREAM_GENE_VARIANT + ] + }, + # removes variants represented in the database + #knownVariantFilter: {}, + frequencyFilter: {maxFrequency: 2.0}, + pathogenicityFilter: {keepNonPathogenic: true}, + # inheritanceFilter and omimPrioritiser should always run AFTER all other filters have completed + # they will analyse genes according to the specified modeOfInheritance above- UNDEFINED will not be analysed. + inheritanceFilter: {}, + # omimPrioritiser isn't mandatory. + omimPrioritiser: {}, + #priorityScoreFilter: {minPriorityScore: 0.4}, + # Other prioritisers: Only combine omimPrioritiser with one of these. + # Don't include any if you only want to filter the variants. + hiPhivePrioritiser: {}, + # or run hiPhive in benchmarking mode: + #hiPhivePrioritiser: {runParams: 'mouse'}, + #phivePrioritiser: {} + #phenixPrioritiser: {} + #exomeWalkerPrioritiser: {seedGeneIds: [11111, 22222, 33333]} +] diff --git a/opencga-analysis/src/test/resources/exomiser/14.0/output.yml b/opencga-analysis/src/test/resources/exomiser/14.0/output.yml new file mode 100644 index 00000000000..695255a573b --- /dev/null +++ b/opencga-analysis/src/test/resources/exomiser/14.0/output.yml @@ -0,0 +1,11 @@ +--- +outputContributingVariantsOnly: true +#numGenes options: 0 = all or specify a limit e.g. 500 for the first 500 results +numGenes: 20 +minExomiserGeneScore: 0.0 +# Path to the desired output directory. Will default to the 'results' subdirectory of the exomiser install directory +outputDirectory: /jobdir/ +# Filename for the output files. Will default to {input-vcf-filename}-exomiser +outputFileName: exomiser_output +#out-format options: HTML, JSON, TSV_GENE, TSV_VARIANT, VCF (default: HTML) +outputFormats: [TSV_VARIANT, JSON, HTML] diff --git a/opencga-app/app/analysis/exomiser/application.properties b/opencga-app/app/analysis/exomiser/13.1/application.properties similarity index 95% rename from opencga-app/app/analysis/exomiser/application.properties rename to opencga-app/app/analysis/exomiser/13.1/application.properties index e722c2fd89b..1775e32d14c 100644 --- a/opencga-app/app/analysis/exomiser/application.properties +++ b/opencga-app/app/analysis/exomiser/13.1/application.properties @@ -40,6 +40,6 @@ exomiser.data-directory=/data exomiser.hg38.data-version=2109 #exomiser.hg19.remm-path=${exomiser.data-directory}/remm/ReMM.v${remm.version}.hg19.tsv.gz #exomiser.hg19.variant-white-list-path=${exomiser.hg19.data-version}_hg19_clinvar_whitelist.tsv.gz -exomiser.hg38.variant-white-list-path=${exomiser.hg38.data-version}_hg38_clinvar_whitelist.tsv.gz +exomiser.hg38.variant-white-list-path=2109_hg38_clinvar_whitelist.tsv.gz exomiser.phenotype.data-version=2109 logging.file.name=/jobdir/exomiser.log diff --git a/opencga-app/app/analysis/exomiser/13.1/exomiser-analysis.yml b/opencga-app/app/analysis/exomiser/13.1/exomiser-analysis.yml new file mode 100644 index 00000000000..5ec7d93d508 --- /dev/null +++ b/opencga-app/app/analysis/exomiser/13.1/exomiser-analysis.yml @@ -0,0 +1,70 @@ +--- +analysisMode: PASS_ONLY +inheritanceModes: { + AUTOSOMAL_DOMINANT: 0.1, + AUTOSOMAL_RECESSIVE_HOM_ALT: 0.1, + AUTOSOMAL_RECESSIVE_COMP_HET: 2.0, + X_DOMINANT: 0.1, + X_RECESSIVE_HOM_ALT: 0.1, + X_RECESSIVE_COMP_HET: 2.0, + MITOCHONDRIAL: 0.2 +} +frequencySources: [ + THOUSAND_GENOMES, + TOPMED, + UK10K, + + ESP_AFRICAN_AMERICAN, ESP_EUROPEAN_AMERICAN, ESP_ALL, + + EXAC_AFRICAN_INC_AFRICAN_AMERICAN, EXAC_AMERICAN, + EXAC_SOUTH_ASIAN, EXAC_EAST_ASIAN, + EXAC_FINNISH, EXAC_NON_FINNISH_EUROPEAN, + EXAC_OTHER, + + GNOMAD_E_AFR, + GNOMAD_E_AMR, + # GNOMAD_E_ASJ, + GNOMAD_E_EAS, + GNOMAD_E_FIN, + GNOMAD_E_NFE, + GNOMAD_E_OTH, + GNOMAD_E_SAS, + + GNOMAD_G_AFR, + GNOMAD_G_AMR, + # GNOMAD_G_ASJ, + GNOMAD_G_EAS, + GNOMAD_G_FIN, + GNOMAD_G_NFE, + GNOMAD_G_OTH, + GNOMAD_G_SAS +] +# Possible pathogenicitySources: (POLYPHEN, MUTATION_TASTER, SIFT), (REVEL, MVP), CADD, REMM +# REMM is trained on non-coding regulatory regions +# *WARNING* if you enable CADD or REMM ensure that you have downloaded and installed the CADD/REMM tabix files +# and updated their location in the application.properties. Exomiser will not run without this. +pathogenicitySources: [ REVEL, MVP ] +#this is the standard exomiser order. +steps: [ + failedVariantFilter: { }, + variantEffectFilter: { + remove: [ + FIVE_PRIME_UTR_EXON_VARIANT, + FIVE_PRIME_UTR_INTRON_VARIANT, + THREE_PRIME_UTR_EXON_VARIANT, + THREE_PRIME_UTR_INTRON_VARIANT, + NON_CODING_TRANSCRIPT_EXON_VARIANT, + NON_CODING_TRANSCRIPT_INTRON_VARIANT, + CODING_TRANSCRIPT_INTRON_VARIANT, + UPSTREAM_GENE_VARIANT, + DOWNSTREAM_GENE_VARIANT, + INTERGENIC_VARIANT, + REGULATORY_REGION_VARIANT + ] + }, + frequencyFilter: { maxFrequency: 2.0 }, + pathogenicityFilter: { keepNonPathogenic: true }, + inheritanceFilter: { }, + omimPrioritiser: { }, + hiPhivePrioritiser: { } +] diff --git a/opencga-app/app/analysis/exomiser/output.yml b/opencga-app/app/analysis/exomiser/13.1/output.yml similarity index 100% rename from opencga-app/app/analysis/exomiser/output.yml rename to opencga-app/app/analysis/exomiser/13.1/output.yml diff --git a/opencga-app/app/analysis/exomiser/14.0/application.properties b/opencga-app/app/analysis/exomiser/14.0/application.properties new file mode 100644 index 00000000000..d1658370370 --- /dev/null +++ b/opencga-app/app/analysis/exomiser/14.0/application.properties @@ -0,0 +1,47 @@ +# +# The Exomiser - A tool to annotate and prioritize genomic variants +# +# Copyright (c) 2016-2021 Queen Mary University of London. +# Copyright (c) 2012-2016 Charité Universitätsmedizin Berlin and Genome Research Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# + +## exomiser root data directory ## +# root path where data is to be downloaded and worked on it is assumed that all the files required by exomiser listed +# in this properties file will be found in the data directory, unless specifically overridden here. +#exomiser.data-directory=/exomiser-data +exomiser.data-directory=/data + +#remm.version=0.3.1.post1 +#cadd.version=1.4 +#exomiser.hg38.data-version=1811 +#exomiser.hg38.data-version=2109 +#exomiser.hg38.remm-path=${exomiser.data-directory}/remm/ReMM.v${remm.version}.hg38.tsv.gz +#exomiser.hg38.variant-white-list-path=${exomiser.hg38.data-version}_hg38_clinvar_whitelist.tsv.gz +#exomiser.phenotype.data-version=2109 +#logging.file.name=/exomiser-data/logs/exomiser.log + + +#remm.version=0.3.1.post1 +#cadd.version=1.4 +#exomiser.hg19.data-version=1811 +exomiser.hg38.data-version=2402 +#exomiser.hg19.remm-path=${exomiser.data-directory}/remm/ReMM.v${remm.version}.hg19.tsv.gz +#exomiser.hg19.variant-white-list-path=${exomiser.hg19.data-version}_hg19_clinvar_whitelist.tsv.gz +#exomiser.hg38.variant-white-list-path=${exomiser.hg38.data-version}_hg38_clinvar_whitelist.tsv.gz +exomiser.phenotype.data-version=2402 +exomiser.hg38.clin-var-data-version=2402 +exomiser.hg38.use-clinvar-white-list=true +logging.file.name=/jobdir/exomiser.log diff --git a/opencga-app/app/analysis/exomiser/14.0/exomiser-analysis.yml b/opencga-app/app/analysis/exomiser/14.0/exomiser-analysis.yml new file mode 100644 index 00000000000..e3d064ba07b --- /dev/null +++ b/opencga-app/app/analysis/exomiser/14.0/exomiser-analysis.yml @@ -0,0 +1,85 @@ +--- +analysisMode: PASS_ONLY +inheritanceModes: { + AUTOSOMAL_DOMINANT: 0.1, + AUTOSOMAL_RECESSIVE_COMP_HET: 2.0, + AUTOSOMAL_RECESSIVE_HOM_ALT: 0.1, + X_DOMINANT: 0.1, + X_RECESSIVE_COMP_HET: 2.0, + X_RECESSIVE_HOM_ALT: 0.1, + MITOCHONDRIAL: 0.2 +} +frequencySources: [ + UK10K, + + GNOMAD_E_AFR, + GNOMAD_E_AMR, + # GNOMAD_E_ASJ, + GNOMAD_E_EAS, + # GNOMAD_E_FIN, + GNOMAD_E_NFE, + # GNOMAD_E_OTH, + GNOMAD_E_SAS, + + GNOMAD_G_AFR, + GNOMAD_G_AMR, + # GNOMAD_G_ASJ, + GNOMAD_G_EAS, + # GNOMAD_G_FIN, + GNOMAD_G_NFE, + # GNOMAD_G_OTH, + GNOMAD_G_SAS +] +# Possible pathogenicitySources: (POLYPHEN, MUTATION_TASTER, SIFT), (REVEL, MVP), CADD, REMM +# REMM is trained on non-coding regulatory regions +# *WARNING* if you enable CADD or REMM ensure that you have downloaded and installed the CADD/REMM tabix files +# and updated their location in the application.properties. Exomiser will not run without this. +pathogenicitySources: [ REVEL, MVP ] +#this is the standard exomiser order. +#all steps are optional +steps: [ + #hiPhivePrioritiser: {}, + #priorityScoreFilter: {priorityType: HIPHIVE_PRIORITY, minPriorityScore: 0.500}, + #intervalFilter: {interval: 'chr10:123256200-123256300'}, + # or for multiple intervals: + #intervalFilter: {intervals: ['chr10:123256200-123256300', 'chr10:123256290-123256350']}, + # or using a BED file - NOTE this should be 0-based, Exomiser otherwise uses 1-based coordinates in line with VCF + #intervalFilter: {bed: /full/path/to/bed_file.bed}, + #genePanelFilter: {geneSymbols: ['FGFR1','FGFR2']}, + #geneBlacklistFilter: { }, + failedVariantFilter: { }, + #qualityFilter: {minQuality: 50.0}, + variantEffectFilter: { + remove: [ + FIVE_PRIME_UTR_EXON_VARIANT, + FIVE_PRIME_UTR_INTRON_VARIANT, + THREE_PRIME_UTR_EXON_VARIANT, + THREE_PRIME_UTR_INTRON_VARIANT, + NON_CODING_TRANSCRIPT_EXON_VARIANT, + UPSTREAM_GENE_VARIANT, + INTERGENIC_VARIANT, + REGULATORY_REGION_VARIANT, + CODING_TRANSCRIPT_INTRON_VARIANT, + NON_CODING_TRANSCRIPT_INTRON_VARIANT, + DOWNSTREAM_GENE_VARIANT + ] + }, + # removes variants represented in the database + #knownVariantFilter: {}, + frequencyFilter: {maxFrequency: 2.0}, + pathogenicityFilter: {keepNonPathogenic: true}, + # inheritanceFilter and omimPrioritiser should always run AFTER all other filters have completed + # they will analyse genes according to the specified modeOfInheritance above- UNDEFINED will not be analysed. + inheritanceFilter: {}, + # omimPrioritiser isn't mandatory. + omimPrioritiser: {}, + #priorityScoreFilter: {minPriorityScore: 0.4}, + # Other prioritisers: Only combine omimPrioritiser with one of these. + # Don't include any if you only want to filter the variants. + hiPhivePrioritiser: {}, + # or run hiPhive in benchmarking mode: + #hiPhivePrioritiser: {runParams: 'mouse'}, + #phivePrioritiser: {} + #phenixPrioritiser: {} + #exomeWalkerPrioritiser: {seedGeneIds: [11111, 22222, 33333]} +] diff --git a/opencga-app/app/analysis/exomiser/14.0/output.yml b/opencga-app/app/analysis/exomiser/14.0/output.yml new file mode 100644 index 00000000000..4fc4a4faa6b --- /dev/null +++ b/opencga-app/app/analysis/exomiser/14.0/output.yml @@ -0,0 +1,11 @@ +--- +outputContributingVariantsOnly: true +#numGenes options: 0 = all or specify a limit e.g. 500 for the first 500 results +numGenes: 20 +minExomiserGeneScore: 0.7 +# Path to the desired output directory. Will default to the 'results' subdirectory of the exomiser install directory +outputDirectory: /jobdir/ +# Filename for the output files. Will default to {input-vcf-filename}-exomiser +outputFileName: exomiser_output +#out-format options: HTML, JSON, TSV_GENE, TSV_VARIANT, VCF (default: HTML) +outputFormats: [TSV_VARIANT, JSON, HTML] diff --git a/opencga-app/app/cloud/docker/compose/conf/client-configuration.yml b/opencga-app/app/cloud/docker/compose/conf/client-configuration.yml deleted file mode 100644 index b82c9f9a883..00000000000 --- a/opencga-app/app/cloud/docker/compose/conf/client-configuration.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- -logLevel: "INFO" - -# Number of seconds that session remains open without any activity [10 min] -cliSessionDuration: 600 - -## REST client configuration options -rest: - host: "http://rest:9090/opencga-2.4.0-SNAPSHOT" - tokenAutoRefresh: true - tlsAllowInvalidCertificates: false - timeout: 90000 - query: - batchSize: 200 - limit: 2000 - - -## gRPC configuration options -grpc: - host: "http://localhost:9091" - - -## Configure the default behaviour when query variants -variant: - unknownGenotype: "0/0" diff --git a/opencga-app/app/cloud/docker/compose/conf/configuration.yml b/opencga-app/app/cloud/docker/compose/conf/configuration.yml deleted file mode 100644 index 627644ab8b0..00000000000 --- a/opencga-app/app/cloud/docker/compose/conf/configuration.yml +++ /dev/null @@ -1,251 +0,0 @@ ---- -logLevel: "INFO" -logDir: /opt/opencga/logs - -databasePrefix: opencga -workspace: /opt/opencga/sessions -jobDir: /opt/opencga/sessions/jobs - -panel: - host: "http://resources.opencb.org/opencb/opencga/disease-panels" - -## Configuration for Catalog databases -catalog: - database: # MongoDB database credentials - hosts: - - mongodb:27017 - user: - password: - options: - authenticationDatabase: - connectionsPerHost: 20 - ## Solr Search engine configuration, by default is the same than storage - searchEngine: - # List of hosts pointing either to the Solr nodes directly using a complete URL or to the zookeper nodes with HOST:PORT - # Example for Solr connection: http://opencga-solr-01.zone:8983/solr - # Example for Zookeeper connection: opencga-zookeeper-01:2181 <-- Recommended for replicated installations - hosts: - - http://solr:8983/solr/ - user: "" - password: "" - options: - mode: "cloud" - timeout: 30000 - insertBatchSize: 2000 - -## We support multiple Authentication providers, if none is provided then we use an internal authentication implementation -authentication: - # Session expiration time in seconds - expiration: 3600 - authenticationOrigins: -# Custom LDAP | LDAPS configuration example -# - id: ldaps # Any id -# type: LDAP -# host: ldaps://localhost:636 # or ldap://localhost:123 -# options: -# authUserId: "" # Auth user id and password if querying the LDAP system requires authentication. By default, empty. -# authPassword: "" # Auth user id and password if querying the LDAP system requires authentication. By default, empty. -# usersSearch: dc=ge,dc=co,dc=uk # Mandatory field. Base search to look for users. By default, empty. -# groupsSearch: ou=general,ou=groups,dc=ge,dc=co,dc=uk # Mandatory field. Base search to look for groups. By default, empty. -# fullNameKey: displayname # Key to get the user's full name when importing users. By default, 'displayname'. -# memberKey: member # Key within groups to extract the user id. By default, 'member'. -# dnKey: dn # Key to get the user's DN or RDN unique identifier. By default, 'dn'. -# dnFormat: "%s" # Formatter to extract the user's DN if it is contained within a larger string. By default, '%s' to take the whole string. -# uidKey: uid # Key to get the user id to be used in OpenCGA. By default, 'uid'. -# uidFormat: "%s" # Formatter to extract the user id if the uid is contained within a larger string. By default, '%s' to take the whole string. -# sslInvalidCertificatesAllowed: false # If using LDAPs with a custom certificate, set to true to accept any certificate. By default, false. -# connectionTimeout: 500 # Connection timeout value. Default: 500 ms -# readTimeout: 1000 # Read timeout value. Default: 1000 ms - -# Azure AD configuration example -# - id: aad # Any id -# type: AzureAD -# host: -# options: -# tenantId: xxxx # Mandatory. Tenant id -# authClientId: xxxx # Mandatory. Client id of the client with permissions to authenticate users. -# syncClientId: xxxx # Mandatory. Client id of the client with permissions to inspect active directory. -# syncSecretKey: xxxx # Mandatory: Secret key of the client with permissions to inspect active directory. -# filters: tokenField1=aa,bb,cc;tokenField2=aa,bb,cc # Optional. Filters to be applied. OpenCGA will check if tokenField1 = aa or bb -# # or cc and tokenField2 = aa or bb or cc. If any of the filters don't succeed, even if the user is properly authenticated -# # in AAD, the user will not be able to generate a token and login in OpenCGA. - -server: - rest: - port: 9090 - logFile: null - defaultLimit: 2000 - maxLimit: 5000 - grpc: - port: 9091 - logFile: null - -optimizations: - simplifyPermissions: false - -audit: - manager: "" # Java manager of the audit implementation to be used to audit. If empty, catalog database will be used. - maxDocuments: 20000000 # Maximum number of documents that will be created in the audit collection. - maxSize: 100 # Maximum size that the audit collection will have in Gigabytes (GB). - -monitor: - daysToRemove: 30 - executionDaemonInterval: 4000 # number of milliseconds between checks - fileDaemonInterval: 8000 # number of milliseconds between checks - port: 9092 - -healthCheck: - interval: 30 # seconds to get actual healthCheck than cache - -#execution: -# mode: LOCAL -# maxConcurrentIndexJobs : 1 # only applies to local executor -# defaultQueue: "" -# availableQueues: "" -# toolsPerQueue: {} -# options: - - -analysis: - scratchDir: "" # Scratch folder for the analysis. - execution: - # Accepted values are "local", "SGE", "azure-batch", "k8s" - # see org.opencb.opencga.master.monitor.executors.ExecutorFactory - id: "LOCAL" - defaultQueue: "" # Default queue to be used to submit jobs - availableQueues: # Other queues for specific applications - toolsPerQueue: - # docker: - # - "circos" - # - "deeptools" - # - "bwa" - # - "fastqc" - # - "gatk" - # - "picard" - # - "plink" - # - "rvtests" - # - "samtools" - # - "relatedness" - # - "family-qc" - # - "sample-qc" - # - "individual-qc" - # - "mutational-signature" - # fast: - # - "alignmentIndex" - # slow: - # - "coverage" - # - "alignmentStats" - maxConcurrentJobs: - variant-index: 20 - variant-annotation-index: 5 - variant-secondary-index: 2 - options: - ## Local executor configuration - local.maxConcurrentJobs: 1 # Max number of concurrent jobs to be executed locally in the master - ## Azure Batch Service configuration example - # azure.batchAccount : "batchAccount" - # azure.batchKey : "batchKey" - # azure.batchUri : "https://batchservice.uksouth.batch.azure.com" - # azure.batchPoolId : "poolId" - # azure.dockerImageName : "openCGADockerImageName" - # azure.dockerArgs : "dockerRunOptions" - - ## Kubernetes executor configuration example - # k8s.masterUrl: "https://192.168.99.100:8443/" - k8s.clientTimeout: 30000 # ms - k8s.imageName: "opencb/opencga-base:2.4.0-SNAPSHOT-hdp3.1" - k8s.imagePullPolicy: "IfNotPresent" - # k8s.imagePullSecrets: - # name : "dockerhub-secrets-name" - k8s.namespace: "default" - # FOR Cluster Autoscaler and dedicated jobs agent pool: - k8s.requests: - cpu: 3 - memory: "12G" - k8s.limits: - cpu: 4 - memory: "13G" - k8s.nodeSelector: - agentpool: jobs - kubernetes.io/os: linux - kubernetes.io/role: agent - # FOR ACI: - # k8s.requests: - # cpu: 2 - # memory: "12G" - # k8s.limits: - # cpu: 2 - # memory: "12G" - # k8s.nodeSelector: - # kubernetes.io/role: agent - # kubernetes.io/os: linux - # type: virtual-kubelet - # k8s.tolerations: - # - key: virtual-kubelet.io/provider - # operator: Exists - # - key: azure.com/aci - # effect: NoSchedule - k8s.volumeMounts: - - name: conf - mountPath: /opt/opencga/conf - - name: sessions - mountPath: /opt/opencga/sessions - - name: variants - mountPath: /opt/opencga/variants - - name: analysisconf - mountPath: /opt/opencga/analysis - k8s.volumes: - - name: conf - persistentVolumeClaim: - claimName: "pvc-opencga-conf" - - name: sessions - persistentVolumeClaim: - claimName: "pvc-opencga-sessions" - - name: variants - persistentVolumeClaim: - claimName: "pvc-opencga-variants" - - name: analysisconf - persistentVolumeClaim: - claimName: "pvc-opencga-analysisconf" - k8s.podSecurityContext: - runAsNonRoot: true - k8s.securityContext: - runAsNonRoot: true - runAsUser: 1001 - readOnlyRootFilesystem: false - - # List of analysis frameworks - frameworks: - - id: "local" - available: true - options: { } - - - id: "spark" - available: false - queue: "hadoop_queue" # Special executor queue to be used for jobs using this framework. - options: # Spark properties from https://spark.apache.org/docs/latest/configuration.html#viewing-spark-properties - spark.executor.memory: "1g" - - - id: "mapreduce" - available: false - queue: "hadoop_queue" # Special executor queue to be used for jobs using this framework. This is NOT a yarn queue. - options: # MapReduce configuration from https://hadoop.apache.org/docs/r2.7.2/hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml - mapreduce.job.queuename: default - -email: - host: smtp.gmail.com - port: 587 - user: juanfe.sanahuja@zettagenomics.com - password: zhwbmcbcefoxmjht - from: "" - ssl: false - -#hooks: -# user@project:study: # Full Qualified Name of the study. -# file: # Entity where the hook will be checked -# - field: "name" # Field of the entity to be checked -# value: "~(.*)SV.vcf.gz$" # Value that needs to be satisfied to perform the hook action -# stage: "CREATE" # Stage when the hook will be checked -# action: "ADD" # Action to be performed -# where: "tags" # Field over which the action will be performed -# what: "SV" # Value to be updated diff --git a/opencga-app/app/cloud/docker/compose/conf/opencga-env.sh b/opencga-app/app/cloud/docker/compose/conf/opencga-env.sh deleted file mode 100644 index f99e1375afe..00000000000 --- a/opencga-app/app/cloud/docker/compose/conf/opencga-env.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env bash - -# Variables defined in main script -# BASEDIR -# PRGDIR -# JAVA_OPTS -# CLASSPATH_PREFIX - - -# Get configuration depending on the prog name -case $(basename "$PRG") in - "opencga-admin.sh") - export JAVA_HEAP=${JAVA_HEAP:-"8192m"} - export OPENCGA_LOG4J_CONFIGURATION_FILE=log4j2.xml - export OPENCGA_LOG_DIR=${OPENCGA_LOG_DIR:-$(grep "logDir" "${BASEDIR}/conf/configuration.yml" | cut -d ":" -f 2 | tr -d '" ')} - ;; - "opencga-internal.sh") - export JAVA_HEAP=${JAVA_HEAP:-"12288m"} - export OPENCGA_LOG4J_CONFIGURATION_FILE=log4j2.internal.xml - export OPENCGA_LOG_DIR=${OPENCGA_LOG_DIR:-$(grep "logDir" "${BASEDIR}/conf/configuration.yml" | cut -d ":" -f 2 | tr -d '" ')} - ;; -# "opencga.sh") - *) - export JAVA_HEAP=${JAVA_HEAP:-"2048m"} - export OPENCGA_LOG4J_CONFIGURATION_FILE=log4j2.xml - export OPENCGA_LOG_DIR="" - ;; -esac - - -#Set log4j properties file -export JAVA_OPTS="${JAVA_OPTS} -Dlog4j2.configurationFile=file:${BASEDIR}/conf/${OPENCGA_LOG4J_CONFIGURATION_FILE}" -if [ -n "$OPENCGA_LOG_DIR" ]; then - export JAVA_OPTS="${JAVA_OPTS} -Dopencga.log.dir=${OPENCGA_LOG_DIR}" -fi -export JAVA_OPTS="${JAVA_OPTS} -Dfile.encoding=UTF-8" -export JAVA_OPTS="${JAVA_OPTS} -Xms256m -Xmx${JAVA_HEAP}" - -# Configure JavaAgent -JAVA_AGENT="" -AGENTS_NUM="$(find "${BASEDIR}/monitor/" -name '*.jar' 2> /dev/null | wc -l)" -if [ "$AGENTS_NUM" -eq 1 ]; then - JAVA_AGENT="$(find "${BASEDIR}/monitor/" -name '*.jar')" - export JAVA_OPTS="${JAVA_OPTS} -javaagent:${JAVA_AGENT}" -elif [ "$AGENTS_NUM" -gt 1 ]; then - echo "ERROR - Multiple java agents found!" 1>&2 - exit 2 -fi - - -export COLUMNS=$(tput cols 2> /dev/null) -export LINES=$(tput lines 2> /dev/null) - -# export OPENCGA_HOME=${BASEDIR} - -if ( ls "${BASEDIR}"/libs/opencga-storage-hadoop-core-*.jar >/dev/null 2>&1 ) ; then - - # Add the folder conf/hadoop to the classpath. - # Add first the user defined hadoop configuration, and then the system hadoop conf, to allow the user to overwrite the configuration. - if [ -z "${CLASSPATH_PREFIX}" ]; then - export CLASSPATH_PREFIX=${BASEDIR}/conf/hadoop/ - else - export CLASSPATH_PREFIX=${CLASSPATH_PREFIX}:${BASEDIR}/conf/hadoop/ - fi - - # Add the system hadoop configuration folders to the classpath. - # Check if the command hbase exists - # see http://stackoverflow.com/questions/592620/check-if-a-program-exists-from-a-bash-script - if ( command -v hbase >/dev/null 2>&1 ); then - hbase_conf=$(hbase classpath | tr ":" "\n" | grep "/conf" | tr "\n" ":") - export CLASSPATH_PREFIX=${CLASSPATH_PREFIX}:${hbase_conf} - fi - - # Add specific Amazon EMR dependencies - if ( command -v hadoop >/dev/null 2>&1 ); then - EMR_DEPENDENCIES=`find $(hadoop classpath | tr ":" " ") -type f -name "hadoop-common-*-amzn-3.jar" -o -name "emrfs-hadoop-assembly-*.jar" 2> /dev/null | tr "\n" ":"` - export CLASSPATH_PREFIX=${CLASSPATH_PREFIX}:${EMR_DEPENDENCIES} - fi - - phoenix="" - if ( command -v phoenix_utils.py > /dev/null 2>&1 ); then - phoenix=$(phoenix_utils.py | grep phoenix_client_jar | cut -f 2 -d " ") - fi - - jackson=$(find "${BASEDIR}/libs/" -name "jackson-*-2.[0-9].[0-9].jar" | tr "\n" ":") - proto=$(find "${BASEDIR}/libs/" -name "protobuf-java-*.jar" | tr "\n" ":") - avro=$(find "${BASEDIR}/libs/" -name "avro-*.jar" | tr "\n" ":") - export HADOOP_CLASSPATH="${phoenix}:${proto}:${avro}:${jackson}:${CLASSPATH_PREFIX}" - export HADOOP_USER_CLASSPATH_FIRST=true -fi \ No newline at end of file diff --git a/opencga-app/app/cloud/docker/compose/conf/storage-configuration.yml b/opencga-app/app/cloud/docker/compose/conf/storage-configuration.yml deleted file mode 100644 index 84b94de81ab..00000000000 --- a/opencga-app/app/cloud/docker/compose/conf/storage-configuration.yml +++ /dev/null @@ -1,219 +0,0 @@ ---- - -## CellBase client configuration, this is used mainly when annotating variants -## This can be set up using maven profiles in your .m2/settings.xml -cellbase: - ## URL host to annotate variants, for example: http://ws.opencb.org/cellbase/ - url: "https://ws.opencb.org/cellbase/" - version: "v4" - -## Storage Query Server configuration. When CLI is launched in 'server' mode a RESTful web server -## is launched in the specified port. -server: - rest: - port: 9090 - logFile: null - grpc: - port: 9091 - logFile: null - -## Solr Search Configuration -search: - # List of hosts pointing either to the Solr nodes directly using a complete URL or to the zookeper nodes with HOST:PORT - # Example for Solr connection: http://opencga-solr-01.zone:8983/solr - # Example for Zookeeper connection: opencga-zookeeper-01:2181 <-- Recommended for replicated installations - hosts: - - http://solr:8983/solr/ - configSet: "opencga-variant-configset-2.4.0-SNAPSHOT" - mode: "cloud" - timeout: 30000 - insertBatchSize: 5000 - -## Clinical database for indexing the pathogenic variants reported. -clinical: - # List of hosts pointing either to the Solr nodes directly using a complete URL or to the zookeper nodes with HOST:PORT - # Example for Solr connection: http://opencga-solr-01.zone:8983/solr - # Example for Zookeeper connection: opencga-zookeeper-01:2181 <-- Recommended for replicated installations - hosts: - - http://solr:8983/solr/ # URL containing host and port, e.g. http://localhost:8983/solr/ - mode: "cloud" - manager: "" - timeout: 30000 - insertBatchSize: 1000 - -## Recessive Gene Analysis database. -rga: - # List of hosts pointing either to the Solr nodes directly using a complete URL or to the zookeper nodes with HOST:PORT - # Example for Solr connection: http://opencga-solr-01.zone:8983/solr - # Example for Zookeeper connection: opencga-zookeeper-01:2181 <-- Recommended for replicated installations - hosts: - - http://solr:8983/solr/ # URL containing host and port, e.g. http://localhost:8983/solr/ - configSet: "opencga-rga-configset-2.4.0-SNAPSHOT" - mode: "cloud" - timeout: 30000 - insertBatchSize: 1000 - -benchmark: - numRepetitions: 20 # Number of query repetitions - concurrency: 5 # Number of concurrent threads - delay: 100 # Delay between queries - connectionType: REST # Select between REST or DIRECT. - mode: FIXED # Select between FIXED or RANDOM - databaseName: "opencga" - rest: "http://localhost:8080/opencga-2.4.0-SNAPSHOT" - -io: - connectors: -# azure: -# class: "org.opencb.opencga.storage.core.io.managers.AzureBlobStorageIOConnector" -# options: -# accountName: "" -# accountKey: "" - -alignment: - bigWigWindowsSize: 1 - -variant: - defaultEngine: "mongodb" - options: - transform.batchSize: 200 - transform.numThreads: 4 - transform.format: "avro" - transform.compression: "gzip" - transform.failOnMalformed: true - - normalization.skip: false # Skip normalization - normalization.referenceGenome: "" # Reference genome localization for improved normalization - normalization.extensions: [ ] # Enable VariantNormalizerExtensions - # - "VAF" # Compute EXT_VAF for each sample. Variant Allele Fraction (VAF), several variant callers supported - - load.batchSize: 100 - load.numThreads: 6 - - stats.defaultGenotype: "0/0" # Default genotype to be used for calculating stats. - stats.multiAllelic: false # Include secondary alternates in the variant stats calculation - stats.calculate.batchSize: 100 - stats.calculate.numThreads: 4 - stats.load.batchSize: 100 - stats.load.numThreads: 4 - - - annotation.batchSize: 100 - annotation.numThreads: 8 - annotation.checkpointSize: 5000000 - annotation.timeout: 600000 # millis - annotation.load.batchSize: 100 - annotation.load.numThreads: 4 - annotation.file.format: "json" - annotator: "cellbase" - #annotator.class: #Allows to inject custom annotators - annotator.cellbase.exclude: "expression" - annotator.cellbase.useCache: true - annotator.cellbase.impreciseVariants: true # Imprecise variants supported by cellbase - annotator.cellbase.starAlternate: false # Variants with stat "*" alternate supported by cellbase - #annotator.cellbase.variantLengthThreshold: # Variants length threshold. By default, unlimited - - query.timeout.default: 10000 #(ms) Default timeout for DBAdaptor operations. Only used if none is provided. - query.timeout.max: 30000 #(ms) Max allowed timeout for DBAdaptor operations. - query.limit.default: 1000 # Default limit in GET operations. To be used only if not defined. - query.limit.max: 5000 # Maximum limit value in GET operations. If tried to be exceeded, the query will fail. - query.sample.limit.default: 100 # Default sampleLimit in GET operations. To be used only if not defined. - query.sample.limit.max: 1000 # Maximum sampleLimit value in GET operations. If tried to be exceeded, the query will fail. - - search.intersect.active: true # Allow intersect queries with the SearchEngine (Solr) - search.intersect.always: false # Force intersect queries - search.intersect.params.threshold: 3 # Minimum number of QueryParams in the query to intersect - - ## The following section defines all available storage engine plugins installed - engines: - ## MongoDB Storage Engine - - id: "mongodb" - engine: "org.opencb.opencga.storage.mongodb.variant.MongoDBVariantStorageEngine" - database: - hosts: - - "mongodb:27017" - user: "" - password: "" - options: ## This is intended for database specific options such as --authenticationDatabase in MongoDB - authenticationDatabase: - connectionsPerHost: 20 - readPreference: "secondaryPreferred" - options: - storage.mongodb.parallelWrite: false - storage.mongodb.stage.parallelWrite: false - storage.mongodb.directLoad.parallelWrite: false - storage.mongodb.merge.parallelWrite: false - storage.mongodb.merge.batchSize: 10 #Number of files to merge directly from first to second collection - - ## Hadoop Storage Engine - - id: "hadoop" - engine: "org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageEngine" - database: - # PENDING Kerberos authentication support: - # See https://github.com/opencb/opencga/issues/1292 - user: "" - password: "" - options: - storage.hadoop.hbase.namespace: "" - storage.hadoop.archive.table.preSplit.splitsPerBatch: 500 - storage.hadoop.archive.table.compression: "gz" # Allowed values: none, snappy, gz - storage.hadoop.archive.table.chunkSize: 1000 - storage.hadoop.archive.table.fileBatchSize: 1000 - storage.hadoop.variant.table.preSplit.numSplits: 500 - storage.hadoop.variant.table.compression: "snappy" # Allowed values: none, snappy, gz - storage.hadoop.sampleIndex.table.preSplit.samplesPerSplit: 15 - storage.hadoop.sampleIndex.table.compression: "snappy" # Allowed values: none, snappy, gz - storage.hadoop.sampleIndex.build.maxSamplesPerMR: 5000 - storage.hadoop.sampleIndex.annotation.maxSamplesPerMR: 5000 - storage.hadoop.annotationIndex.table.compression: "snappy" # Allowed values: none, snappy, gz - storage.hadoop.pendingAnnotation.table.compression: "snappy" # Allowed values: none, snappy, gz - - # Batch size for querying phoenix - storage.hadoop.phoenix.fetchSize: -1 - - # Hadoop executable file. Used to lunch MapReduce applications - storage.hadoop.bin: "hadoop" - - ## If undefined, it will be generated based on the build version. - #storage.hadoop.mr.jarWithDependencies: "opencga-storage-hadoop-core-2.4.0-SNAPSHOT-jar-with-dependencies.jar" - - # Define the MapReduce job executor. - storage.hadoop.mr.executor: "system" # Either "system" or "ssh". - - # Use external hadoop installation. ssh to a hadoop edge node - storage.hadoop.mr.executor.ssh.host: "" # Hadoop edge node host name - storage.hadoop.mr.executor.ssh.user: "" # Hadoop edge node user name - #storage.hadoop.mr.executor.ssh.key: "~/.ssh/id_rsa" # Hadoop edge node ssh-key file - storage.hadoop.mr.executor.ssh.password: "" # Hadoop edge node password. Only if ssh-key is not present. Requires sshpass to run - storage.hadoop.mr.executor.ssh.remoteOpenCgaHome: # Remote opencga home location. Only if different than local location. - - # Increase the ScannerTimeoutPeriod from 60000 (1min) to 300000 (5min) to avoid ScannerTimeoutExceptions - # See opencb/opencga#352 for more info. - storage.hadoop.mr.scanner.timeout: 300000 - - mapreduce.map.memory.mb: 2048 - DeleteHBaseColumnDriver: - storage.hadoop.write.mappers.limit.factor: 4 - DiscoverPendingVariantsDriver: - mapreduce.map.memory.mb: 750 - VariantStatsDriver: - mapreduce.map.memory.mb: 2048 - SampleIndexDriver: - mapreduce.map.memory.mb: 4096 - max-columns-per-scan: 4000 - SampleIndexAnnotationLoaderDriver: - mapreduce.map.memory.mb: 4096 - VariantMigration200Driver: - mapreduce.map.memory.mb: 1024 - - -## PENDING -## Cache Configuration -cache: - host: localhost:6379 - active: true - serialization: "json" - slowThreshold: 50 - allowedTypes: "aln,var" - maxResultSize: 5000 - password: "" \ No newline at end of file diff --git a/opencga-app/app/cloud/docker/docker-build.py b/opencga-app/app/cloud/docker/docker-build.py index 2d4bd0ba07e..49459de914d 100755 --- a/opencga-app/app/cloud/docker/docker-build.py +++ b/opencga-app/app/cloud/docker/docker-build.py @@ -88,7 +88,7 @@ def build(): def tag_latest(image): - if "hdp" in tag or "dev" in tag: + if "hdi" in tag or "emr" in tag or "dev" in tag or "SNAPSHOT" in tag or "TASK" in tag: print("Don't use tag " + tag + " as latest") return if server: @@ -160,7 +160,7 @@ def delete(): build_folder = str(Path(__file__).resolve().parents[2]) # 2. Set docker tag to default value if not set -if args.tag is not None: +if args.tag is not None and not args.tag == "": tag = args.tag else: # Read OpenCGA version from git.properties @@ -177,9 +177,9 @@ def delete(): # Get hadoop_flavour from JAR library file name hadoop_flavour = None for file in os.listdir(build_folder + "/libs/"): - if (file.startswith("opencga-storage-hadoop-deps")): + if (file.startswith("opencga-storage-hadoop-lib-") and file.endswith(".jar")): if hadoop_flavour is not None: - exit("Error. Multiple libs/opencga-storage-hadoop-deps*.jar found") + exit("Error. Multiple libs/opencga-storage-hadoop-lib*.jar found") hadoop_flavour = file.split("-")[4] # Create docker tag diff --git a/opencga-app/app/cloud/docker/opencga-init/override_yaml.py b/opencga-app/app/cloud/docker/opencga-init/override_yaml.py index 1993f85232b..04bd8139e82 100644 --- a/opencga-app/app/cloud/docker/opencga-init/override_yaml.py +++ b/opencga-app/app/cloud/docker/opencga-init/override_yaml.py @@ -19,9 +19,6 @@ parser.add_argument("--catalog-database-authentication-database", required=False, default="admin") parser.add_argument("--catalog-database-authentication-mechanism", required=False) parser.add_argument("--catalog-database-replica-set", required=False) -parser.add_argument("--catalog-search-hosts", required=True) -parser.add_argument("--catalog-search-user", required=False) -parser.add_argument("--catalog-search-password", required=False) parser.add_argument("--rest-host", required=True) parser.add_argument("--grpc-host", required=True) parser.add_argument("--analysis-execution-mode", required=False) @@ -138,13 +135,6 @@ def hostOverride(conf,hosts_var): if args.catalog_database_authentication_mechanism is not None: config["catalog"]["database"]["options"]["authenticationMechanism"] = args.catalog_database_authentication_mechanism -# Inject search database -hostOverride(config["catalog"]["searchEngine"], args.catalog_search_hosts) - -if args.catalog_search_user is not None: - config["catalog"]["searchEngine"]["user"] = args.catalog_search_user - config["catalog"]["searchEngine"]["password"] = args.catalog_search_password - # Inject execution settings config["analysis"]["scratchDir"] = "/tmp/opencga_scratch" if args.max_concurrent_jobs is not None: diff --git a/opencga-app/app/cloud/docker/opencga-init/test/test_override_yaml.py b/opencga-app/app/cloud/docker/opencga-init/test/test_override_yaml.py index 9d28765d399..ee6a547ea2b 100644 --- a/opencga-app/app/cloud/docker/opencga-init/test/test_override_yaml.py +++ b/opencga-app/app/cloud/docker/opencga-init/test/test_override_yaml.py @@ -1,11 +1,10 @@ +import os import subprocess -from shutil import copyfile +import sys import unittest import yaml from io import StringIO -import sys -import os - +from shutil import copyfile os.chdir(sys.path[0]) @@ -40,9 +39,6 @@ def test_end_2_end(self): "--catalog-database-hosts", "test-catalog-database-host1,test-catalog-database-host2,test-catalog-database-host3", "--catalog-database-user", "test-catalog-database-user", "--catalog-database-password", "test-catalog-database-password", - "--catalog-search-hosts", "test-catalog-search-host1,test-catalog-search-host2", - "--catalog-search-user", "test-catalog-search-user", - "--catalog-search-password", "test-catalog-search-password", "--rest-host", "test-rest-host", "--grpc-host", "test-grpc-host", "--max-concurrent-jobs", "25", @@ -170,18 +166,6 @@ def test_end_2_end(self): self.assertEqual(config["catalog"]["database"]["options"]["sslEnabled"], True) self.assertEqual(config["catalog"]["database"]["options"]["sslInvalidCertificatesAllowed"], True) self.assertEqual(config["catalog"]["database"]["options"]["authenticationDatabase"], "admin") - self.assertEqual( - config["catalog"]["searchEngine"]["hosts"][0], "test-catalog-search-host1" - ) - self.assertEqual( - config["catalog"]["searchEngine"]["hosts"][1], "test-catalog-search-host2" - ) - self.assertEqual( - config["catalog"]["searchEngine"]["user"], "test-catalog-search-user" - ) - self.assertEqual( - config["catalog"]["searchEngine"]["password"], "test-catalog-search-password" - ) self.assertEqual(config["analysis"]["execution"]["id"], "test-analysis-execution-mode") self.assertEqual(config["analysis"]["execution"]["maxConcurrentJobs"]["variant-index"], 25) self.assertEqual(client_config["rest"]["hosts"][0]["url"], "test-rest-host") @@ -200,9 +184,6 @@ def test_azure_batch_execution(self): "--catalog-database-hosts", "test-catalog-database-host1,test-catalog-database-host2,test-catalog-database-host3", "--catalog-database-user", "test-catalog-database-user", "--catalog-database-password", "test-catalog-database-password", - "--catalog-search-hosts", "test-catalog-search-host1,test-catalog-search-host2", - "--catalog-search-user", "test-catalog-search-user", - "--catalog-search-password", "test-catalog-search-password", "--rest-host", "test-rest-host", "--grpc-host", "test-grpc-host", "--analysis-execution-mode", "AZURE", @@ -273,9 +254,6 @@ def test_kubernetes_execution(self): "--catalog-database-hosts", "test-catalog-database-host1,test-catalog-database-host2,test-catalog-database-host3", "--catalog-database-user", "test-catalog-database-user", "--catalog-database-password", "test-catalog-database-password", - "--catalog-search-hosts", "test-catalog-search-host1,test-catalog-search-host2", - "--catalog-search-user", "test-catalog-search-user", - "--catalog-search-password", "test-catalog-search-password", "--rest-host", "test-rest-host", "--grpc-host", "test-grpc-host", "--analysis-execution-mode", "k8s", @@ -357,12 +335,6 @@ def test_cellbasedb_with_empty_hosts(self): "test-catalog-database-user", "--catalog-database-password", "test-catalog-database-password", - "--catalog-search-hosts", - "test-catalog-search-host1,test-catalog-search-host2", - "--catalog-search-user", - "test-catalog-search-user", - "--catalog-search-password", - "test-catalog-search-password", "--rest-host", "test-rest-host", "--grpc-host", @@ -437,12 +409,6 @@ def test_cellbasedb_with_no_db_hosts(self): "test-catalog-database-user", "--catalog-database-password", "test-catalog-database-password", - "--catalog-search-hosts", - "test-catalog-search-host1,test-catalog-search-host2", - "--catalog-search-user", - "test-catalog-search-user", - "--catalog-search-password", - "test-catalog-search-password", "--rest-host", "test-rest-host", "--grpc-host", @@ -519,12 +485,6 @@ def test_cellbase_rest_set(self): "test-catalog-database-user", "--catalog-database-password", "test-catalog-database-password", - "--catalog-search-hosts", - "test-catalog-search-host1,test-catalog-search-host2", - "--catalog-search-user", - "test-catalog-search-user", - "--catalog-search-password", - "test-catalog-search-password", "--rest-host", "test-rest-host", "--grpc-host", @@ -604,12 +564,6 @@ def test_cellbase_rest_empty_set(self): "test-catalog-database-user", "--catalog-database-password", "test-catalog-database-password", - "--catalog-search-hosts", - "test-catalog-search-host1,test-catalog-search-host2", - "--catalog-search-user", - "test-catalog-search-user", - "--catalog-search-password", - "test-catalog-search-password", "--rest-host", "test-rest-host", "--grpc-host", @@ -688,12 +642,6 @@ def test_cellbase_rest_not_set(self): "test-catalog-database-user", "--catalog-database-password", "test-catalog-database-password", - "--catalog-search-hosts", - "test-catalog-search-host1,test-catalog-search-host2", - "--catalog-search-user", - "test-catalog-search-user", - "--catalog-search-password", - "test-catalog-search-password", "--rest-host", "test-rest-host", "--grpc-host", diff --git a/opencga-app/app/misc/clients/copyright.txt b/opencga-app/app/misc/clients/copyright.txt new file mode 100644 index 00000000000..5c551251867 --- /dev/null +++ b/opencga-app/app/misc/clients/copyright.txt @@ -0,0 +1,13 @@ +Copyright 2015-2024 OpenCB + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/opencga-app/app/misc/clients/java_client_generator.py b/opencga-app/app/misc/clients/java_client_generator.py index 9555f4fe4b1..f65a89983e9 100755 --- a/opencga-app/app/misc/clients/java_client_generator.py +++ b/opencga-app/app/misc/clients/java_client_generator.py @@ -36,19 +36,7 @@ def get_imports(self): headers = [] headers.append('/*') - headers.append('* Copyright 2015-' + str(date.today().year) + ' OpenCB') - headers.append('*') - headers.append('* Licensed under the Apache License, Version 2.0 (the "License");') - headers.append('* you may not use this file except in compliance with the License.') - headers.append('* You may obtain a copy of the License at') - headers.append('*') - headers.append('* http://www.apache.org/licenses/LICENSE-2.0') - headers.append('*') - headers.append('* Unless required by applicable law or agreed to in writing, software') - headers.append('* distributed under the License is distributed on an "AS IS" BASIS,') - headers.append('* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.') - headers.append('* See the License for the specific language governing permissions and') - headers.append('* limitations under the License.') + headers.append(self.get_copyright_message(prefix='* ')) headers.append('*/') headers.append('') # We need to calculate the Java package and no use: headers.append('package org.opencb.opencga.client.rest.clients;') @@ -99,7 +87,6 @@ def get_class_definition(self, category): text.append('/**') text.append(' * This class contains methods for the {} webservices.'.format( self.categories[self.get_category_name(category)])) - text.append(' *{}Client version: {}'.format(' ' * 4, self.version)) text.append(' *{}PATH: {}'.format(' ' * 4, self.get_category_path(category))) text.append(' */') text.append('public class {}Client extends {} {{'.format(self.categories[self.get_category_name(category)], parentClientClass)) diff --git a/opencga-app/app/misc/clients/javascript_client_generator.py b/opencga-app/app/misc/clients/javascript_client_generator.py index b6955e46fe4..e471d09ac48 100644 --- a/opencga-app/app/misc/clients/javascript_client_generator.py +++ b/opencga-app/app/misc/clients/javascript_client_generator.py @@ -28,18 +28,10 @@ def __init__(self, server_url, output_dir): } def get_imports(self): + copyright = self.get_copyright_message(prefix=' * ') auto_msg = "\n * ".join(self.get_autogenerated_message()) return (f'/**\n' - f' * Copyright 2015-2020 OpenCB\n' - f' * Licensed under the Apache License, Version 2.0 (the "License");\n' - f' * you may not use this file except in compliance with the License.\n' - f' * You may obtain a copy of the License at\n' - f' * http://www.apache.org/licenses/LICENSE-2.0\n' - f' * Unless required by applicable law or agreed to in writing, software\n' - f' * distributed under the License is distributed on an "AS IS" BASIS,\n' - f' * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n' - f' * See the License for the specific language governing permissions and\n' - f' * limitations under the License.\n' + f'{copyright}\n' f' * {auto_msg}' f' \n *\n**/\n\n' f'import OpenCGAParentClass from "./../opencga-parent-class.js";\n\n') @@ -124,7 +116,7 @@ def get_method_definition(self, category, endpoint): f'"{self.get_endpoint_subcategory()}"' if self.subcategory else "null", self.get_endpoint_id2() if self.get_endpoint_id2() else "null", f'"{self.get_endpoint_action()}"' if self.get_endpoint_action() else "null", - "data" if self.has_body() else False, + "data" if self.has_body() else ("null" if self.get_endpoint_method(endpoint).lower() == "post" else False), query_string_params ] if s) return (f' {self.get_method_doc(endpoint)}' diff --git a/opencga-app/app/misc/clients/python_client_generator.py b/opencga-app/app/misc/clients/python_client_generator.py index 8a0ba4320c6..ada636dc5c1 100644 --- a/opencga-app/app/misc/clients/python_client_generator.py +++ b/opencga-app/app/misc/clients/python_client_generator.py @@ -70,7 +70,6 @@ def get_class_definition(self, category): text.append('{}"""'.format(' ' * 4)) text.append('{}This class contains methods for the \'{}\' webservices'.format(' ' * 4, self.get_category_name(category))) - text.append('{}Client version: {}'.format(' ' * 4, self.version)) text.append('{}PATH: {}'.format(' ' * 4, category['path'])) text.append('{}"""'.format(' ' * 4)) text.append('') diff --git a/opencga-app/app/misc/clients/r_client_generator.py b/opencga-app/app/misc/clients/r_client_generator.py index 0e8fd8ca7ec..2d2a4840a97 100644 --- a/opencga-app/app/misc/clients/r_client_generator.py +++ b/opencga-app/app/misc/clients/r_client_generator.py @@ -11,6 +11,7 @@ def __init__(self, server_url, output_dir): super().__init__(server_url, output_dir) self.categories = { + 'Organizations': 'Organization', 'Users': 'User', 'Projects': 'Project', 'Studies': 'Study', @@ -26,7 +27,7 @@ def __init__(self, server_url, output_dir): 'Analysis - Clinical': 'Clinical', 'Operations - Variant Storage': 'Operation', 'Meta': 'Meta', - 'Cva': 'Cva', + 'Cvdb': 'Cvdb', 'GA4GH': 'GA4GH', 'Admin': 'Admin' } @@ -74,6 +75,7 @@ def get_class_definition(self, category): myEndpoint['path'], ", ".join(endpoint_params))) path_params = set(class_path_params) + path_params = sorted(list(path_params)) text.append("#'\n#' @md") text.append("#' @seealso \\url{http://docs.opencb.org/display/opencga/Using+OpenCGA} and the RESTful API documentation") text.append("#' \\url{http://bioinfo.hpc.cam.ac.uk/opencga-prod/webservices/}") diff --git a/opencga-app/app/misc/clients/rest_client_generator.py b/opencga-app/app/misc/clients/rest_client_generator.py index bfb3b78489f..53ed2bdfd8f 100644 --- a/opencga-app/app/misc/clients/rest_client_generator.py +++ b/opencga-app/app/misc/clients/rest_client_generator.py @@ -35,6 +35,7 @@ def __init__(self, rest_api_file, output_dir): 'panels/import': {'method_name': 'import_panels'} } self.categories = { + 'Organizations': 'Organization', 'Users': 'User', 'Projects': 'Project', 'Studies': 'Study', @@ -50,19 +51,44 @@ def __init__(self, rest_api_file, output_dir): 'Analysis - Clinical': 'ClinicalAnalysis', 'Operations - Variant Storage': 'VariantOperation', 'Meta': 'Meta', - 'Cva': 'Cva', + 'Cvdb': 'Cvdb', 'GA4GH': 'GA4GH', 'Admin': 'Admin' } + @staticmethod + def get_copyright_message(prefix=""): + # copyright = [ + # 'Copyright 2015-2024 OpenCB', + # '', + # 'Licensed under the Apache License, Version 2.0 (the "License");', + # 'you may not use this file except in compliance with the License.', + # 'You may obtain a copy of the License at', + # '', + # ' http://www.apache.org/licenses/LICENSE-2.0', + # '', + # 'Unless required by applicable law or agreed to in writing, software', + # 'distributed under the License is distributed on an "AS IS" BASIS,', + # 'WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.', + # 'See the License for the specific language governing permissions and', + # 'limitations under the License.' + # ] + copyright_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'copyright.txt') + print("Reading from file: ", copyright_file) + with open(copyright_file, 'r') as file: + copyright = file.read().splitlines() + ## Add prefix to each line + if prefix: + for line in range(len(copyright)): + copyright[line] = (prefix + copyright[line]).rstrip() + return "\n".join(copyright) + @staticmethod def get_autogenerated_message(): - date_ = datetime.now().strftime("%Y-%m-%d") return [ 'WARNING: AUTOGENERATED CODE', '', 'This code was generated by a tool.', - 'Autogenerated on: ' + date_, '', 'Manual changes to this file may cause unexpected behavior in your application.', 'Manual changes to this file will be overwritten if the code is regenerated.' diff --git a/opencga-app/app/misc/scripts/hadoop-ssh.sh b/opencga-app/app/misc/scripts/hadoop-ssh.sh index 30141a04074..dc07d1b96f9 100755 --- a/opencga-app/app/misc/scripts/hadoop-ssh.sh +++ b/opencga-app/app/misc/scripts/hadoop-ssh.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env sh +#!/usr/bin/env bash if [ -z "${HADOOP_SSH_USER}" ] ; then @@ -25,6 +25,8 @@ if [ -n "${HADOOP_SSH_KEY}" ] && [ -f "${HADOOP_SSH_KEY}" ] ; then SSH_OPTS="${SSH_OPTS} -i ${HADOOP_SSH_KEY}" fi +trap 'echo "SSH Process interrupted! Run time : ${SECONDS}s" && exit 1 1>&2 ' INT TERM + echo "Connect to Hadoop edge node ${HADOOP_SSH_USER}@${HADOOP_SSH_HOST}" 1>&2 echo "${SSHPASS_CMD}ssh ${SSH_OPTS} ${HADOOP_SSH_USER}@${HADOOP_SSH_HOST}" 1>&2 @@ -37,10 +39,12 @@ for arg in "$@" ; do arg=$(echo "$arg" | sed "s/'/'\\\\\\''/g") # aaa'aaa --> 'aaa'\''aaa' CMD="${CMD}'${arg}' " done -echo ${CMD} +echo ${CMD} 1>&2 ${SSHPASS_CMD} ssh ${SSH_OPTS} "${HADOOP_SSH_USER}@${HADOOP_SSH_HOST}" /bin/bash << EOF +echo "PID=\$\$" >&2 + export HADOOP_CLASSPATH=${HADOOP_CLASSPATH} export HADOOP_USER_CLASSPATH_FIRST=${HADOOP_USER_CLASSPATH_FIRST} @@ -59,3 +63,10 @@ exec ${CMD} EOF +EXIT_CODE=$? + +echo "SSH Process completed!" 1>&2 +echo " - Run time : ${SECONDS}s" 1>&2 +echo " - Exit code: ${EXIT_CODE}" 1>&2 + +exit ${EXIT_CODE} \ No newline at end of file diff --git a/opencga-app/app/misc/scripts/opencga-storage-hadoop.sh b/opencga-app/app/misc/scripts/opencga-storage-hadoop.sh deleted file mode 100755 index 1268c474c4b..00000000000 --- a/opencga-app/app/misc/scripts/opencga-storage-hadoop.sh +++ /dev/null @@ -1,111 +0,0 @@ -#!/bin/sh - -# resolve links - $0 may be a softlink -PRG="$0" - -while [ -h "$PRG" ]; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`/"$link" - fi -done - -PRGDIR=`dirname "$PRG"` -BASEDIR=`cd "$PRGDIR/../.." >/dev/null; pwd` - -# Reset the REPO variable. If you need to influence this use the environment setup file. -REPO= -[ -f "$BASEDIR"/bin/../conf/opencga-env.sh ] && . "$BASEDIR"/conf/opencga-env.sh - -# OS specific support. $var _must_ be set to either true or false. -cygwin=false; -darwin=false; -case "`uname`" in - CYGWIN*) cygwin=true ;; - Darwin*) darwin=true - if [ -z "$JAVA_VERSION" ] ; then - JAVA_VERSION="CurrentJDK" - else - echo "Using Java version: $JAVA_VERSION" - fi - if [ -z "$JAVA_HOME" ]; then - if [ -x "/usr/libexec/java_home" ]; then - JAVA_HOME=`/usr/libexec/java_home` - else - JAVA_HOME=/System/Library/Frameworks/JavaVM.framework/Versions/${JAVA_VERSION}/Home - fi - fi - ;; -esac - -if [ -z "$JAVA_HOME" ] ; then - if [ -r /etc/gentoo-release ] ; then - JAVA_HOME=`java-config --jre-home` - fi -fi - -# For Cygwin, ensure paths are in UNIX format before anything is touched -if $cygwin ; then - [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"` - [ -n "$CLASSPATH" ] && CLASSPATH=`cygpath --path --unix "$CLASSPATH"` -fi - -# If a specific java binary isn't specified search for the standard 'java' binary -if [ -z "$JAVACMD" ] ; then - if [ -n "$JAVA_HOME" ] ; then - if [ -x "$JAVA_HOME/jre/sh/java" ] ; then - # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" - else - JAVACMD="$JAVA_HOME/bin/java" - fi - else - JAVACMD=`which java` - fi -fi - -if [ ! -x "$JAVACMD" ] ; then - echo "Error: JAVA_HOME is not defined correctly." 1>&2 - echo " We cannot execute $JAVACMD" 1>&2 - exit 1 -fi - -if [ -z "$REPO" ] -then - REPO="$BASEDIR"/libs -fi - -HADOOP_WITH_DEPENDENCIES="$(ls $BASEDIR/opencga-storage-hadoop-core-*.jar):$(ls $BASEDIR/libs/log4j-slf4j-impl-*.jar)" -CLASSPATH=$HADOOP_WITH_DEPENDENCIES - -ENDORSED_DIR= -if [ -n "$ENDORSED_DIR" ] ; then - CLASSPATH=$BASEDIR/$ENDORSED_DIR/*:$CLASSPATH -fi - -if [ -n "$CLASSPATH_PREFIX" ] ; then - CLASSPATH=$CLASSPATH_PREFIX:$CLASSPATH -fi - -# For Cygwin, switch paths to Windows format before running java -if $cygwin; then - [ -n "$CLASSPATH" ] && CLASSPATH=`cygpath --path --windows "$CLASSPATH"` - [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` - [ -n "$HOME" ] && HOME=`cygpath --path --windows "$HOME"` - [ -n "$BASEDIR" ] && BASEDIR=`cygpath --path --windows "$BASEDIR"` - [ -n "$REPO" ] && REPO=`cygpath --path --windows "$REPO"` -fi - -exec "$JAVACMD" $JAVA_OPTS \ - -classpath "$CLASSPATH" \ - -Dapp.name="opencga-storage-hadoop.sh" \ - -Dapp.pid="$$" \ - -Dapp.repo="$REPO" \ - -Dapp.home="$BASEDIR" \ - -Dbasedir="$BASEDIR" \ - org.opencb.opencga.storage.hadoop.app.Main \ - "$@" - diff --git a/opencga-app/app/misc/solr/INSTALL.md b/opencga-app/app/misc/solr/INSTALL.md index ec696372690..8130f8fa9d3 100644 --- a/opencga-app/app/misc/solr/INSTALL.md +++ b/opencga-app/app/misc/solr/INSTALL.md @@ -13,10 +13,4 @@ In order to upload all of them, you need to execute the following commands: ``` $ ./bin/solr zk upconfig -n opencga-variant-configset-REPLACEME_OPENCGA_VERSION -d ~/opencga/build/misc/solr/opencga-variant-configset-REPLACEME_OPENCGA_VERSION -z localhost:9983 $ ./bin/solr zk upconfig -n opencga-rga-configset-REPLACEME_OPENCGA_VERSION -d ~/opencga/build/misc/solr/opencga-rga-configset-REPLACEME_OPENCGA_VERSION -z localhost:9983 -$ ./bin/solr zk upconfig -n opencga-cohort-configset-REPLACEME_OPENCGA_VERSION -d ~/opencga/build/misc/solr/opencga-cohort-configset-REPLACEME_OPENCGA_VERSION -z localhost:9983 -$ ./bin/solr zk upconfig -n opencga-family-configset-REPLACEME_OPENCGA_VERSION -d ~/opencga/build/misc/solr/opencga-family-configset-REPLACEME_OPENCGA_VERSION -z localhost:9983 -$ ./bin/solr zk upconfig -n opencga-file-configset-REPLACEME_OPENCGA_VERSION -d ~/opencga/build/misc/solr/opencga-file-configset-REPLACEME_OPENCGA_VERSION -z localhost:9983 -$ ./bin/solr zk upconfig -n opencga-individual-configset-REPLACEME_OPENCGA_VERSION -d ~/opencga/build/misc/solr/opencga-individual-configset-REPLACEME_OPENCGA_VERSION -z localhost:9983 -$ ./bin/solr zk upconfig -n opencga-sample-configset-REPLACEME_OPENCGA_VERSION -d ~/opencga/build/misc/solr/opencga-sample-configset-REPLACEME_OPENCGA_VERSION -z localhost:9983 - ``` \ No newline at end of file diff --git a/opencga-app/app/misc/solr/install.sh b/opencga-app/app/misc/solr/install.sh index 586d08fc7eb..e381a2660e2 100755 --- a/opencga-app/app/misc/solr/install.sh +++ b/opencga-app/app/misc/solr/install.sh @@ -15,10 +15,4 @@ cd ${OPENCGA_HOME}/misc/solr ${SOLR_HOME}/bin/solr zk upconfig -n opencga-variant-configset-${VERSION} -d ./opencga-variant-configset-${VERSION} -z ${ZK_HOST} ${SOLR_HOME}/bin/solr zk upconfig -n opencga-rga-configset-${VERSION} -d ./opencga-rga-configset-${VERSION} -z ${ZK_HOST} -${SOLR_HOME}/bin/solr zk upconfig -n opencga-rga-aux-configset-${VERSION} -d ./opencga-rga-aux-configset-${VERSION} -z ${ZK_HOST} -${SOLR_HOME}/bin/solr zk upconfig -n opencga-cohort-configset-${VERSION} -d ./opencga-cohort-configset-${VERSION} -z ${ZK_HOST} -${SOLR_HOME}/bin/solr zk upconfig -n opencga-family-configset-${VERSION} -d ./opencga-family-configset-${VERSION} -z ${ZK_HOST} -${SOLR_HOME}/bin/solr zk upconfig -n opencga-file-configset-${VERSION} -d ./opencga-file-configset-${VERSION} -z ${ZK_HOST} -${SOLR_HOME}/bin/solr zk upconfig -n opencga-individual-configset-${VERSION} -d ./opencga-individual-configset-${VERSION} -z ${ZK_HOST} -${SOLR_HOME}/bin/solr zk upconfig -n opencga-sample-configset-${VERSION} -d ./opencga-sample-configset-${VERSION} -z ${ZK_HOST} -${SOLR_HOME}/bin/solr zk upconfig -n opencga-job-configset-${VERSION} -d ./opencga-job-configset-${VERSION} -z ${ZK_HOST} \ No newline at end of file +${SOLR_HOME}/bin/solr zk upconfig -n opencga-rga-aux-configset-${VERSION} -d ./opencga-rga-aux-configset-${VERSION} -z ${ZK_HOST} \ No newline at end of file diff --git a/opencga-app/app/misc/solr/prepare_configsets.sh b/opencga-app/app/misc/solr/prepare_configsets.sh index 554dcc7c3d1..a4a26213cb1 100755 --- a/opencga-app/app/misc/solr/prepare_configsets.sh +++ b/opencga-app/app/misc/solr/prepare_configsets.sh @@ -1,5 +1,9 @@ #!/bin/bash +set -e +set -o pipefail +set -o nounset + if [ $# -eq 2 ]; then echo $0 $@ else @@ -19,7 +23,7 @@ sed -i "s/REPLACEME_OPENCGA_VERSION/${VERSION}/g" "${SOLR_DIR}/INSTALL.md" sed -i "s/REPLACEME_OPENCGA_VERSION/${VERSION}/g" "${SOLR_DIR}/install.sh" # Iterate over the different config sets -for name in variant rga rga-aux file sample individual family cohort job; do +for name in variant rga rga-aux; do CONFIG_SET_NAME="opencga-$name-configset-$VERSION" CONFIG_SET_DIR="$SOLR_DIR/$CONFIG_SET_NAME" diff --git a/opencga-app/pom.xml b/opencga-app/pom.xml index d0ab2466da5..b738ad32c0e 100644 --- a/opencga-app/pom.xml +++ b/opencga-app/pom.xml @@ -56,6 +56,10 @@ org.opencb.opencga opencga-master + + org.opencb.opencga + opencga-test + org.opencb.opencga opencga-client @@ -138,10 +142,6 @@ org.apache.commons commons-lang3 - - commons-lang - commons-lang - org.slf4j slf4j-api @@ -239,12 +239,43 @@ jline-terminal-jna runtime + + org.opencb.opencga opencga-storage-core test-jar test + + org.opencb.opencga + opencga-storage-hadoop-core + test + + + org.opencb.opencga + opencga-storage-hadoop-core + test + test-jar + + + org.mockito + mockito-core + test + + + org.opencb.opencga.hadoop.thirdparty + ${opencga-hadoop-shaded.artifactId} + ${opencga.hadoop.thirdparty.version} + test + test-jar + + + org.yaml + snakeyaml + + + @@ -254,14 +285,6 @@ src/main/java - - src/main/resources - true - ${project.basedir}/target/clients/R - - compileR.sh - - src/main/resources @@ -301,7 +324,7 @@ false ../conf/${opencga.env.file} - + opencga.sh org.opencb.opencga.app.cli.main.OpencgaMain @@ -343,7 +366,7 @@ Preparing ${build.dir} folder - + @@ -359,12 +382,12 @@ Preparing ${build.dir} folder - + - + @@ -376,7 +399,7 @@ Copying to ${build.dir} - + @@ -391,78 +414,78 @@ Copying docker-compose files - + - + + + + + + + + - + - Copying command lines and completion scripts to ${build.dir}/bin and - ${build.dir}/completion + Copying command lines and completion scripts to ${build.dir}/bin and ${build.dir}/completion - + - - - - - - - - + + - + Copying Python and R to ${build.dir}/clients - + - + - + - + - + Copying configuration files - + @@ -473,46 +496,28 @@ - + Copying war file - + - Copying Acceptance testing files - - - - - - - - - - - - - - - - + - - + - @@ -522,6 +527,21 @@ + + Create clicompletion + + + + + + + + + + + + + @@ -534,7 +554,8 @@ Preparing 'opencga-client' folder - + + @@ -542,7 +563,7 @@ Copying files to 'opencga-client' folder - + @@ -553,29 +574,29 @@ - + - + - + - + Creating 'opencga-client' tar.gz file - + @@ -584,7 +605,7 @@ Deleting 'opencga-client' folder - + @@ -611,6 +632,7 @@ * + true * @@ -648,7 +670,7 @@ Copying storage to ${build.dir} - + @@ -670,9 +692,8 @@ storage-hadoop - storage-hadoop + !skipStorageHadoop - true @@ -681,13 +702,37 @@ org.opencb.opencga - ${opencga-storage-hadoop-deps.artifactId} + ${opencga-hadoop-lib.artifactId} ${project.parent.version} - shaded + true + + org.codehaus.mojo + appassembler-maven-plugin + 1.10 + + + package + + assemble + + + + + + + opencga-storage-hadoop.sh + org.opencb.opencga.storage.hadoop.app.Main + + unix + + + + + maven-antrun-plugin 1.8 @@ -698,10 +743,93 @@ Copying hadoop jar with dependencies to ${build.dir} - - + + + + + + + + + + + + + Ensure that only one opencga-hadoop-shaded*.jar and opencga-storage-hadoop-compat*.jar libs are included + + + + + + + + + + + + run + + + + + + + + + opencga-test + + + !skipOpencgaTest + + + + + org.opencb.opencga + opencga-test + + + + + + org.codehaus.mojo + appassembler-maven-plugin + 1.10 + + + package + + assemble + + + + + + + opencga-test.sh + org.opencb.opencga.test.manager.TestGeneratorManager + + unix + + + + + + + maven-antrun-plugin + 1.8 + + + install-test-binaries + install + + + Copying Acceptance testing files + + + + @@ -739,12 +867,12 @@ Compiling R client - + + - Copying docker-compose files Moving R client to ${build.dir}/clients/R/ - + @@ -759,5 +887,58 @@ + + docker-build + + + docker + + + + + base + opencb + docker.io + + + + + maven-antrun-plugin + 1.8 + + + execute docker build + install + + run + + + + Execute docker + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/GeneralCliOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/GeneralCliOptions.java index 5dcde472d41..aca3974c6b6 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/GeneralCliOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/GeneralCliOptions.java @@ -100,7 +100,7 @@ public static class StudyOption { public static class StudyListOption { - @Parameter(names = {"-s", "--study"}, description = "Study list [[user@]project:]study where study and project can be either the id" + @Parameter(names = {"-s", "--study"}, description = "Study list [[organization@]project:]study where study and project can be either the id" + " or alias.", arity = 1) public String study; } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/AdminCliOptionsParser.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/AdminCliOptionsParser.java index 28787847b65..d9f06524289 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/AdminCliOptionsParser.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/AdminCliOptionsParser.java @@ -27,7 +27,6 @@ import org.opencb.opencga.app.cli.admin.options.MigrationCommandOptions; import org.opencb.opencga.app.cli.admin.options.StorageCommandOptions; import org.opencb.opencga.core.common.GitRepositoryState; -import org.opencb.opencga.core.models.user.Account; import java.util.List; @@ -63,14 +62,14 @@ public AdminCliOptionsParser() { catalogCommandOptions = new CatalogCommandOptions(); jCommander.addCommand("catalog", catalogCommandOptions); JCommander catalogSubCommands = jCommander.getCommands().get("catalog"); - catalogSubCommands.addCommand("demo", catalogCommandOptions.demoCatalogCommandOptions); +// catalogSubCommands.addCommand("demo", catalogCommandOptions.demoCatalogCommandOptions); catalogSubCommands.addCommand("install", catalogCommandOptions.installCatalogCommandOptions); catalogSubCommands.addCommand("status", catalogCommandOptions.statusCatalogCommandOptions); catalogSubCommands.addCommand("delete", catalogCommandOptions.deleteCatalogCommandOptions); catalogSubCommands.addCommand("index", catalogCommandOptions.indexCatalogCommandOptions); - catalogSubCommands.addCommand("clean", catalogCommandOptions.cleanCatalogCommandOptions); - catalogSubCommands.addCommand("stats", catalogCommandOptions.statsCatalogCommandOptions); - catalogSubCommands.addCommand("dump", catalogCommandOptions.dumpCatalogCommandOptions); +// catalogSubCommands.addCommand("clean", catalogCommandOptions.cleanCatalogCommandOptions); +// catalogSubCommands.addCommand("stats", catalogCommandOptions.statsCatalogCommandOptions); +// catalogSubCommands.addCommand("dump", catalogCommandOptions.dumpCatalogCommandOptions); catalogSubCommands.addCommand("export", catalogCommandOptions.exportCatalogCommandOptions); catalogSubCommands.addCommand("import", catalogCommandOptions.importCatalogCommandOptions); catalogSubCommands.addCommand("daemon", catalogCommandOptions.daemonCatalogCommandOptions); @@ -123,6 +122,7 @@ public AdminCliOptionsParser() { migrationSubCommands.addCommand("search", this.migrationCommandOptions.getSearchCommandOptions()); migrationSubCommands.addCommand("run", this.migrationCommandOptions.getRunCommandOptions()); migrationSubCommands.addCommand("run-manual", this.migrationCommandOptions.getRunManualCommandOptions()); + migrationSubCommands.addCommand("v3.0.0", this.migrationCommandOptions.getOrganizationCommandOptions()); this.storageCommandOptions = new StorageCommandOptions(jCommander, commonCommandOptions); this.jCommander.addCommand("storage", this.storageCommandOptions); @@ -198,14 +198,14 @@ public IgnorePasswordCommonCommandOptions(GeneralCliOptions.CommonCommandOptions @Parameters(commandNames = {"catalog"}, commandDescription = "Implements different tools interact with Catalog database") public class CatalogCommandOptions extends CommandOptions { - public final DemoCatalogCommandOptions demoCatalogCommandOptions; +// public final DemoCatalogCommandOptions demoCatalogCommandOptions; public final StatusCatalogCommandOptions statusCatalogCommandOptions; public final InstallCatalogCommandOptions installCatalogCommandOptions; public final DeleteCatalogCommandOptions deleteCatalogCommandOptions; public final IndexCatalogCommandOptions indexCatalogCommandOptions; - public final CleanCatalogCommandOptions cleanCatalogCommandOptions; - public final StatsCatalogCommandOptions statsCatalogCommandOptions; - public final DumpCatalogCommandOptions dumpCatalogCommandOptions; +// public final CleanCatalogCommandOptions cleanCatalogCommandOptions; +// public final StatsCatalogCommandOptions statsCatalogCommandOptions; +// public final DumpCatalogCommandOptions dumpCatalogCommandOptions; public final ExportCatalogCommandOptions exportCatalogCommandOptions; public final ImportCatalogCommandOptions importCatalogCommandOptions; public final DaemonCatalogCommandOptions daemonCatalogCommandOptions; @@ -213,14 +213,14 @@ public class CatalogCommandOptions extends CommandOptions { public AdminCommonCommandOptions commonOptions = AdminCliOptionsParser.this.commonCommandOptions; public CatalogCommandOptions() { - this.demoCatalogCommandOptions = new DemoCatalogCommandOptions(); +// this.demoCatalogCommandOptions = new DemoCatalogCommandOptions(); this.installCatalogCommandOptions = new InstallCatalogCommandOptions(); this.statusCatalogCommandOptions = new StatusCatalogCommandOptions(); this.deleteCatalogCommandOptions = new DeleteCatalogCommandOptions(); this.indexCatalogCommandOptions = new IndexCatalogCommandOptions(); - this.cleanCatalogCommandOptions = new CleanCatalogCommandOptions(); - this.statsCatalogCommandOptions = new StatsCatalogCommandOptions(); - this.dumpCatalogCommandOptions = new DumpCatalogCommandOptions(); +// this.cleanCatalogCommandOptions = new CleanCatalogCommandOptions(); +// this.statsCatalogCommandOptions = new StatsCatalogCommandOptions(); +// this.dumpCatalogCommandOptions = new DumpCatalogCommandOptions(); this.exportCatalogCommandOptions = new ExportCatalogCommandOptions(); this.importCatalogCommandOptions = new ImportCatalogCommandOptions(); this.daemonCatalogCommandOptions = new DaemonCatalogCommandOptions(); @@ -359,20 +359,20 @@ public static class CatalogDatabaseCommandOptions { * CATALOG SUB-COMMANDS */ - @Parameters(commandNames = {"demo"}, commandDescription = "Install and populate a catalog database with demonstration purposes.") - public class DemoCatalogCommandOptions { - - @ParametersDelegate - public AdminCommonCommandOptions commonOptions = AdminCliOptionsParser.this.commonCommandOptions; - - @Parameter(names = {"--database-prefix"}, description = "Prefix name for the catalog demo database. If not present, it will be " - + "set to 'demo'.") - public String prefix; - - @Parameter(names = {"--force"}, description = "If this parameters is set, it will override the database installation.") - public boolean force; - - } +// @Parameters(commandNames = {"demo"}, commandDescription = "Install and populate a catalog database with demonstration purposes.") +// public class DemoCatalogCommandOptions { +// +// @ParametersDelegate +// public AdminCommonCommandOptions commonOptions = AdminCliOptionsParser.this.commonCommandOptions; +// +// @Parameter(names = {"--database-prefix"}, description = "Prefix name for the catalog demo database. If not present, it will be " +// + "set to 'demo'.") +// public String prefix; +// +// @Parameter(names = {"--force"}, description = "If this parameters is set, it will override the database installation.") +// public boolean force; +// +// } @Parameters(commandNames = {"install"}, commandDescription = "Install Catalog database and collections together with the indexes") public class InstallCatalogCommandOptions extends CatalogDatabaseCommandOptions { @@ -383,9 +383,6 @@ public class InstallCatalogCommandOptions extends CatalogDatabaseCommandOptions @Parameter(names = {"--email"}, description = "Administrator e-mail", arity = 1) public String email; - @Parameter(names = {"--organization"}, description = "Administrator organization", arity = 1) - public String organization; - @Parameter(names = {"--secret-key"}, description = "Secret key needed to authenticate through OpenCGA (JWT).") public String secretKey; @@ -423,40 +420,43 @@ public class IndexCatalogCommandOptions extends CatalogDatabaseCommandOptions { @ParametersDelegate public AdminCommonCommandOptions commonOptions = AdminCliOptionsParser.this.commonCommandOptions; - @Parameter(names = {"--reset"}, description = "Remove existing indexes before creting the new one") - public boolean reset; - } - - @Parameters(commandNames = {"clean"}, commandDescription = "Query audit data from Catalog database") - public class CleanCatalogCommandOptions extends CatalogDatabaseCommandOptions { - - @ParametersDelegate - public AdminCommonCommandOptions commonOptions = AdminCliOptionsParser.this.commonCommandOptions; - - @Parameter(names = {"--collections"}, description = "A comma-separated list of collections to clean up") - public String filter = "ALL"; - - } - - @Parameters(commandNames = {"stats"}, commandDescription = "Print some summary stats of Catalog database") - public class StatsCatalogCommandOptions extends CatalogDatabaseCommandOptions { - - @ParametersDelegate - public AdminCommonCommandOptions commonOptions = AdminCliOptionsParser.this.commonCommandOptions; - - @Parameter(names = {"--collections"}, description = "A comma-separated list of collections for the stats") - public String collections = "ALL"; - } - - @Parameters(commandNames = {"dump"}, commandDescription = "Create a dump of Catalog database") - public class DumpCatalogCommandOptions extends CatalogDatabaseCommandOptions { - - @ParametersDelegate - public AdminCommonCommandOptions commonOptions = AdminCliOptionsParser.this.commonCommandOptions; - - @Parameter(names = {"--collections"}, description = "A comma-separated list of collections to be dumped", arity = 1) - public String collections = "ALL"; - } + @Parameter(names = {"--organization"}, description = "Organization id. If empty, will execute for all organizations.", arity = 1) + public String organizationId; + +// @Parameter(names = {"--reset"}, description = "Remove existing indexes before creating the new one") +// public boolean reset; + } + +// @Parameters(commandNames = {"clean"}, commandDescription = "Query audit data from Catalog database") +// public class CleanCatalogCommandOptions extends CatalogDatabaseCommandOptions { +// +// @ParametersDelegate +// public AdminCommonCommandOptions commonOptions = AdminCliOptionsParser.this.commonCommandOptions; +// +// @Parameter(names = {"--collections"}, description = "A comma-separated list of collections to clean up") +// public String filter = "ALL"; +// +// } + +// @Parameters(commandNames = {"stats"}, commandDescription = "Print some summary stats of Catalog database") +// public class StatsCatalogCommandOptions extends CatalogDatabaseCommandOptions { +// +// @ParametersDelegate +// public AdminCommonCommandOptions commonOptions = AdminCliOptionsParser.this.commonCommandOptions; +// +// @Parameter(names = {"--collections"}, description = "A comma-separated list of collections for the stats") +// public String collections = "ALL"; +// } + +// @Parameters(commandNames = {"dump"}, commandDescription = "Create a dump of Catalog database") +// public class DumpCatalogCommandOptions extends CatalogDatabaseCommandOptions { +// +// @ParametersDelegate +// public AdminCommonCommandOptions commonOptions = AdminCliOptionsParser.this.commonCommandOptions; +// +// @Parameter(names = {"--collections"}, description = "A comma-separated list of collections to be dumped", arity = 1) +// public String collections = "ALL"; +// } @Parameters(commandNames = {"export"}, commandDescription = "Export a project up to the specified release") public class ExportCatalogCommandOptions extends CatalogDatabaseCommandOptions { @@ -489,6 +489,9 @@ public class ImportCatalogCommandOptions extends CatalogDatabaseCommandOptions { @ParametersDelegate public AdminCommonCommandOptions commonOptions = AdminCliOptionsParser.this.commonCommandOptions; + @Parameter(names = {"--organization"}, description = "Organization id", arity = 1) + public String organizationId; + @Parameter(names = {"--directory"}, description = "Directory containing the files generated by the export command line.", arity = 1, required = true) public String directory; @@ -541,6 +544,9 @@ public class CreateUserCommandOptions extends CatalogDatabaseCommandOptions { @ParametersDelegate public AdminCommonCommandOptions commonOptions = AdminCliOptionsParser.this.commonCommandOptions; + @Parameter(names = {"--organization"}, description = "Organization id", arity = 1) + public String organizationId; + @Parameter(names = {"-u", "--id"}, description = "User id", required = true, arity = 1) public String userId; @@ -550,13 +556,10 @@ public class CreateUserCommandOptions extends CatalogDatabaseCommandOptions { @Parameter(names = {"--user-password"}, description = "User password", required = true, arity = 1) public String userPassword; - @Parameter(names = {"--type"}, description = "User account type of the user (guest or full).", arity = 1) - public Account.AccountType type = Account.AccountType.FULL; - @Parameter(names = {"--email"}, description = "User email", required = true, arity = 1) public String userEmail; - @Parameter(names = {"--organization"}, description = "User organization", required = false, arity = 1) + @Parameter(names = {"--user-organization"}, description = "User organization", required = false, arity = 1) public String userOrganization; @Parameter(names = {"--quota"}, description = "User disk quota", required = false, arity = 1) @@ -570,6 +573,9 @@ public class ImportCommandOptions extends CatalogDatabaseCommandOptions { @ParametersDelegate public AdminCommonCommandOptions commonOptions = AdminCliOptionsParser.this.commonCommandOptions; + @Parameter(names = {"--organization"}, description = "Organization id", arity = 1) + public String organizationId; + @Parameter(names = {"-u", "--user"}, description = DEPRECATED + "Use --id and --resource-type instead.", arity = 1) public String user; @@ -579,7 +585,7 @@ public class ImportCommandOptions extends CatalogDatabaseCommandOptions { @Parameter(names = {"--id"}, description = "Comma separated list of resource ids (users or applications) or single group id to be imported.", arity = 1) public String id; - @Parameter(names = {"-s", "--study"}, description = "Study [[user@]project:]study where the users or group will be associated to." + @Parameter(names = {"-s", "--study"}, description = "Study [[organization@]project:]study where the users or group will be associated to." + " Parameter --study-group is needed to perform this action.", arity = 1) public String study; @@ -591,15 +597,12 @@ public class ImportCommandOptions extends CatalogDatabaseCommandOptions { + " to be used to import users from.", arity = 1, required = true) public String authOrigin; - @Parameter(names = {"--type"}, description = "User account type of the users to be imported (guest or full).", arity = 1) - public String type = Account.AccountType.GUEST.name(); - @Parameter(names = {"--resource-type"}, description = "Resource to be imported. One of 'user', 'group' or 'application'", arity = 1) public String resourceType = "user"; - @Parameter(names = {"--expiration-date"}, description = "Expiration date (DD/MM/YYYY). By default, one year starting from the " - + "import day", arity = 1) - public String expDate; +// @Parameter(names = {"--expiration-date"}, description = "Expiration date (DD/MM/YYYY). By default, one year starting from the " +// + "import day", arity = 1) +// public String expDate; } @Parameters(commandNames = {"sync"}, commandDescription = "Sync a group of users from an authentication origin with a group in a study from catalog") @@ -608,13 +611,16 @@ public class SyncCommandOptions extends CatalogDatabaseCommandOptions { @ParametersDelegate public AdminCommonCommandOptions commonOptions = AdminCliOptionsParser.this.commonCommandOptions; + @Parameter(names = {"--organization"}, description = "Organization id", arity = 1) + public String organizationId; + @Parameter(names = {"--from"}, description = "Group defined in the authenticated origin to be synchronised", arity = 1) public String from; @Parameter(names = {"--to"}, description = "Group in a study that will be synchronised", arity = 1) public String to; - @Parameter(names = {"-s", "--study"}, description = "Study [[user@]project:]study where the list of users will be associated to.", + @Parameter(names = {"-s", "--study"}, description = "Study [[organization@]project:]study where the list of users will be associated to.", required = true, arity = 1) public String study; @@ -627,16 +633,13 @@ public class SyncCommandOptions extends CatalogDatabaseCommandOptions { + "is active.", arity = 0) public boolean syncAll; - @Parameter(names = {"--type"}, description = "User account type of the users to be imported (guest or full).", arity = 1) - public Account.AccountType type = Account.AccountType.GUEST; - @Parameter(names = {"--force"}, description = "Flag to force the synchronisation into groups that already exist and were not " + "previously synchronised.", arity = 0) public boolean force; - @Parameter(names = {"--expiration-date"}, description = "Expiration date (DD/MM/YYYY). By default, 1 year starting from the " - + "import day", arity = 1) - public String expDate; +// @Parameter(names = {"--expiration-date"}, description = "Expiration date (DD/MM/YYYY). By default, 1 year starting from the " +// + "import day", arity = 1) +// public String expDate; } @Parameters(commandNames = {"delete"}, commandDescription = "Delete the user Catalog database entry and the workspace") @@ -645,6 +648,8 @@ public class DeleteUserCommandOptions extends CatalogDatabaseCommandOptions { @ParametersDelegate public AdminCommonCommandOptions commonOptions = AdminCliOptionsParser.this.commonCommandOptions; + @Parameter(names = {"--organization"}, description = "Organization id", arity = 1) + public String organizationId; @Parameter(names = {"-u", "--user-id"}, description = "Full name of the study where the file is classified", required = true, arity = 1) public String userId; @@ -787,6 +792,9 @@ public class MetaKeyCommandOptions extends CatalogDatabaseCommandOptions { @ParametersDelegate public AdminCommonCommandOptions commonOptions = AdminCliOptionsParser.this.commonCommandOptions; + @Parameter(names = {"--organization"}, description = "Organization id", arity = 1) + public String organizationId; + @Parameter( names = {"--key"}, description = "Update secret key in OpenCGA", arity = 1) public String updateSecretKey; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/AdminCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/AdminCommandExecutor.java index 9dd3c922c14..f4106569f24 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/AdminCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/AdminCommandExecutor.java @@ -22,7 +22,6 @@ import org.opencb.opencga.app.cli.CommandExecutor; import org.opencb.opencga.app.cli.admin.AdminCliOptionsParser; import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.core.config.Admin; import java.util.Collections; @@ -82,9 +81,6 @@ protected void setCatalogDatabaseCredentials(String host, String prefix, String configuration.getCatalog().getDatabase().setUser(user); } - if (configuration.getAdmin() == null) { - configuration.setAdmin(new Admin()); - } if (StringUtils.isNotEmpty(password)) { configuration.getCatalog().getDatabase().setPassword(password); } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/CatalogCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/CatalogCommandExecutor.java index 40ece671445..090802f3741 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/CatalogCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/CatalogCommandExecutor.java @@ -21,15 +21,13 @@ import org.bson.Document; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.commons.datastore.mongodb.MongoDBCollection; +import org.opencb.commons.datastore.mongodb.MongoDataStore; import org.opencb.opencga.app.cli.admin.AdminCliOptionsParser; -import org.opencb.opencga.catalog.auth.authentication.JwtManager; import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; import org.opencb.opencga.catalog.db.mongodb.MongoDBUtils; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.CatalogManager; import org.opencb.opencga.core.common.JacksonUtils; -import org.opencb.opencga.core.common.PasswordUtils; import org.opencb.opencga.master.monitor.MonitorService; import javax.ws.rs.client.Client; @@ -80,7 +78,7 @@ public void execute() throws Exception { break; default: logger.error("Subcommand not valid"); - break; + break;//demo, dump , stats } } @@ -107,7 +105,7 @@ private void importDatabase() throws CatalogException, IOException { CatalogManager catalogManager = new CatalogManager(configuration); String token = catalogManager.getUserManager().loginAsAdmin(adminPassword).getToken(); - catalogManager.getProjectManager().importReleases(commandOptions.owner, commandOptions.directory, token); + catalogManager.getProjectManager().importReleases(commandOptions.organizationId, commandOptions.owner, commandOptions.directory, token); } private void status() throws CatalogException, JsonProcessingException { @@ -121,45 +119,45 @@ private void status() throws CatalogException, JsonProcessingException { ObjectMap result = new ObjectMap(); if (catalogManager.getDatabaseStatus()) { result.put("mongodbStatus", true); - result.put("catalogDBName", catalogManager.getCatalogDatabase()); + String adminDatabase = catalogManager.getCatalogAdminDatabase(); + result.put("catalogDBName", adminDatabase); + result.put("catalogDBNames", catalogManager.getCatalogDatabaseNames()); result.put("databasePrefix", catalogManager.getConfiguration().getDatabasePrefix()); if (commandOptions.uri) { // check login catalogManager.getUserManager().loginAsAdmin(getAdminPassword(true)); result.put("mongodbUri", MongoDBUtils.getMongoDBUri(configuration.getCatalog().getDatabase())); + result.put("mongodbUriRedacted", MongoDBUtils.getMongoDBUriRedacted(configuration.getCatalog().getDatabase())); result.put("mongodbUriWithDatabase", MongoDBUtils.getMongoDBUri( - configuration.getCatalog().getDatabase(), catalogManager.getCatalogDatabase())); + configuration.getCatalog().getDatabase(), adminDatabase)); result.put("mongodbCliOpts", MongoDBUtils.getMongoDBCliOpts(configuration.getCatalog().getDatabase())); result.put("mongodbCli", MongoDBUtils.getMongoDBCli( - configuration.getCatalog().getDatabase(), catalogManager.getCatalogDatabase())); + configuration.getCatalog().getDatabase(), adminDatabase)); } if (catalogManager.existsCatalogDB()) { result.put("installed", true); - - MongoDBAdaptorFactory factory = new MongoDBAdaptorFactory(configuration, catalogManager.getIoManagerFactory()); - MongoDBCollection metaCollection = factory.getMongoDBCollectionMap().get(MongoDBAdaptorFactory.METADATA_COLLECTION); - Document metaDocument = metaCollection.find(new Document(), QueryOptions.empty()).first(); - - result.put("creationDate", metaDocument.get("creationDate")); - result.put("version", metaDocument.get("version")); - - Object fullVersion = metaDocument.get("_fullVersion"); - int version = 20000; - int release = 4; - int lastJavaUpdate = 0; - int lastJsUpdate = 0; - if (fullVersion != null) { - version = ((Document) fullVersion).getInteger("version"); - release = ((Document) fullVersion).getInteger("release"); - lastJavaUpdate = ((Document) fullVersion).getInteger("lastJavaUpdate"); - lastJsUpdate = ((Document) fullVersion).getInteger("lastJsUpdate"); - } - result.put("versionNumeric", version); - result.put("release", release); - result.put("lastJsUpdate", lastJsUpdate); - result.put("lastJavaUpdate", lastJavaUpdate); } else { - result.put("installed", false); + String oldDatabase = configuration.getDatabasePrefix() + "_catalog"; + MongoDBAdaptorFactory mongoDBAdaptorFactory = new MongoDBAdaptorFactory(configuration, catalogManager.getIoManagerFactory()); + MongoDataStore oldDatastore = mongoDBAdaptorFactory.getMongoManager().get(oldDatabase, mongoDBAdaptorFactory.getMongoDbConfiguration()); + try { + if (oldDatastore.getCollectionNames().contains("metadata")) { + Document metadata = oldDatastore.getCollection("metadata").find(new Document(), QueryOptions.empty()).first(); + if (metadata != null) { + result.put("PENDING_3_0_0_MIGRATION", "Please, execute 'opencga-admin.sh migration v3.0.0' " + + "to update your database to the latest version."); +// result.put("oldVersion", metadata.get("version")); + + result.put("catalogDBName", oldDatabase); + result.put("catalogDBNames", Collections.singletonList(oldDatabase)); + + result.put("installed", true); + } + } + } catch (Exception e) { + logger.debug("Error checking old database. Assume doesn't exist", e); + } + result.putIfAbsent("installed", false); } } else { result.put("mongodbStatus", false); @@ -173,20 +171,13 @@ private void install() throws CatalogException { validateConfiguration(commandOptions); - this.configuration.getAdmin().setAlgorithm("HS256"); - - this.configuration.getAdmin().setSecretKey(commandOptions.secretKey); - if (StringUtils.isEmpty(configuration.getAdmin().getSecretKey())) { - configuration.getAdmin().setSecretKey(PasswordUtils.getStrongRandomPassword(JwtManager.SECRET_KEY_MIN_LENGTH)); - } - if (StringUtils.isEmpty(commandOptions.commonOptions.adminPassword)) { throw new CatalogException("No admin password found. Please, insert your password."); } try (CatalogManager catalogManager = new CatalogManager(configuration)) { - catalogManager.installCatalogDB(configuration.getAdmin().getSecretKey(), commandOptions.commonOptions.adminPassword, - commandOptions.email, commandOptions.organization, commandOptions.force); + catalogManager.installCatalogDB("HS256", commandOptions.secretKey, commandOptions.commonOptions.adminPassword, + commandOptions.email, commandOptions.force); } } @@ -194,8 +185,8 @@ private void delete() throws CatalogException { validateConfiguration(catalogCommandOptions.deleteCatalogCommandOptions); try (CatalogManager catalogManager = new CatalogManager(configuration)) { - logger.info("\nDeleting database {} from {}\n", catalogManager.getCatalogDatabase(), configuration.getCatalog().getDatabase() - .getHosts()); + logger.info("Deleting databases {} from {}\n", catalogManager.getCatalogDatabaseNames(), + configuration.getCatalog().getDatabase().getHosts()); catalogManager.deleteCatalogDB(catalogCommandOptions.deleteCatalogCommandOptions.commonOptions.adminPassword); } } @@ -207,10 +198,18 @@ private void index() throws CatalogException { String token = catalogManager.getUserManager() .loginAsAdmin(catalogCommandOptions.indexCatalogCommandOptions.commonOptions.adminPassword).getToken(); - logger.info("\nChecking and installing non-existing indexes in {} in {}\n", - catalogManager.getCatalogDatabase(), configuration.getCatalog().getDatabase().getHosts()); + String organizationId = catalogCommandOptions.indexCatalogCommandOptions.organizationId; + if (StringUtils.isEmpty(organizationId)) { + logger.info("Checking and installing non-existing indexes in {} fom {}\n", + catalogManager.getCatalogDatabaseNames(), configuration.getCatalog().getDatabase().getHosts()); - catalogManager.installIndexes(token); + catalogManager.installIndexes(token); + } else { + logger.info("Checking and installing non-existing indexes in {} fom {}\n", + catalogManager.getCatalogDatabase(organizationId), configuration.getCatalog().getDatabase().getHosts()); + + catalogManager.installIndexes(organizationId, token); + } } private void daemons() throws Exception { diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/MetaCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/MetaCommandExecutor.java index 1e9b66a9ac5..d138ee71c64 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/MetaCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/MetaCommandExecutor.java @@ -63,7 +63,7 @@ private void insertUpdatedAAdmin() throws CatalogException, IOException { params.putIfNotEmpty(MetaDBAdaptor.SECRET_KEY, executor.updateSecretKey); params.putIfNotEmpty(MetaDBAdaptor.ALGORITHM, executor.algorithm); - catalogManager.updateJWTParameters(params, token); + catalogManager.updateJWTParameters(executor.organizationId, params, token); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/MigrationCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/MigrationCommandExecutor.java index a360eaa2f1e..3f5f7cee80f 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/MigrationCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/MigrationCommandExecutor.java @@ -6,20 +6,23 @@ import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.app.cli.admin.options.MigrationCommandOptions; import org.opencb.opencga.app.cli.main.io.Table; +import org.opencb.opencga.app.migrations.v3.v3_0_0.OrganizationMigration; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.CatalogManager; import org.opencb.opencga.catalog.migration.Migration; import org.opencb.opencga.catalog.migration.MigrationManager; -import org.opencb.opencga.catalog.migration.MigrationRun; import org.opencb.opencga.catalog.migration.MigrationSummary; import org.opencb.opencga.core.common.GitRepositoryState; import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.models.migration.MigrationRun; import java.io.IOException; import java.nio.file.Paths; import java.time.temporal.ChronoUnit; +import java.util.Collections; import java.util.List; +import java.util.Map; /** * Created on 08/09/17. @@ -53,12 +56,25 @@ public void execute() throws Exception { case "run-manual": runManual(); break; + case "v3.0.0": + runMigrationToV3(); + break; default: logger.error("Subcommand '{}' not valid", subCommandString); break; } } + private void runMigrationToV3() throws Exception { + MigrationCommandOptions.OrganizationMigrationCommandOptions options = migrationCommandOptions.getOrganizationCommandOptions(); + setCatalogDatabaseCredentials(options, options.commonOptions); + + OrganizationMigration organizationMigration = new OrganizationMigration(configuration, options.commonOptions.adminPassword, + options.user); + organizationMigration.execute(); + } + + private void summary() throws Exception { MigrationCommandOptions.SummaryCommandOptions options = migrationCommandOptions.getSummaryCommandOptions(); setCatalogDatabaseCredentials(options, options.commonOptions); @@ -66,10 +82,10 @@ private void summary() throws Exception { try (CatalogManager catalogManager = new CatalogManager(configuration)) { String token = catalogManager.getUserManager().loginAsAdmin(options.commonOptions.adminPassword).getToken(); catalogManager.getMigrationManager().updateMigrationRuns(token); - MigrationSummary migrationSummary = catalogManager.getMigrationManager().getMigrationSummary(); - System.out.println(JacksonUtils.getDefaultObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(migrationSummary)); + Map migrationSummaryMap = catalogManager.getMigrationManager().getMigrationSummary(); + System.out.println(JacksonUtils.getDefaultObjectMapper().writerWithDefaultPrettyPrinter() + .writeValueAsString(migrationSummaryMap)); } - } private void search() throws Exception { @@ -79,44 +95,54 @@ private void search() throws Exception { try (CatalogManager catalogManager = new CatalogManager(configuration)) { String token = catalogManager.getUserManager().loginAsAdmin(options.commonOptions.adminPassword).getToken(); - List> rows = catalogManager.getMigrationManager() - .getMigrationRuns(options.version, options.domain, options.status, token); - - if (options.commonOptions.commonOptions.outputFormat.toLowerCase().contains("json")) { - for (Pair pair : rows) { - System.out.println(JacksonUtils.getDefaultObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(pair)); - } + List organizationIds; + if (StringUtils.isNotEmpty(options.organizationId)) { + organizationIds = Collections.singletonList(options.organizationId); } else { - Table> table = new Table>(Table.PrinterType.JANSI) - .addColumn("ID", p -> p.getKey().id(), 50) - .addColumn("Description", p -> p.getKey().description(), 50) - .addColumnEnum("Domain", p -> p.getKey().domain()) - .addColumn("Version", p -> p.getKey().version()) - .addColumnEnum("Language", p -> p.getKey().language()) - .addColumn("Manual", p -> Boolean.toString(p.getKey().manual())) - .addColumn("Offline", p -> Boolean.toString(p.getKey().offline())) - .addColumnNumber("Patch", p -> p.getKey().patch()) - .addColumn("Status", MigrationCommandExecutor::getMigrationStatus) - .addColumnNumber("RunPatch", p -> p.getValue().getPatch()) - .addColumn("ExecutionTime", p -> p.getValue().getStart() + " " + TimeUtils.durationToString(ChronoUnit.MILLIS.between( - p.getValue().getStart().toInstant(), - p.getValue().getEnd().toInstant()))) - .addColumn("Events", p -> { - StringBuilder v = new StringBuilder(); - if (StringUtils.isNotEmpty(p.getValue().getException())) { - v.append("Exception: ").append(p.getValue().getException()); - } - if (p.getValue().getEvents() != null) { - for (Event event : p.getValue().getEvents()) { - if (v.length() > 0) { - v.append("\n"); + organizationIds = catalogManager.getAdminManager().getOrganizationIds(token); + } + + for (String organizationId : organizationIds) { + System.out.println("Organization '" + organizationId + "'"); + List> rows = catalogManager.getMigrationManager() + .getMigrationRuns(organizationId, options.version, options.domain, options.status, token); + + if (options.commonOptions.commonOptions.outputFormat.toLowerCase().contains("json")) { + for (Pair pair : rows) { + System.out.println(JacksonUtils.getDefaultObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(pair)); + } + } else { + Table> table = new Table>(Table.PrinterType.JANSI) + .addColumn("ID", p -> p.getKey().id(), 50) + .addColumn("Description", p -> p.getKey().description(), 50) + .addColumnEnum("Domain", p -> p.getKey().domain()) + .addColumn("Version", p -> p.getKey().version()) + .addColumnEnum("Language", p -> p.getKey().language()) + .addColumn("Manual", p -> Boolean.toString(p.getKey().manual())) + .addColumn("Offline", p -> Boolean.toString(p.getKey().offline())) + .addColumnNumber("Patch", p -> p.getKey().patch()) + .addColumn("Status", MigrationCommandExecutor::getMigrationStatus) + .addColumnNumber("RunPatch", p -> p.getValue().getPatch()) + .addColumn("ExecutionTime", p -> p.getValue().getStart() + " " + TimeUtils.durationToString(ChronoUnit.MILLIS.between( + p.getValue().getStart().toInstant(), + p.getValue().getEnd().toInstant()))) + .addColumn("Events", p -> { + StringBuilder v = new StringBuilder(); + if (StringUtils.isNotEmpty(p.getValue().getException())) { + v.append("Exception: ").append(p.getValue().getException()); + } + if (p.getValue().getEvents() != null) { + for (Event event : p.getValue().getEvents()) { + if (v.length() > 0) { + v.append("\n"); + } + v.append(event.getType()).append(": ").append(event.getMessage()); } - v.append(event.getType()).append(": ").append(event.getMessage()); } - } - return v.toString(); - }); - table.printTable(rows); + return v.toString(); + }); + table.printTable(rows); + } } } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/StorageCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/StorageCommandExecutor.java index a03ddae1bfc..4bf8f695e45 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/StorageCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/StorageCommandExecutor.java @@ -16,7 +16,7 @@ package org.opencb.opencga.app.cli.admin.executors; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient; @@ -26,6 +26,8 @@ import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; import org.opencb.opencga.app.cli.admin.options.StorageCommandOptions; +import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.CatalogManager; import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.models.project.DataStore; @@ -72,12 +74,12 @@ private void status() throws Exception { try (CatalogManager catalogManager = new CatalogManager(configuration); VariantStorageManager variantStorageManager = new VariantStorageManager(catalogManager, factory)) { String adminPassword = getAdminPassword(true); -// token = catalogManager.getUserManager().loginAsAdmin(commandOptions.commonOptions.adminPassword).getToken(); token = catalogManager.getUserManager().loginAsAdmin(adminPassword).getToken(); ObjectMap status = new ObjectMap(); Collection liveNodes; - SolrClient solrClient = factory.getVariantStorageEngine(factory.getDefaultStorageEngineId(), "test_connection").getVariantSearchManager().getSolrManager().getSolrClient(); + SolrClient solrClient = factory.getVariantStorageEngine(factory.getDefaultStorageEngineId(), "test_connection") + .getVariantSearchManager().getSolrManager().getSolrClient(); if (solrClient instanceof CloudSolrClient) { liveNodes = ((CloudSolrClient) solrClient).getClusterStateProvider().getLiveNodes().stream() .map(s -> "http://" + s) @@ -89,8 +91,10 @@ private void status() throws Exception { liveNodes = storageConfiguration.getSearch().getHosts(); } boolean solrAlive; + Object collections = Collections.emptyList(); try { solrAlive = variantStorageManager.isSolrAvailable(); + collections = solrClient.request(new CollectionAdminRequest.List()).get("collections"); } catch (Exception e) { logger.warn("Solr not alive", e); solrAlive = false; @@ -98,11 +102,14 @@ private void status() throws Exception { ObjectMap solrStatus = new ObjectMap(); solrStatus.put("alive", solrAlive); solrStatus.put("live_nodes", liveNodes); - solrStatus.put("collections", solrClient.request(new CollectionAdminRequest.List()).get("collections")); + solrStatus.put("collections", collections); status.put("solr", solrStatus); + List organizationIds = parseOrganizationIds(catalogManager, commandOptions.organizationId); + logger.info("OrganizationIds: {}", organizationIds); List dataStores = new ArrayList<>(); - List variantStorageProjects = getVariantStorageProjects(catalogManager, variantStorageManager); + List variantStorageProjects = getVariantStorageProjects(organizationIds, catalogManager, variantStorageManager); + logger.info("Variant storage projects: {}", variantStorageProjects); for (String project : variantStorageProjects) { DataStore dataStore = variantStorageManager.getDataStoreByProjectId(project, token); ObjectMap map = new ObjectMap("project", project); @@ -138,11 +145,13 @@ private void updateDatabasePrefix() throws Exception { String adminPassword = getAdminPassword(true); token = catalogManager.getUserManager().loginAsAdmin(adminPassword).getToken(); Set variantStorageProjects = Collections.emptySet(); + List organizationIds = parseOrganizationIds(catalogManager, commandOptions.organizationId); + if (commandOptions.projectsWithoutStorage || commandOptions.projectsWithStorage) { - variantStorageProjects = new HashSet<>(getVariantStorageProjects(catalogManager, variantStorageManager)); + variantStorageProjects = new HashSet<>(getVariantStorageProjects(organizationIds, catalogManager, variantStorageManager)); } - for (Project project : catalogManager.getProjectManager().search(new Query(), new QueryOptions(), token).getResults()) { + for (Project project : getAllProjects(catalogManager, organizationIds)) { if (projects != null && !projects.contains(project.getFqn())) { logger.info("Skip project '{}'", project.getFqn()); continue; @@ -161,57 +170,71 @@ private void updateDatabasePrefix() throws Exception { continue; } } - final DataStore actualDataStore; + final DataStore currentDataStore; if (project.getInternal() != null && project.getInternal().getDatastores() != null) { - actualDataStore = project.getInternal().getDatastores().getVariant(); + currentDataStore = project.getInternal().getDatastores().getVariant(); } else { - actualDataStore = null; + currentDataStore = null; } if (commandOptions.projectsWithUndefinedDBName) { // Only accept projects with UNDEFINED dbname. so discard projects with DEFINED (without undefined) dbname - boolean undefinedDBName = actualDataStore == null || StringUtils.isEmpty(actualDataStore.getDbName()); + boolean undefinedDBName = currentDataStore == null || StringUtils.isEmpty(currentDataStore.getDbName()); if (!undefinedDBName) { logger.info("Skip project '{}' as its dbName is already defined. dbName : '{}'", project.getFqn(), - actualDataStore.getDbName()); + currentDataStore.getDbName()); continue; } } final DataStore defaultDataStore; if (StringUtils.isEmpty(commandOptions.dbPrefix)) { - defaultDataStore = VariantStorageManager.defaultDataStore(catalogManager, project, token); + defaultDataStore = VariantStorageManager.defaultDataStore(catalogManager, project); } else { - defaultDataStore = VariantStorageManager.defaultDataStore(catalogManager, project, commandOptions.dbPrefix, token); + defaultDataStore = VariantStorageManager.defaultDataStore(commandOptions.dbPrefix, project.getFqn()); } final DataStore newDataStore; logger.info("------"); - logger.info("Project " + project.getFqn()); - if (actualDataStore == null) { + logger.info("Project '" + project.getFqn() + "'"); + if (currentDataStore == null) { newDataStore = defaultDataStore; logger.info("Old DBName: null"); } else { - logger.info("Old DBName: " + actualDataStore.getDbName()); - actualDataStore.setDbName(defaultDataStore.getDbName()); - newDataStore = actualDataStore; + logger.info("Old DBName: " + currentDataStore.getDbName()); + currentDataStore.setDbName(defaultDataStore.getDbName()); + newDataStore = currentDataStore; } logger.info("New DBName: " + newDataStore.getDbName()); - catalogManager.getProjectManager().setDatastoreVariant(project.getUuid(), newDataStore, token); catalogManager.getProjectManager().setDatastoreVariant(project.getUuid(), defaultDataStore, token); + catalogManager.getProjectManager().setDatastoreVariant(project.getFqn(), newDataStore, token); } } } + private List parseOrganizationIds(CatalogManager catalogManager, String organizationId) throws CatalogException { + return StringUtils.isEmpty(organizationId) + ? catalogManager.getOrganizationManager().getOrganizationIds(token) + : Arrays.asList(organizationId.split(",")); + } + + private List getAllProjects(CatalogManager catalogManager, List organizationIds) throws CatalogException { + List projects = new ArrayList<>(); + for (String organizationId : organizationIds) { + projects.addAll(catalogManager.getProjectManager().search(organizationId, new Query(), new QueryOptions(), token).getResults()); + } + return projects; + } + /** * Get list of projects that exist at the VariantStorage. * @return List of projects * @throws Exception on error */ - protected final List getVariantStorageProjects(CatalogManager catalogManager, VariantStorageManager variantStorageManager) throws Exception { + protected final List getVariantStorageProjects(List organizationIds, CatalogManager catalogManager, VariantStorageManager variantStorageManager) throws Exception { Set projects = new LinkedHashSet<>(); - for (String studyFqn : getVariantStorageStudies(catalogManager, variantStorageManager)) { + for (String studyFqn : getVariantStorageStudies(organizationIds, catalogManager, variantStorageManager)) { projects.add(catalogManager.getStudyManager().getProjectFqn(studyFqn)); } @@ -223,13 +246,23 @@ protected final List getVariantStorageProjects(CatalogManager catalogMan * @return List of projects * @throws Exception on error */ - protected final List getVariantStorageStudies(CatalogManager catalogManager, VariantStorageManager variantStorageManager) throws Exception { + protected final List getVariantStorageStudies(List organizationIds, CatalogManager catalogManager, VariantStorageManager variantStorageManager) throws Exception { Set studies = new LinkedHashSet<>(); - for (Study study : catalogManager.getStudyManager().search(new Query(), new QueryOptions(QueryOptions.INCLUDE, - Arrays.asList("fqn")), token).getResults()) { - if (variantStorageManager.exists(study.getFqn(), token)) { - studies.add(study.getFqn()); + for (String organizationId : organizationIds) { + int studiesCount = 0; + for (Study study : catalogManager.getStudyManager().searchInOrganization(organizationId, new Query(), + new QueryOptions(QueryOptions.INCLUDE, + Arrays.asList(StudyDBAdaptor.QueryParams.FQN.key(), StudyDBAdaptor.QueryParams.ID.key())), + token).getResults()) { + studiesCount++; + if (variantStorageManager.exists(study.getFqn(), token)) { + logger.info("Study '{}' exists at the VariantStorage", study.getFqn()); + studies.add(study.getFqn()); + } else { + logger.info("Study '{}' does not exist at the VariantStorage", study.getFqn()); + } } + logger.info("Found {} studies in organization '{}'", studiesCount, organizationId); } return new ArrayList<>(studies); } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/UsersCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/UsersCommandExecutor.java index 1162fdb2dc6..fc700c5a6de 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/UsersCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/UsersCommandExecutor.java @@ -82,9 +82,9 @@ private void syncGroups() throws CatalogException { String token = catalogManager.getUserManager().loginAsAdmin(executor.commonOptions.adminPassword).getToken(); if (executor.syncAll) { - catalogManager.getUserManager().syncAllUsersOfExternalGroup(executor.study, executor.authOrigin, token); + catalogManager.getUserManager().syncAllUsersOfExternalGroup(executor.organizationId, executor.study, executor.authOrigin, token); } else { - catalogManager.getUserManager().importRemoteGroupOfUsers(executor.authOrigin, executor.from, executor.to, executor.study, + catalogManager.getUserManager().importRemoteGroupOfUsers(executor.organizationId, executor.authOrigin, executor.from, executor.to, executor.study, true, token); } } @@ -113,10 +113,10 @@ private void importUsersAndGroups() throws CatalogException { } if ("user".equalsIgnoreCase(executor.resourceType) || "application".equalsIgnoreCase(executor.resourceType)) { - catalogManager.getUserManager().importRemoteEntities(executor.authOrigin, Arrays.asList(executor.id.split(",")), + catalogManager.getUserManager().importRemoteEntities(executor.organizationId, executor.authOrigin, Arrays.asList(executor.id.split(",")), executor.resourceType.equalsIgnoreCase("application"), executor.studyGroup, executor.study, token); } else if ("group".equalsIgnoreCase(executor.resourceType)) { - catalogManager.getUserManager().importRemoteGroupOfUsers(executor.authOrigin, executor.id, executor.studyGroup, + catalogManager.getUserManager().importRemoteGroupOfUsers(executor.organizationId, executor.authOrigin, executor.id, executor.studyGroup, executor.study, false, token); } else { logger.error("Unknown resource type. Please use one of 'user', 'group' or 'application'"); @@ -155,7 +155,7 @@ private void create() throws CatalogException { User user; try { user = catalogManager.getUserManager().create(commandOptions.userId, commandOptions.userName, commandOptions.userEmail, - commandOptions.userPassword, commandOptions.userOrganization, userQuota, commandOptions.type, token).first(); + commandOptions.userPassword, commandOptions.userOrganization, userQuota, token).first(); } catch (CatalogException e) { if (e.getMessage().contains("already exists")) { logger.warn(e.getMessage()); @@ -179,7 +179,8 @@ private void delete() throws CatalogException, IOException { catalogManager.getUserManager().loginAsAdmin(usersCommandOptions.deleteUserCommandOptions.commonOptions.adminPassword); DataResult deletedUsers = catalogManager.getUserManager() - .delete(usersCommandOptions.deleteUserCommandOptions.userId, new QueryOptions("force", true), null); + .delete(usersCommandOptions.deleteUserCommandOptions.organizationId, + usersCommandOptions.deleteUserCommandOptions.userId, new QueryOptions("force", true), null); for (User user : deletedUsers.getResults()) { if (user != null) { System.out.println("The user has been successfully deleted from the database: " + user.toString()); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/migration/storage/AbstractStorageMigrator.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/migration/storage/AbstractStorageMigrator.java index 21a0be6c4a5..f7fcb7ee158 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/migration/storage/AbstractStorageMigrator.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/migration/storage/AbstractStorageMigrator.java @@ -1,27 +1,13 @@ package org.opencb.opencga.app.cli.admin.executors.migration.storage; -import org.opencb.commons.datastore.core.Query; -import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; -import org.opencb.opencga.catalog.db.api.ProjectDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.CatalogManager; import org.opencb.opencga.core.config.storage.StorageConfiguration; -import org.opencb.opencga.core.models.file.File; -import org.opencb.opencga.core.models.project.DataStore; -import org.opencb.opencga.core.models.project.Project; -import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.storage.core.StorageEngineFactory; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - /** * Created on 14/03/19. * @@ -38,40 +24,40 @@ public AbstractStorageMigrator(StorageConfiguration storageConfiguration, Catalo this.storageConfiguration = storageConfiguration; } - public void migrate(String sessionId) throws CatalogException, StorageEngineException { - - StorageEngineFactory storageEngineFactory = StorageEngineFactory.get(storageConfiguration); - - List projects = catalogManager.getProjectManager().search(new Query(), new QueryOptions( - QueryOptions.INCLUDE, Arrays.asList( - ProjectDBAdaptor.QueryParams.NAME.key(), - ProjectDBAdaptor.QueryParams.ID.key(), - ProjectDBAdaptor.QueryParams.FQN.key(), - ProjectDBAdaptor.QueryParams.ORGANISM.key(), - ProjectDBAdaptor.QueryParams.STUDY.key() - )), sessionId).getResults(); - - Set dataStores = new HashSet<>(); - for (Project project : projects) { - logger.info("Migrating project " + project.getName()); - - for (Study study : project.getStudies()) { - logger.info("Migrating study " + study.getName()); - - DataStore dataStore = VariantStorageManager.getDataStore(catalogManager, study.getFqn(), File.Bioformat.VARIANT, sessionId); - // Check only once per datastore - if (dataStores.add(dataStore)) { - VariantStorageEngine variantStorageEngine = - storageEngineFactory.getVariantStorageEngine(dataStore.getStorageEngine(), dataStore.getDbName()); - - migrate(variantStorageEngine, sessionId); - - } else { - logger.info("Nothing to migrate!"); - } - } - } - } +// public void migrate(String sessionId) throws CatalogException, StorageEngineException { +// +// StorageEngineFactory storageEngineFactory = StorageEngineFactory.get(storageConfiguration); +// +// List projects = catalogManager.getProjectManager().search(organizationId, new Query(), new QueryOptions( +// QueryOptions.INCLUDE, Arrays.asList( +// ProjectDBAdaptor.QueryParams.NAME.key(), +// ProjectDBAdaptor.QueryParams.ID.key(), +// ProjectDBAdaptor.QueryParams.FQN.key(), +// ProjectDBAdaptor.QueryParams.ORGANISM.key(), +// ProjectDBAdaptor.QueryParams.STUDY.key() +// )), sessionId).getResults(); +// +// Set dataStores = new HashSet<>(); +// for (Project project : projects) { +// logger.info("Migrating project " + project.getName()); +// +// for (Study study : project.getStudies()) { +// logger.info("Migrating study " + study.getName()); +// +// DataStore dataStore = VariantStorageManager.getDataStore(catalogManager, study.getFqn(), File.Bioformat.VARIANT, sessionId); +// // Check only once per datastore +// if (dataStores.add(dataStore)) { +// VariantStorageEngine variantStorageEngine = +// storageEngineFactory.getVariantStorageEngine(dataStore.getStorageEngine(), dataStore.getDbName()); +// +// migrate(variantStorageEngine, sessionId); +// +// } else { +// logger.info("Nothing to migrate!"); +// } +// } +// } +// } protected abstract void migrate(VariantStorageEngine variantStorageEngine, String sessionId) throws StorageEngineException, CatalogException; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/migration/v2_0_0/VariantStorage200MigrationTool.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/migration/v2_0_0/VariantStorage200MigrationTool.java index 39319cd310e..7ef509bbafc 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/migration/v2_0_0/VariantStorage200MigrationTool.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/migration/v2_0_0/VariantStorage200MigrationTool.java @@ -1,8 +1,6 @@ package org.opencb.opencga.app.cli.admin.executors.migration.v2_0_0; import org.opencb.opencga.analysis.tools.OpenCgaTool; -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.tools.annotations.Tool; import org.opencb.opencga.core.tools.annotations.ToolParams; @@ -13,7 +11,8 @@ resource = Enums.Resource.VARIANT, type = Tool.Type.OPERATION, scope = Tool.Scope.PROJECT, - description = VariantStorage200MigrationTool.DESCRIPTION) + description = VariantStorage200MigrationTool.DESCRIPTION, + priority = Enums.Priority.HIGH) public class VariantStorage200MigrationTool extends OpenCgaTool { public static final String ID = "variant-storage-migration-2.0.0"; @@ -26,10 +25,8 @@ public class VariantStorage200MigrationTool extends OpenCgaTool { protected void check() throws Exception { super.check(); - String userId = getCatalogManager().getUserManager().getUserId(getToken()); - if (!userId.equals(ParamConstants.OPENCGA_USER_ID)) { - throw new CatalogAuthenticationException("Only user '" + ParamConstants.OPENCGA_USER_ID + "' can run this operation!"); - } + getCatalogManager().getAuthorizationManager().checkIsOpencgaAdministrator( + catalogManager.getUserManager().validateToken(getToken())); } @Override diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/options/MigrationCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/options/MigrationCommandOptions.java index 074a104cc00..a306f222c38 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/options/MigrationCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/options/MigrationCommandOptions.java @@ -23,6 +23,7 @@ public class MigrationCommandOptions extends GeneralCliOptions { private final RunCommandOptions runCommandOptions; private final RunManualCommandOptions runManualCommandOptions; private final SummaryCommandOptions summaryCommandOptions; + private final OrganizationMigrationCommandOptions organizationCommandOptions; private final AdminCliOptionsParser.AdminCommonCommandOptions commonOptions; public MigrationCommandOptions(JCommander jCommander, AdminCliOptionsParser.AdminCommonCommandOptions commonOptions) { @@ -32,9 +33,21 @@ public MigrationCommandOptions(JCommander jCommander, AdminCliOptionsParser.Admi this.searchCommandOptions = new SearchCommandOptions(); this.runCommandOptions = new RunCommandOptions(); this.runManualCommandOptions = new RunManualCommandOptions(); + this.organizationCommandOptions = new OrganizationMigrationCommandOptions(); } + @Parameters(commandNames = {"v3.0.0"}, commandDescription = "Migrate to version 3.0.0") + public class OrganizationMigrationCommandOptions extends AdminCliOptionsParser.CatalogDatabaseCommandOptions { + + @ParametersDelegate + public AdminCliOptionsParser.AdminCommonCommandOptions commonOptions = MigrationCommandOptions.this.commonOptions; + + @Parameter(names = {"--user"}, description = "User whose data is going to be migrated. If more than one user of type FULL contains" + + " projects and studies, only the one provided will keep the data and will be fully migrated.") + public String user; + } + @Parameters(commandNames = {"summary"}, commandDescription = "Obtain migrations status summary") public class SummaryCommandOptions extends AdminCliOptionsParser.CatalogDatabaseCommandOptions { @@ -58,6 +71,9 @@ public class SearchCommandOptions extends AdminCliOptionsParser.CatalogDatabaseC @ParametersDelegate public AdminCliOptionsParser.AdminCommonCommandOptions commonOptions = MigrationCommandOptions.this.commonOptions; + @Parameter(names = {"--organization"}, description = "Organization id") + public String organizationId; + @Parameter(names = {"--status"}, description = "Filter migrations by status. PENDING, ERROR, DONE") public List status; @@ -128,6 +144,10 @@ public RunManualCommandOptions getRunManualCommandOptions() { return runManualCommandOptions; } + public OrganizationMigrationCommandOptions getOrganizationCommandOptions() { + return organizationCommandOptions; + } + public AdminCliOptionsParser.AdminCommonCommandOptions getCommonOptions() { return commonOptions; } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/options/StorageCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/options/StorageCommandOptions.java index 3d71214919f..504ead2bfd5 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/options/StorageCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/options/StorageCommandOptions.java @@ -35,12 +35,18 @@ public class StatusCommandOptions extends AdminCliOptionsParser.CatalogDatabaseC public AdminCliOptionsParser.IgnorePasswordCommonCommandOptions commonOptions = new AdminCliOptionsParser.IgnorePasswordCommonCommandOptions(getCommonOptions().commonOptions); + @Parameter(names = {"--organization"}, description = "Organization id. If empty, will run for all organizations.", arity = 1) + public String organizationId; + } @Parameters(commandNames = {"update-database-prefix"}, commandDescription = "Update database prefix of the variant storage stored in " + "catalog. Does not modify the actual database names, just the database names stored in catalog.") public class UpdateDatabasePrefix extends AdminCliOptionsParser.CatalogDatabaseCommandOptions { + @Parameter(names = {"--organization"}, description = "Organization id. If empty, will run for all organizations.", arity = 1) + public String organizationId; + @Parameter(names = {"--projects"}, description = "List of projects") public String projects; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/InternalCliOptionsParser.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/InternalCliOptionsParser.java index 0079461d25a..96522720e12 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/InternalCliOptionsParser.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/InternalCliOptionsParser.java @@ -35,6 +35,7 @@ import static org.opencb.opencga.app.cli.internal.options.AlignmentCommandOptions.SamtoolsCommandOptions.SAMTOOLS_RUN_COMMAND; import static org.opencb.opencga.app.cli.internal.options.ClinicalCommandOptions.CancerTieringCommandOptions.CANCER_TIERING_INTERPRETATION_RUN_COMMAND; import static org.opencb.opencga.app.cli.internal.options.ClinicalCommandOptions.ExomiserInterpretationCommandOptions.EXOMISER_INTERPRETATION_RUN_COMMAND; +import static org.opencb.opencga.app.cli.internal.options.ClinicalCommandOptions.ImportClinicalAnalysesCommandOptions.IMPORT_COMMAND; import static org.opencb.opencga.app.cli.internal.options.ClinicalCommandOptions.RgaAuxiliarSecondaryIndexCommandOptions.RGA_AUX_INDEX_RUN_COMMAND; import static org.opencb.opencga.app.cli.internal.options.ClinicalCommandOptions.RgaSecondaryIndexCommandOptions.RGA_INDEX_RUN_COMMAND; import static org.opencb.opencga.app.cli.internal.options.ClinicalCommandOptions.TeamCommandOptions.TEAM_INTERPRETATION_RUN_COMMAND; @@ -226,6 +227,7 @@ public InternalCliOptionsParser() { clinicalSubCommands.addCommand(RGA_INDEX_RUN_COMMAND, clinicalCommandOptions.rgaSecondaryIndexCommandOptions); clinicalSubCommands.addCommand(RGA_AUX_INDEX_RUN_COMMAND, clinicalCommandOptions.rgaAuxiliarSecondaryIndexCommandOptions); clinicalSubCommands.addCommand(EXOMISER_INTERPRETATION_RUN_COMMAND, clinicalCommandOptions.exomiserInterpretationCommandOptions); + clinicalSubCommands.addCommand(IMPORT_COMMAND, clinicalCommandOptions.importClinicalAnalysesCommandOptions); clinicalSubCommands.addCommand("tsv-load", clinicalCommandOptions.tsvLoad); fileCommandOptions = new FileCommandOptions(commonCommandOptions, jCommander); @@ -235,37 +237,37 @@ public InternalCliOptionsParser() { fileSubCommands.addCommand("unlink", fileCommandOptions.unlinkCommandOptions); fileSubCommands.addCommand("fetch", fileCommandOptions.fetchCommandOptions); fileSubCommands.addCommand("postlink", fileCommandOptions.postlinkCommandOptions); - fileSubCommands.addCommand("secondary-index", fileCommandOptions.secondaryIndex); +// fileSubCommands.addCommand("secondary-index", fileCommandOptions.secondaryIndex); fileSubCommands.addCommand("tsv-load", fileCommandOptions.tsvLoad); sampleCommandOptions = new SampleCommandOptions(commonCommandOptions, jCommander); jCommander.addCommand("samples", sampleCommandOptions); JCommander sampleSubCommands = jCommander.getCommands().get("samples"); - sampleSubCommands.addCommand("secondary-index", sampleCommandOptions.secondaryIndex); +// sampleSubCommands.addCommand("secondary-index", sampleCommandOptions.secondaryIndex); sampleSubCommands.addCommand("tsv-load", sampleCommandOptions.tsvLoad); individualCommandOptions = new IndividualCommandOptions(commonCommandOptions, jCommander); jCommander.addCommand("individuals", individualCommandOptions); JCommander individualSubCommands = jCommander.getCommands().get("individuals"); - individualSubCommands.addCommand("secondary-index", individualCommandOptions.secondaryIndex); +// individualSubCommands.addCommand("secondary-index", individualCommandOptions.secondaryIndex); individualSubCommands.addCommand("tsv-load", individualCommandOptions.tsvLoad); cohortCommandOptions = new CohortCommandOptions(commonCommandOptions, jCommander); jCommander.addCommand("cohorts", cohortCommandOptions); JCommander cohortSubCommands = jCommander.getCommands().get("cohorts"); - cohortSubCommands.addCommand("secondary-index", cohortCommandOptions.secondaryIndex); +// cohortSubCommands.addCommand("secondary-index", cohortCommandOptions.secondaryIndex); cohortSubCommands.addCommand("tsv-load", cohortCommandOptions.tsvLoad); familyCommandOptions = new FamilyCommandOptions(commonCommandOptions, jCommander); jCommander.addCommand("families", familyCommandOptions); JCommander familySubCommands = jCommander.getCommands().get("families"); - familySubCommands.addCommand("secondary-index", familyCommandOptions.secondaryIndex); +// familySubCommands.addCommand("secondary-index", familyCommandOptions.secondaryIndex); familySubCommands.addCommand("tsv-load", familyCommandOptions.tsvLoad); - jobCommandOptions = new JobCommandOptions(commonCommandOptions, jCommander); - jCommander.addCommand("jobs", jobCommandOptions); - JCommander jobSubCommands = jCommander.getCommands().get("jobs"); - jobSubCommands.addCommand("secondary-index", jobCommandOptions.secondaryIndex); +// jobCommandOptions = new JobCommandOptions(commonCommandOptions, jCommander); +// jCommander.addCommand("jobs", jobCommandOptions); +// JCommander jobSubCommands = jCommander.getCommands().get("jobs"); +// jobSubCommands.addCommand("secondary-index", jobCommandOptions.secondaryIndex); panelInternalCommandOptions = new DiseasePanelInternalCommandOptions(commonCommandOptions, jCommander); jCommander.addCommand("panels", panelInternalCommandOptions); @@ -295,6 +297,9 @@ public static class JobOptions { @Parameter(names = {"--job"}, description = "Job id executing the command line", arity = 1) public String jobId; + @Parameter(names = {"--dry-run"}, description = "Dry run mode execution", arity = 0) + public boolean dryRun; + @Parameter(names = {"--job-id"}, hidden = true, description = "Deprecated, use --job", arity = 1) @Deprecated public void setJobId(String job) { diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/AlignmentCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/AlignmentCommandExecutor.java index 436fff7b061..699b415da04 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/AlignmentCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/AlignmentCommandExecutor.java @@ -19,7 +19,6 @@ import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.analysis.alignment.AlignmentCoverageAnalysis; import org.opencb.opencga.analysis.alignment.AlignmentIndexOperation; -import org.opencb.opencga.analysis.alignment.AlignmentStorageManager; import org.opencb.opencga.analysis.alignment.qc.AlignmentGeneCoverageStatsAnalysis; import org.opencb.opencga.analysis.alignment.qc.AlignmentQcAnalysis; import org.opencb.opencga.analysis.wrappers.bwa.BwaWrapperAnalysis; @@ -51,6 +50,7 @@ public class AlignmentCommandExecutor extends InternalCommandExecutor { private AlignmentCommandOptions alignmentCommandOptions; private String jobId; + private boolean dryRun; // private AlignmentStorageEngine alignmentStorageManager; public AlignmentCommandExecutor(AlignmentCommandOptions options) { @@ -66,6 +66,7 @@ public void execute() throws Exception { configure(); jobId = alignmentCommandOptions.internalJobOptions.jobId; + dryRun = alignmentCommandOptions.internalJobOptions.dryRun; switch (subCommandString) { case "index-run": @@ -125,7 +126,7 @@ private void indexRun() throws Exception { cliOptions.overwrite ).toObjectMap(cliOptions.commonOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(AlignmentIndexOperation.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(AlignmentIndexOperation.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } @@ -166,7 +167,7 @@ private void qcRun() throws ToolException { ).toObjectMap(cliOptions.commonOptions.params) .append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(AlignmentQcAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(AlignmentQcAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void geneCoverageStatsRun() throws ToolException { @@ -180,7 +181,7 @@ private void geneCoverageStatsRun() throws ToolException { ).toObjectMap(cliOptions.commonOptions.params) .append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(AlignmentGeneCoverageStatsAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(AlignmentGeneCoverageStatsAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } // private void statsRun() throws ToolException { @@ -243,7 +244,7 @@ private void coverageRun() throws ToolException { cliOptions.windowSize ).toObjectMap(cliOptions.commonOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(AlignmentCoverageAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(AlignmentCoverageAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void delete() { @@ -268,7 +269,7 @@ private void bwa() throws Exception { cliOptions.bwaParams) .toObjectMap(cliOptions.commonOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(BwaWrapperAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(BwaWrapperAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } // Samtools @@ -283,7 +284,7 @@ private void samtools() throws Exception { cliOptions.samtoolsParams) .toObjectMap(cliOptions.commonOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(SamtoolsWrapperAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(SamtoolsWrapperAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } // Deeptools @@ -297,7 +298,7 @@ private void deeptools() throws Exception { cliOptions.deeptoolsParams) .toObjectMap(cliOptions.commonOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(DeeptoolsWrapperAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(DeeptoolsWrapperAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } // FastQC @@ -311,7 +312,7 @@ private void fastqc() throws Exception { cliOptions.fastqcParams) .toObjectMap(cliOptions.commonOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(FastqcWrapperAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(FastqcWrapperAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } // Picard @@ -325,7 +326,7 @@ private void picard() throws Exception { cliOptions.picardParams) .toObjectMap(cliOptions.commonOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(PicardWrapperAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(PicardWrapperAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } //------------------------------------------------------------------------- diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/ClinicalCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/ClinicalCommandExecutor.java index 14256ac1652..4800ec88c19 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/ClinicalCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/ClinicalCommandExecutor.java @@ -40,7 +40,7 @@ import org.opencb.opencga.app.cli.internal.options.ClinicalCommandOptions; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.exceptions.ToolException; -import org.opencb.opencga.core.models.clinical.RgaAnalysisParams; +import org.opencb.opencga.core.models.clinical.*; import java.io.FileInputStream; import java.io.IOException; @@ -97,6 +97,9 @@ public void execute() throws Exception { case EXOMISER_INTERPRETATION_RUN_COMMAND: exomiserInterpretation(); break; + case ClinicalCommandOptions.ImportClinicalAnalysesCommandOptions.IMPORT_COMMAND: + //importClinicalAnalyses(); + break; case "tsv-load": tsvLoad(); break; @@ -112,7 +115,7 @@ private void rgaIndex() throws ToolException { ObjectMap params = new RgaAnalysisParams(options.file) .toObjectMap(options.commonOptions.params) .append(ParamConstants.STUDY_PARAM, options.study); - toolRunner.execute(RgaAnalysis.class, params, outDir, options.jobOptions.jobId, options.commonOptions.token); + toolRunner.execute(RgaAnalysis.class, params, outDir, options.jobOptions.jobId, options.jobOptions.dryRun, options.commonOptions.token); } private void auxRgaIndex() throws ToolException { @@ -122,7 +125,7 @@ private void auxRgaIndex() throws ToolException { ObjectMap params = new ObjectMap() .appendAll(options.commonOptions.params) .append(ParamConstants.STUDY_PARAM, options.study); - toolRunner.execute(AuxiliarRgaAnalysis.class, params, outDir, options.jobOptions.jobId, options.commonOptions.token); + toolRunner.execute(AuxiliarRgaAnalysis.class, params, outDir, options.jobOptions.jobId, options.jobOptions.dryRun, options.commonOptions.token); } private void tiering() throws Exception { @@ -314,7 +317,8 @@ private void exomiserInterpretation() throws Exception { ExomiserInterpretationAnalysis exomiserInterpretationAnalysis = new ExomiserInterpretationAnalysis(); exomiserInterpretationAnalysis.setUp(opencgaHome.toString(), new ObjectMap(), outDir, token); exomiserInterpretationAnalysis.setStudyId(cliOptions.study) - .setClinicalAnalysisId(cliOptions.clinicalAnalysis); + .setClinicalAnalysisId(cliOptions.clinicalAnalysis) + .setExomiserVersion(cliOptions.exomiserVersion); // exomiserInterpretationAnalysis.setPrimary(cliOptions.primary); exomiserInterpretationAnalysis.start(); } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/CohortCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/CohortCommandExecutor.java index d49650625b6..9ab3797e455 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/CohortCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/CohortCommandExecutor.java @@ -1,7 +1,6 @@ package org.opencb.opencga.app.cli.internal.executors; import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.opencga.analysis.cohort.CohortIndexTask; import org.opencb.opencga.analysis.cohort.CohortTsvAnnotationLoader; import org.opencb.opencga.app.cli.internal.options.CohortCommandOptions; import org.opencb.opencga.core.exceptions.ToolException; @@ -25,9 +24,6 @@ public void execute() throws Exception { String subCommandString = getParsedSubCommand(cohortCommandOptions.jCommander); configure(); switch (subCommandString) { - case "secondary-index": - secondaryIndex(); - break; case "tsv-load": tsvLoad(); break; @@ -53,10 +49,4 @@ private void tsvLoad() throws ToolException { annotationLoader.start(); } - private void secondaryIndex() throws ToolException { - CohortCommandOptions.SecondaryIndex options = cohortCommandOptions.secondaryIndex; - Path outDir = Paths.get(options.outDir); - toolRunner.execute(CohortIndexTask.class, new ObjectMap(), outDir, options.jobOptions.jobId, options.commonOptions.token); - } - } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/DiseasePanelInternalCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/DiseasePanelInternalCommandExecutor.java index 2529baaee1e..aa39f3d0919 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/DiseasePanelInternalCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/DiseasePanelInternalCommandExecutor.java @@ -45,7 +45,8 @@ private void importPanels() throws ToolException { .toObjectMap(options.commonOptions.params) .append(ParamConstants.STUDY_PARAM, options.studyId); - toolRunner.execute(PanelImportTask.class, params, Paths.get(options.outDir), jobId, token); + toolRunner.execute(PanelImportTask.class, params, Paths.get(options.outDir), jobId, + diseasePanelCommandOptions.internalJobOptions.dryRun, token); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/FamilyCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/FamilyCommandExecutor.java index fa65aa26823..916d0c44094 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/FamilyCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/FamilyCommandExecutor.java @@ -1,7 +1,6 @@ package org.opencb.opencga.app.cli.internal.executors; import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.opencga.analysis.family.FamilyIndexTask; import org.opencb.opencga.analysis.family.FamilyTsvAnnotationLoader; import org.opencb.opencga.app.cli.internal.options.FamilyCommandOptions; import org.opencb.opencga.core.exceptions.ToolException; @@ -25,9 +24,6 @@ public void execute() throws Exception { String subCommandString = getParsedSubCommand(familyCommandOptions.jCommander); configure(); switch (subCommandString) { - case "secondary-index": - secondaryIndex(); - break; case "tsv-load": tsvLoad(); break; @@ -38,12 +34,6 @@ public void execute() throws Exception { } } - private void secondaryIndex() throws ToolException { - FamilyCommandOptions.SecondaryIndex options = familyCommandOptions.secondaryIndex; - Path outDir = Paths.get(options.outDir); - toolRunner.execute(FamilyIndexTask.class, new ObjectMap(), outDir, options.jobOptions.jobId, options.commonOptions.token); - } - private void tsvLoad() throws ToolException { FamilyCommandOptions.TsvLoad options = familyCommandOptions.tsvLoad; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/FileCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/FileCommandExecutor.java index e03cec10594..04bafdfaada 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/FileCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/FileCommandExecutor.java @@ -40,9 +40,6 @@ public void execute() throws Exception { case "postlink": postlink(); break; - case "secondary-index": - secondaryIndex(); - break; case "tsv-load": tsvLoad(); break; @@ -93,7 +90,8 @@ private void postlink() throws ToolException { .toObjectMap(options.commonOptions.params) .append(ParamConstants.STUDY_PARAM, options.studyId); - toolRunner.execute(PostLinkSampleAssociation.class, params, outDir, fileCommandOptions.internalJobOptions.jobId, token); + toolRunner.execute(PostLinkSampleAssociation.class, params, outDir, fileCommandOptions.internalJobOptions.jobId, + fileCommandOptions.internalJobOptions.dryRun, token); } private void fetch() throws ToolException { @@ -101,13 +99,7 @@ private void fetch() throws ToolException { Path outDir = Paths.get(options.outDir); - toolRunner.execute(FetchAndRegisterTask.class, new FileFetch(options.url, options.path), outDir, null, options.commonOptions.token); - } - - private void secondaryIndex() throws ToolException { - FileCommandOptions.SecondaryIndex options = fileCommandOptions.secondaryIndex; - Path outDir = Paths.get(options.outDir); - toolRunner.execute(FileIndexTask.class, new ObjectMap(), outDir, options.jobOptions.jobId, options.commonOptions.token); + toolRunner.execute(FetchAndRegisterTask.class, new FileFetch(options.url, options.path), outDir, null, false, options.commonOptions.token); } private void tsvLoad() throws ToolException { diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/IndividualCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/IndividualCommandExecutor.java index b0b510e1b78..d59bdff28b9 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/IndividualCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/IndividualCommandExecutor.java @@ -1,7 +1,6 @@ package org.opencb.opencga.app.cli.internal.executors; import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.opencga.analysis.individual.IndividualIndexTask; import org.opencb.opencga.analysis.individual.IndividualTsvAnnotationLoader; import org.opencb.opencga.app.cli.internal.options.IndividualCommandOptions; import org.opencb.opencga.core.exceptions.ToolException; @@ -25,9 +24,6 @@ public void execute() throws Exception { String subCommandString = getParsedSubCommand(individualCommandOptions.jCommander); configure(); switch (subCommandString) { - case "secondary-index": - secondaryIndex(); - break; case "tsv-load": tsvLoad(); break; @@ -38,12 +34,6 @@ public void execute() throws Exception { } } - private void secondaryIndex() throws ToolException { - IndividualCommandOptions.SecondaryIndex options = individualCommandOptions.secondaryIndex; - Path outDir = Paths.get(options.outDir); - toolRunner.execute(IndividualIndexTask.class, new ObjectMap(), outDir, options.jobOptions.jobId, options.commonOptions.token); - } - private void tsvLoad() throws ToolException { IndividualCommandOptions.TsvLoad options = individualCommandOptions.tsvLoad; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/InternalCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/InternalCommandExecutor.java index bffc809a1da..6f00be2dfee 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/InternalCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/InternalCommandExecutor.java @@ -16,19 +16,13 @@ package org.opencb.opencga.app.cli.internal.executors; -import org.opencb.commons.datastore.core.Query; -import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.analysis.tools.ToolRunner; import org.opencb.opencga.app.cli.CommandExecutor; import org.opencb.opencga.app.cli.GeneralCliOptions; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.CatalogManager; -import org.opencb.opencga.core.models.study.Study; import org.opencb.opencga.storage.core.StorageEngineFactory; -import java.util.Map; -import java.util.stream.Collectors; - /** * Created on 03/05/16 * @@ -52,15 +46,7 @@ protected void configure() throws IllegalAccessException, ClassNotFoundException // Creating StorageManagerFactory storageEngineFactory = StorageEngineFactory.get(storageConfiguration); - toolRunner = new ToolRunner(appHome, catalogManager, storageEngineFactory); - } - - protected Map getStudyIds(String sessionId) throws CatalogException { - return catalogManager.getStudyManager().search(new Query(), new QueryOptions("include", "projects.studies.id,projects.studies" + - ".alias"), sessionId) - .getResults() - .stream() - .collect(Collectors.toMap(Study::getUid, Study::getId)); + toolRunner = new ToolRunner(appHome, catalogManager, storageEngineFactory, configuration); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/JobCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/JobCommandExecutor.java index 3c14cb3a3ae..50109b63747 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/JobCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/JobCommandExecutor.java @@ -1,12 +1,6 @@ package org.opencb.opencga.app.cli.internal.executors; -import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.opencga.analysis.job.JobIndexTask; import org.opencb.opencga.app.cli.internal.options.JobCommandOptions; -import org.opencb.opencga.core.exceptions.ToolException; - -import java.nio.file.Path; -import java.nio.file.Paths; public class JobCommandExecutor extends InternalCommandExecutor { @@ -24,9 +18,6 @@ public void execute() throws Exception { String subCommandString = getParsedSubCommand(jobCommandOptions.jCommander); configure(); switch (subCommandString) { - case "secondary-index": - secondaryIndex(); - break; default: logger.error("Subcommand not valid"); break; @@ -34,10 +25,4 @@ public void execute() throws Exception { } } - private void secondaryIndex() throws ToolException { - JobCommandOptions.SecondaryIndex options = jobCommandOptions.secondaryIndex; - Path outDir = Paths.get(options.outDir); - toolRunner.execute(JobIndexTask.class, new ObjectMap(), outDir, options.jobOptions.jobId, options.commonOptions.token); - } - } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/SampleCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/SampleCommandExecutor.java index 21055921e11..92278fcb06c 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/SampleCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/SampleCommandExecutor.java @@ -1,7 +1,6 @@ package org.opencb.opencga.app.cli.internal.executors; import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.opencga.analysis.sample.SampleIndexTask; import org.opencb.opencga.analysis.sample.SampleTsvAnnotationLoader; import org.opencb.opencga.app.cli.internal.options.SampleCommandOptions; import org.opencb.opencga.core.exceptions.ToolException; @@ -25,9 +24,6 @@ public void execute() throws Exception { String subCommandString = getParsedSubCommand(sampleCommandOptions.jCommander); configure(); switch (subCommandString) { - case "secondary-index": - secondaryIndex(); - break; case "tsv-load": tsvLoad(); break; @@ -38,12 +34,6 @@ public void execute() throws Exception { } } - private void secondaryIndex() throws ToolException { - SampleCommandOptions.SecondaryIndex options = sampleCommandOptions.secondaryIndex; - Path outDir = Paths.get(options.outDir); - toolRunner.execute(SampleIndexTask.class, new ObjectMap(), outDir, options.jobOptions.jobId, options.commonOptions.token); - } - private void tsvLoad() throws ToolException { SampleCommandOptions.TsvLoad options = sampleCommandOptions.tsvLoad; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/StudyCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/StudyCommandExecutor.java index 0da33d83bfe..aa4eb7471d8 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/StudyCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/StudyCommandExecutor.java @@ -45,6 +45,6 @@ private void templateRun() throws ToolException { .toObjectMap(cliOptions.commonOptions.params) .append(ParamConstants.STUDY_PARAM, cliOptions.studyId); - toolRunner.execute(TemplateRunner.class, params, Paths.get(cliOptions.outdir), cliOptions.jobOptions.jobId, token); + toolRunner.execute(TemplateRunner.class, params, Paths.get(cliOptions.outdir), cliOptions.jobOptions.jobId, cliOptions.jobOptions.dryRun, token); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/ToolsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/ToolsCommandExecutor.java index 7a0f6b55a6f..ebcf46144db 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/ToolsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/ToolsCommandExecutor.java @@ -1,5 +1,6 @@ package org.opencb.opencga.app.cli.internal.executors; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.analysis.tools.OpenCgaTool; @@ -52,7 +53,7 @@ public void execute() throws Exception { private void executeTool() throws ToolException { ToolsCommandOptions.ExecuteToolCommandOptions cliOptions = this.toolCommandOptions.executeToolCommandOptions; toolRunner.execute(cliOptions.toolId, new ObjectMap(cliOptions.params), Paths.get(cliOptions.outDir), - toolCommandOptions.internalJobOptions.jobId, token); + toolCommandOptions.internalJobOptions.jobId, toolCommandOptions.internalJobOptions.dryRun, token); } private void executeJob() throws CatalogException, ToolException { @@ -61,7 +62,13 @@ private void executeJob() throws CatalogException, ToolException { } private void listTools() { - Collection> tools = new ToolFactory().getTools(); + Collection> tools; + if (configuration != null && configuration.getAnalysis() != null + && CollectionUtils.isNotEmpty(configuration.getAnalysis().getPackages())) { + tools = new ToolFactory().getTools(configuration.getAnalysis().getPackages()); + } else { + tools = new ToolFactory().getTools(); + } int toolIdSize = tools.stream().mapToInt(c -> c.getAnnotation(Tool.class).id().length()).max().orElse(0) + 2; int toolTypeSize = Arrays.stream(Tool.Type.values()).mapToInt(e->e.toString().length()).max().orElse(0) + 2; int toolResourceSize = Arrays.stream(Enums.Resource.values()).mapToInt(e->e.toString().length()).max().orElse(0) + 2; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java index b39a1387329..7d20deccb09 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java @@ -62,7 +62,6 @@ import org.opencb.opencga.core.common.YesNoAuto; import org.opencb.opencga.core.exceptions.AnalysisExecutionException; import org.opencb.opencga.core.exceptions.ToolException; -import org.opencb.opencga.core.models.clinical.ClinicalAnalysis; import org.opencb.opencga.core.models.clinical.ExomiserWrapperParams; import org.opencb.opencga.core.models.common.mixins.GenericRecordAvroJsonMixin; import org.opencb.opencga.core.models.operations.variant.*; @@ -129,6 +128,7 @@ public class VariantInternalCommandExecutor extends InternalCommandExecutor { // private AnalysisCliOptionsParser.VariantCommandOptions variantCommandOptions; private VariantCommandOptions variantCommandOptions; private String jobId; + private boolean dryRun; public VariantInternalCommandExecutor(VariantCommandOptions variantCommandOptions) { super(variantCommandOptions.commonCommandOptions); @@ -144,6 +144,7 @@ public void execute() throws Exception { configure(); jobId = variantCommandOptions.internalJobOptions.jobId; + dryRun = variantCommandOptions.internalJobOptions.dryRun; switch (subCommandString) { case VARIANT_DELETE_COMMAND: @@ -344,7 +345,7 @@ private void query(VariantCommandOptions.AbstractVariantQueryCommandOptions cliO cliOptions.outputFileName, cliOptions.outputFileFormat, cliOptions.variantsFile); - toolRunner.execute(VariantExportTool.class, toolParams, queryOptions, Paths.get(outdir), jobId, token); + toolRunner.execute(VariantExportTool.class, toolParams, queryOptions, Paths.get(outdir), jobId, dryRun, token); } } @@ -363,7 +364,7 @@ private void fileDelete() throws ToolException { cliOptions.genericVariantDeleteOptions.resume); toolRunner.execute(VariantFileDeleteOperationTool.class, params.toObjectMap(cliOptions.commonOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study), - Paths.get(cliOptions.outdir), jobId, token); + Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void index() throws ToolException { @@ -405,7 +406,7 @@ private void index() throws ToolException { .append(VariantStorageOptions.STDIN.key(), cliOptions.stdin) .append(VariantStorageOptions.STDOUT.key(), cliOptions.stdout); - toolRunner.execute(VariantIndexOperationTool.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(VariantIndexOperationTool.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void secondaryIndex() throws ToolException { @@ -420,9 +421,9 @@ private void secondaryIndex() throws ToolException { .append(ParamConstants.PROJECT_PARAM, cliOptions.project); if (CollectionUtils.isEmpty(cliOptions.sample)) { - toolRunner.execute(VariantSecondaryAnnotationIndexOperationTool.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(VariantSecondaryAnnotationIndexOperationTool.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } else { - toolRunner.execute(VariantSecondaryIndexSamplesOperationTool.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(VariantSecondaryIndexSamplesOperationTool.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } } @@ -455,7 +456,7 @@ private void statsRun() throws ToolException { .toObjectMap(cliOptions.commonOptions.params) .append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(VariantStatsAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(VariantStatsAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } // private void statsIndex() throws ToolException { @@ -486,7 +487,7 @@ private void scoreLoad() throws ToolException { .toObjectMap(cliOptions.commonOptions.params) .append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(VariantScoreIndexOperationTool.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(VariantScoreIndexOperationTool.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void scoreRemove() throws ToolException { @@ -499,7 +500,7 @@ private void scoreRemove() throws ToolException { .toObjectMap(cliOptions.commonOptions.params) .append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(VariantScoreDeleteOperationTool.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(VariantScoreDeleteOperationTool.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void sampleIndex() @@ -517,7 +518,7 @@ private void sampleIndex() toolRunner.execute(VariantSecondarySampleIndexOperationTool.class, params.toObjectMap(cliOptions.commonOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study), Paths.get(cliOptions.outdir), - jobId, token); + jobId, dryRun, token); } private void familyIndex() @@ -533,7 +534,7 @@ private void familyIndex() toolRunner.execute(VariantFamilyIndexOperationTool.class, params.toObjectMap(cliOptions.commonOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study), Paths.get(cliOptions.outdir), - jobId, token); + jobId, dryRun, token); } private void annotate() throws ToolException { @@ -558,7 +559,7 @@ private void annotate() throws ToolException { .append(ParamConstants.PROJECT_PARAM, cliOptions.project) .append(ParamConstants.STUDY_PARAM, cliOptions.study), Paths.get(cliOptions.outdir), - jobId, token); + jobId, dryRun, token); } private void annotationSave() throws ToolException { @@ -568,7 +569,7 @@ private void annotationSave() throws ToolException { .toObjectMap(cliOptions.commonOptions.params) .append(ParamConstants.PROJECT_PARAM, cliOptions.project); - toolRunner.execute(VariantAnnotationSaveOperationTool.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(VariantAnnotationSaveOperationTool.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void annotationDelete() throws ToolException { @@ -578,7 +579,7 @@ private void annotationDelete() throws ToolException { .toObjectMap(cliOptions.commonOptions.params) .append(ParamConstants.PROJECT_PARAM, cliOptions.project); - toolRunner.execute(VariantAnnotationDeleteOperationTool.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(VariantAnnotationDeleteOperationTool.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void annotationQuery() throws CatalogException, IOException, StorageEngineException { @@ -646,7 +647,7 @@ private void aggregateFamily() throws ToolException { .toObjectMap(cliOptions.commonOptions.params) .append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(VariantAggregateFamilyOperationTool.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(VariantAggregateFamilyOperationTool.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void aggregate() throws ToolException { @@ -658,7 +659,7 @@ private void aggregate() throws ToolException { .toObjectMap(cliOptions.commonOptions.params) .append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(VariantAggregateOperationTool.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(VariantAggregateOperationTool.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void sampleRun() throws Exception { @@ -668,7 +669,7 @@ private void sampleRun() throws Exception { toolRunner.execute(SampleVariantFilterAnalysis.class, cliOptions.toolParams.toObjectMap(cliOptions.commonOptions.params) .append(ParamConstants.STUDY_PARAM, cliOptions.study), - Paths.get(cliOptions.outdir), jobId, token); + Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void gwas() throws Exception { @@ -690,7 +691,7 @@ private void gwas() throws Exception { } GwasAnalysis gwasAnalysis = new GwasAnalysis(); gwasAnalysis.setUp(appHome, catalogManager, storageEngineFactory, params, Paths.get(cliOptions.outdir), - variantCommandOptions.internalJobOptions.jobId, token); + variantCommandOptions.internalJobOptions.jobId, dryRun, token); gwasAnalysis.setStudy(cliOptions.study) .setPhenotype(cliOptions.phenotype) .setIndex(cliOptions.index) @@ -724,7 +725,7 @@ private void knockout() throws Exception { .toObjectMap(cliOptions.commonOptions.params) .append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(KnockoutAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(KnockoutAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void sampleEligibility() throws Exception { @@ -737,7 +738,7 @@ private void sampleEligibility() throws Exception { .toObjectMap(cliOptions.commonOptions.params) .append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(SampleEligibilityAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(SampleEligibilityAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void sampleStats() throws Exception { @@ -756,7 +757,7 @@ private void sampleStats() throws Exception { cliOptions.batchSize, variantQuery ); - toolRunner.execute(SampleVariantStatsAnalysis.class, toolParams, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(SampleVariantStatsAnalysis.class, toolParams, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void cohortStats() throws Exception { @@ -775,7 +776,7 @@ private void cohortStats() throws Exception { CohortVariantStatsAnalysis cohortVariantStatsAnalysis = new CohortVariantStatsAnalysis(); cohortVariantStatsAnalysis.setUp(appHome, catalogManager, storageEngineFactory, params, Paths.get(cliOptions.outdir), - variantCommandOptions.internalJobOptions.jobId, token); + variantCommandOptions.internalJobOptions.jobId, dryRun, token); cohortVariantStatsAnalysis.setStudy(cliOptions.study) .setCohortName(cliOptions.cohort) .setIndex(cliOptions.index) @@ -796,7 +797,7 @@ private void julie() throws Exception { Path outdir = Paths.get(cliOptions.outdir); - toolRunner.execute(JulieTool.class, toolParams, params, outdir, jobId, token); + toolRunner.execute(JulieTool.class, toolParams, params, outdir, jobId, dryRun, token); } private void mutationalSignature() throws Exception { @@ -824,7 +825,7 @@ private void mutationalSignature() throws Exception { cliOptions.outdir) .toObjectMap(cliOptions.commonOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(MutationalSignatureAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(MutationalSignatureAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void hrDetect() throws Exception { @@ -846,7 +847,7 @@ private void hrDetect() throws Exception { cliOptions.outdir) .toObjectMap(cliOptions.commonOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(HRDetectAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(HRDetectAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void genomePlot() throws Exception { @@ -860,7 +861,7 @@ private void genomePlot() throws Exception { cliOptions.outdir) .toObjectMap(cliOptions.commonOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(GenomePlotAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(GenomePlotAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void mendelianError() throws Exception { @@ -870,7 +871,7 @@ private void mendelianError() throws Exception { MendelianErrorAnalysis mendelianErrorAnalysis = new MendelianErrorAnalysis(); mendelianErrorAnalysis.setUp(appHome, catalogManager, storageEngineFactory, params, Paths.get(cliOptions.outdir), - variantCommandOptions.internalJobOptions.jobId, token); + variantCommandOptions.internalJobOptions.jobId, dryRun, token); mendelianErrorAnalysis.setStudy(cliOptions.study) .setFamilyId(cliOptions.family) .setIndividualId(cliOptions.individual) @@ -885,7 +886,7 @@ private void inferredSex() throws Exception { InferredSexAnalysis inferredSexAnalysis = new InferredSexAnalysis(); inferredSexAnalysis.setUp(appHome, catalogManager, storageEngineFactory, params, Paths.get(cliOptions.outdir), - variantCommandOptions.internalJobOptions.jobId, token); + variantCommandOptions.internalJobOptions.jobId, dryRun, token); inferredSexAnalysis.setStudyId(cliOptions.study) .setIndividualId(cliOptions.individual) .setSampleId(cliOptions.sample) @@ -899,7 +900,7 @@ private void relatedness() throws Exception { RelatednessAnalysis relatednessAnalysis = new RelatednessAnalysis(); relatednessAnalysis.setUp(appHome, catalogManager, storageEngineFactory, params, Paths.get(cliOptions.outdir), - variantCommandOptions.internalJobOptions.jobId, token); + variantCommandOptions.internalJobOptions.jobId, dryRun, token); relatednessAnalysis.setStudyId(cliOptions.study) .setIndividualIds(cliOptions.individuals) .setSampleIds(cliOptions.samples) @@ -915,7 +916,7 @@ private void familyQc() throws Exception { FamilyQcAnalysis familyQcAnalysis = new FamilyQcAnalysis(); familyQcAnalysis.setUp(appHome, catalogManager, storageEngineFactory, params, Paths.get(cliOptions.outdir), - variantCommandOptions.internalJobOptions.jobId, token); + variantCommandOptions.internalJobOptions.jobId, dryRun, token); familyQcAnalysis.setStudyId(cliOptions.study) .setFamilyId(cliOptions.family) .setRelatednessMethod(cliOptions.relatednessMethod) @@ -930,7 +931,7 @@ private void individualQc() throws Exception { IndividualQcAnalysis individualQcAnalysis = new IndividualQcAnalysis(); individualQcAnalysis.setUp(appHome, catalogManager, storageEngineFactory, params, Paths.get(cliOptions.outdir), - variantCommandOptions.internalJobOptions.jobId, token); + variantCommandOptions.internalJobOptions.jobId, dryRun, token); individualQcAnalysis.setStudyId(cliOptions.study) .setIndividualId(cliOptions.individual) .setSampleId(cliOptions.sample) @@ -973,7 +974,7 @@ private void sampleQc() throws Exception { cliOptions.outdir) .toObjectMap(cliOptions.commonOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(SampleQcAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(SampleQcAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } // Wrappers @@ -986,7 +987,7 @@ private void plink() throws Exception { cliOptions.plinkParams) .toObjectMap(cliOptions.basicOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(PlinkWrapperAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(PlinkWrapperAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void rvtests() throws Exception { @@ -998,7 +999,7 @@ private void rvtests() throws Exception { cliOptions.rvtestsParams) .toObjectMap(cliOptions.basicOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(RvtestsWrapperAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(RvtestsWrapperAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void gatk() throws Exception { @@ -1010,7 +1011,7 @@ private void gatk() throws Exception { cliOptions.gatkParams) .toObjectMap(cliOptions.basicOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(GatkWrapperAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(GatkWrapperAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void exomiser() throws Exception { @@ -1019,10 +1020,11 @@ private void exomiser() throws Exception { ObjectMap params = new ExomiserWrapperParams( cliOptions.sample, cliOptions.clinicalAnalysisType, + cliOptions.exomiserVersion, cliOptions.outdir) .toObjectMap(cliOptions.commonOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(ExomiserWrapperAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(ExomiserWrapperAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void checkSignatureVersion(String sigVersion) throws ClientException { diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/AlignmentCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/AlignmentCommandOptions.java index 3834c5ee4c5..c9d92e51d4f 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/AlignmentCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/AlignmentCommandOptions.java @@ -470,7 +470,7 @@ public class BwaCommandOptions { @ParametersDelegate public Object internalJobOptions = internalJobOptionsObject; - @Parameter(names = {"-s", "--study"}, description = "Study [[user@]project:]study.", arity = 1) + @Parameter(names = {"-s", "--study"}, description = "Study [[organization@]project:]study.", arity = 1) public String study; @Parameter(names = {"--command"}, description = BWA_COMMAND_DESCRIPTION) @@ -538,7 +538,7 @@ public class DeeptoolsCommandOptions { @ParametersDelegate public Object internalJobOptions = internalJobOptionsObject; - @Parameter(names = {"-s", "--study"}, description = "Study [[user@]project:]study.", arity = 1) + @Parameter(names = {"-s", "--study"}, description = "Study [[organization@]project:]study.", arity = 1) public String study; @Parameter(names = {"--command"}, description = DEEPTOOLS_COMMAND_DESCRIPTION) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/ClinicalCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/ClinicalCommandOptions.java index 9c7f8806fc7..c2bac1cd1aa 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/ClinicalCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/ClinicalCommandOptions.java @@ -23,6 +23,8 @@ import static org.opencb.opencga.analysis.clinical.InterpretationAnalysis.*; import static org.opencb.opencga.analysis.variant.manager.VariantCatalogQueryUtils.*; +import static org.opencb.opencga.core.api.FieldConstants.EXOMISER_CLINICAL_ANALYSIS_DESCRIPTION; +import static org.opencb.opencga.core.api.FieldConstants.EXOMISER_VERSION_DESCRIPTION; import static org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam.*; @Parameters(commandNames = {"clinical"}, commandDescription = "Clinical analysis commands") @@ -35,6 +37,7 @@ public class ClinicalCommandOptions { public final RgaSecondaryIndexCommandOptions rgaSecondaryIndexCommandOptions; public final RgaAuxiliarSecondaryIndexCommandOptions rgaAuxiliarSecondaryIndexCommandOptions; public final ExomiserInterpretationCommandOptions exomiserInterpretationCommandOptions; + public final ImportClinicalAnalysesCommandOptions importClinicalAnalysesCommandOptions; public final TsvLoad tsvLoad; public JCommander jCommander; @@ -56,6 +59,7 @@ public ClinicalCommandOptions(GeneralCliOptions.CommonCommandOptions commonComma this.rgaSecondaryIndexCommandOptions = new RgaSecondaryIndexCommandOptions(); this.rgaAuxiliarSecondaryIndexCommandOptions = new RgaAuxiliarSecondaryIndexCommandOptions(); this.exomiserInterpretationCommandOptions = new ExomiserInterpretationCommandOptions(); + this.importClinicalAnalysesCommandOptions = new ImportClinicalAnalysesCommandOptions(); this.tsvLoad = new TsvLoad(); } @@ -333,13 +337,33 @@ public class ExomiserInterpretationCommandOptions extends GeneralCliOptions.Stud @ParametersDelegate public InternalCliOptionsParser.JobOptions jobOptions = internalJobOptions; - @Parameter(names = {"--" + CLINICAL_ANALYISIS_PARAM_NAME}, description = "Clinical analysis", required = true, arity = 1) + @Parameter(names = {"--" + CLINICAL_ANALYISIS_PARAM_NAME}, description = EXOMISER_CLINICAL_ANALYSIS_DESCRIPTION, + required = true, arity = 1) public String clinicalAnalysis; + @Parameter(names = {"--exomiser-version"}, description = EXOMISER_VERSION_DESCRIPTION) + public String exomiserVersion; + @Parameter(names = {"-o", "--outdir"}, description = "Directory where output files will be saved", arity = 1) public String outdir; } + @Parameters(commandNames = {ImportClinicalAnalysesCommandOptions.IMPORT_COMMAND}, + commandDescription = "Import clinical analyses from a folder") + public class ImportClinicalAnalysesCommandOptions extends GeneralCliOptions.StudyOption { + + public static final String IMPORT_COMMAND = "import"; + + @ParametersDelegate + public GeneralCliOptions.CommonCommandOptions commonOptions = commonCommandOptions; + + @ParametersDelegate + public InternalCliOptionsParser.JobOptions jobOptions = internalJobOptions; + + @Parameter(names = {"-i", "--input"}, description = "Input directory where clinical analysis JSON files are located", arity = 1) + public String input; + } + @Parameters(commandNames = {"tsv-load"}, commandDescription = "Load annotations from a TSV file") public class TsvLoad extends GeneralCliOptions.StudyOption { diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/CohortCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/CohortCommandOptions.java index ca05542c202..75f21fba86c 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/CohortCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/CohortCommandOptions.java @@ -13,7 +13,6 @@ public class CohortCommandOptions { public final static String OUTDIR_PARAM_NAME = "outdir"; - public SecondaryIndex secondaryIndex; public TsvLoad tsvLoad; public GeneralCliOptions.CommonCommandOptions cohortCommandOptions; @@ -25,27 +24,9 @@ public CohortCommandOptions(GeneralCliOptions.CommonCommandOptions cohortCommand this.internalJobOptions = new InternalCliOptionsParser.JobOptions(); this.jCommander = jCommander; - this.secondaryIndex = new SecondaryIndex(); this.tsvLoad = new TsvLoad(); } - @Parameters(commandNames = {"secondary-index"}, commandDescription = "Creates a secondary index for cohorts using a search engine") - public class SecondaryIndex { - @ParametersDelegate - public GeneralCliOptions.CommonCommandOptions commonOptions = cohortCommandOptions; - - @ParametersDelegate - public InternalCliOptionsParser.JobOptions jobOptions = internalJobOptions; - - @Parameter(names = {"-s", "--" + ParamConstants.STUDY_PARAM}, description = "Study [[user@]project:]study.", required = false, - arity = 1) - public String studyId; - - @Parameter(names = {"-o", "--" + OUTDIR_PARAM_NAME}, description = "Directory where output files will be saved", required = true, - arity = 1) - public String outDir; - } - @Parameters(commandNames = {"tsv-load"}, commandDescription = "Load annotations from a TSV file") public class TsvLoad { @ParametersDelegate @@ -54,7 +35,7 @@ public class TsvLoad { @ParametersDelegate public InternalCliOptionsParser.JobOptions jobOptions = internalJobOptions; - @Parameter(names = {"-s", "--" + ParamConstants.STUDY_PARAM}, description = "Study [[user@]project:]study.", required = true, + @Parameter(names = {"-s", "--" + ParamConstants.STUDY_PARAM}, description = "Study [[organization@]project:]study.", required = true, arity = 1) public String studyId; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/DiseasePanelInternalCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/DiseasePanelInternalCommandOptions.java index f0854cccb50..12e5db805c0 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/DiseasePanelInternalCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/DiseasePanelInternalCommandOptions.java @@ -36,7 +36,7 @@ public class PanelImportCommandOptions { @ParametersDelegate public InternalCliOptionsParser.JobOptions jobOptions = internalJobOptions; - @Parameter(names = {"-s", "--" + ParamConstants.STUDY_PARAM}, description = "Study [[user@]project:]study.", arity = 1) + @Parameter(names = {"-s", "--" + ParamConstants.STUDY_PARAM}, description = "Study [[organization@]project:]study.", arity = 1) public String studyId; @Parameter(names = {"-o", "--" + OUTDIR_PARAM_NAME}, description = "Directory where output files will be saved", required = true, diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/FamilyCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/FamilyCommandOptions.java index 3702dc0b8a5..d7d7010a63d 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/FamilyCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/FamilyCommandOptions.java @@ -13,7 +13,6 @@ public class FamilyCommandOptions { public final static String OUTDIR_PARAM_NAME = "outdir"; - public SecondaryIndex secondaryIndex; public TsvLoad tsvLoad; public final GeneralCliOptions.CommonCommandOptions familyCommandOptions; @@ -25,27 +24,9 @@ public FamilyCommandOptions(GeneralCliOptions.CommonCommandOptions familyCommand this.internalJobOptions = new InternalCliOptionsParser.JobOptions(); this.jCommander = jCommander; - this.secondaryIndex = new SecondaryIndex(); this.tsvLoad = new TsvLoad(); } - @Parameters(commandNames = {"secondary-index"}, commandDescription = "Creates a secondary index for families using a search engine") - public class SecondaryIndex { - @ParametersDelegate - public GeneralCliOptions.CommonCommandOptions commonOptions = familyCommandOptions; - - @ParametersDelegate - public InternalCliOptionsParser.JobOptions jobOptions = internalJobOptions; - - @Parameter(names = {"-s", "--" + ParamConstants.STUDY_PARAM}, description = "Study [[user@]project:]study.", required = false, - arity = 1) - public String studyId; - - @Parameter(names = {"-o", "--" + OUTDIR_PARAM_NAME}, description = "Directory where output files will be saved", required = true, - arity = 1) - public String outDir; - } - @Parameters(commandNames = {"tsv-load"}, commandDescription = "Load annotations from a TSV file") public class TsvLoad { @ParametersDelegate @@ -54,7 +35,7 @@ public class TsvLoad { @ParametersDelegate public InternalCliOptionsParser.JobOptions jobOptions = internalJobOptions; - @Parameter(names = {"-s", "--" + ParamConstants.STUDY_PARAM}, description = "Study [[user@]project:]study.", required = true, + @Parameter(names = {"-s", "--" + ParamConstants.STUDY_PARAM}, description = "Study [[organization@]project:]study.", required = true, arity = 1) public String studyId; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/FileCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/FileCommandOptions.java index 8fa80e486a3..4825ec26306 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/FileCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/FileCommandOptions.java @@ -4,7 +4,6 @@ import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; import com.beust.jcommander.ParametersDelegate; -import com.microsoft.graph.models.extensions.Post; import org.opencb.opencga.app.cli.GeneralCliOptions; import org.opencb.opencga.app.cli.internal.InternalCliOptionsParser; import org.opencb.opencga.core.api.ParamConstants; @@ -20,7 +19,6 @@ public class FileCommandOptions { public UnlinkCommandOptions unlinkCommandOptions; public FetchCommandOptions fetchCommandOptions; public PostlinkCommandOptions postlinkCommandOptions; - public SecondaryIndex secondaryIndex; public TsvLoad tsvLoad; public final GeneralCliOptions.CommonCommandOptions fileCommonOptions; @@ -36,7 +34,6 @@ public FileCommandOptions(GeneralCliOptions.CommonCommandOptions fileCommonComma this.unlinkCommandOptions = new UnlinkCommandOptions(); this.fetchCommandOptions = new FetchCommandOptions(); this.postlinkCommandOptions = new PostlinkCommandOptions(); - this.secondaryIndex = new SecondaryIndex(); this.tsvLoad = new TsvLoad(); } @@ -49,7 +46,7 @@ public class DeleteCommandOptions { @ParametersDelegate public InternalCliOptionsParser.JobOptions jobOptions = internalJobOptions; - @Parameter(names = {"-s", "--" + ParamConstants.STUDY_PARAM}, description = "Study [[user@]project:]study.", required = true, + @Parameter(names = {"-s", "--" + ParamConstants.STUDY_PARAM}, description = "Study [[organization@]project:]study.", required = true, arity = 1) public String studyId; @@ -73,7 +70,7 @@ public class UnlinkCommandOptions { @ParametersDelegate public InternalCliOptionsParser.JobOptions jobOptions = internalJobOptions; - @Parameter(names = {"-s", "--" + ParamConstants.STUDY_PARAM}, description = "Study [[user@]project:]study.", required = true, + @Parameter(names = {"-s", "--" + ParamConstants.STUDY_PARAM}, description = "Study [[organization@]project:]study.", required = true, arity = 1) public String studyId; @@ -93,7 +90,7 @@ public class PostlinkCommandOptions { @ParametersDelegate public InternalCliOptionsParser.JobOptions jobOptions = internalJobOptions; - @Parameter(names = {"-s", "--" + ParamConstants.STUDY_PARAM}, description = "Study [[user@]project:]study.", required = true, + @Parameter(names = {"-s", "--" + ParamConstants.STUDY_PARAM}, description = "Study [[organization@]project:]study.", required = true, arity = 1) public String studyId; @@ -116,7 +113,7 @@ public class FetchCommandOptions { @ParametersDelegate public InternalCliOptionsParser.JobOptions jobOptions = internalJobOptions; - @Parameter(names = {"-s", "--" + ParamConstants.STUDY_PARAM}, description = "Study [[user@]project:]study.", required = true, + @Parameter(names = {"-s", "--" + ParamConstants.STUDY_PARAM}, description = "Study [[organization@]project:]study.", required = true, arity = 1) public String studyId; @@ -133,23 +130,6 @@ public class FetchCommandOptions { public String outDir; } - @Parameters(commandNames = {"secondary-index"}, commandDescription = "Creates a secondary index for files using a search engine") - public class SecondaryIndex { - @ParametersDelegate - public GeneralCliOptions.CommonCommandOptions commonOptions = fileCommonOptions; - - @ParametersDelegate - public InternalCliOptionsParser.JobOptions jobOptions = internalJobOptions; - - @Parameter(names = {"-s", "--" + ParamConstants.STUDY_PARAM}, description = "Study [[user@]project:]study.", required = false, - arity = 1) - public String studyId; - - @Parameter(names = {"-o", "--" + OUTDIR_PARAM_NAME}, description = "Directory where output files will be saved", required = true, - arity = 1) - public String outDir; - } - @Parameters(commandNames = {"tsv-load"}, commandDescription = "Load annotations from a TSV file") public class TsvLoad { @ParametersDelegate @@ -158,7 +138,7 @@ public class TsvLoad { @ParametersDelegate public InternalCliOptionsParser.JobOptions jobOptions = internalJobOptions; - @Parameter(names = {"-s", "--" + ParamConstants.STUDY_PARAM}, description = "Study [[user@]project:]study.", required = true, + @Parameter(names = {"-s", "--" + ParamConstants.STUDY_PARAM}, description = "Study [[organization@]project:]study.", required = true, arity = 1) public String studyId; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/IndividualCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/IndividualCommandOptions.java index 88df4c71771..f4ffb77665e 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/IndividualCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/IndividualCommandOptions.java @@ -13,7 +13,6 @@ public class IndividualCommandOptions { public final static String OUTDIR_PARAM_NAME = "outdir"; - public SecondaryIndex secondaryIndex; public TsvLoad tsvLoad; public GeneralCliOptions.CommonCommandOptions individualCommandOptions; @@ -25,27 +24,9 @@ public IndividualCommandOptions(GeneralCliOptions.CommonCommandOptions individua this.internalJobOptions = new InternalCliOptionsParser.JobOptions(); this.jCommander = jCommander; - this.secondaryIndex = new SecondaryIndex(); this.tsvLoad = new TsvLoad(); } - @Parameters(commandNames = {"secondary-index"}, commandDescription = "Creates a secondary index for individuals using a search engine") - public class SecondaryIndex { - @ParametersDelegate - public GeneralCliOptions.CommonCommandOptions commonOptions = individualCommandOptions; - - @ParametersDelegate - public InternalCliOptionsParser.JobOptions jobOptions = internalJobOptions; - - @Parameter(names = {"-s", "--" + ParamConstants.STUDY_PARAM}, description = "Study [[user@]project:]study.", required = false, - arity = 1) - public String studyId; - - @Parameter(names = {"-o", "--" + OUTDIR_PARAM_NAME}, description = "Directory where output files will be saved", required = true, - arity = 1) - public String outDir; - } - @Parameters(commandNames = {"tsv-load"}, commandDescription = "Load annotations from a TSV file") public class TsvLoad { @ParametersDelegate @@ -54,7 +35,7 @@ public class TsvLoad { @ParametersDelegate public InternalCliOptionsParser.JobOptions jobOptions = internalJobOptions; - @Parameter(names = {"-s", "--" + ParamConstants.STUDY_PARAM}, description = "Study [[user@]project:]study.", required = true, + @Parameter(names = {"-s", "--" + ParamConstants.STUDY_PARAM}, description = "Study [[organization@]project:]study.", required = true, arity = 1) public String studyId; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/JobCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/JobCommandOptions.java index 3f425a27de8..95ddfcc207c 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/JobCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/JobCommandOptions.java @@ -1,20 +1,15 @@ package org.opencb.opencga.app.cli.internal.options; import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; -import com.beust.jcommander.ParametersDelegate; import org.opencb.opencga.app.cli.GeneralCliOptions; import org.opencb.opencga.app.cli.internal.InternalCliOptionsParser; -import org.opencb.opencga.core.api.ParamConstants; @Parameters(commandNames = {"job"}, commandDescription = "Implement several job tasks") public class JobCommandOptions { public final static String OUTDIR_PARAM_NAME = "outdir"; - public SecondaryIndex secondaryIndex; - public GeneralCliOptions.CommonCommandOptions jobCommandOptions; public final InternalCliOptionsParser.JobOptions internalJobOptions; public JCommander jCommander; @@ -24,24 +19,6 @@ public JobCommandOptions(GeneralCliOptions.CommonCommandOptions jobCommandOption this.internalJobOptions = new InternalCliOptionsParser.JobOptions(); this.jCommander = jCommander; - this.secondaryIndex = new SecondaryIndex(); - } - - @Parameters(commandNames = {"secondary-index"}, commandDescription = "Creates a secondary index for jobs using a search engine") - public class SecondaryIndex { - @ParametersDelegate - public GeneralCliOptions.CommonCommandOptions commonOptions = jobCommandOptions; - - @ParametersDelegate - public InternalCliOptionsParser.JobOptions jobOptions = internalJobOptions; - - @Parameter(names = {"-s", "--" + ParamConstants.STUDY_PARAM}, description = "Study [[user@]project:]study.", required = false, - arity = 1) - public String studyId; - - @Parameter(names = {"-o", "--" + OUTDIR_PARAM_NAME}, description = "Directory where output files will be saved", required = true, - arity = 1) - public String outDir; } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/SampleCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/SampleCommandOptions.java index 88287d90fe7..d15646ad607 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/SampleCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/SampleCommandOptions.java @@ -14,7 +14,6 @@ public class SampleCommandOptions { public final static String OUTDIR_PARAM_NAME = "outdir"; public final GeneralCliOptions.CommonCommandOptions sampleCommonOptions; public final InternalCliOptionsParser.JobOptions internalJobOptions; - public SecondaryIndex secondaryIndex; public TsvLoad tsvLoad; public JCommander jCommander; @@ -22,27 +21,9 @@ public SampleCommandOptions(GeneralCliOptions.CommonCommandOptions sampleCommonO this.sampleCommonOptions = sampleCommonOptions; this.internalJobOptions = new InternalCliOptionsParser.JobOptions(); this.jCommander = jCommander; - this.secondaryIndex = new SecondaryIndex(); this.tsvLoad = new TsvLoad(); } - @Parameters(commandNames = {"secondary-index"}, commandDescription = "Creates a secondary index for samples using a search engine") - public class SecondaryIndex { - @ParametersDelegate - public GeneralCliOptions.CommonCommandOptions commonOptions = sampleCommonOptions; - - @ParametersDelegate - public InternalCliOptionsParser.JobOptions jobOptions = internalJobOptions; - - @Parameter(names = {"-s", "--" + ParamConstants.STUDY_PARAM}, description = "Study [[user@]project:]study.", required = false, - arity = 1) - public String studyId; - - @Parameter(names = {"-o", "--" + OUTDIR_PARAM_NAME}, description = "Directory where output files will be saved", required = true, - arity = 1) - public String outDir; - } - @Parameters(commandNames = {"tsv-load"}, commandDescription = "Load annotations from a TSV file") public class TsvLoad { @ParametersDelegate @@ -51,7 +32,7 @@ public class TsvLoad { @ParametersDelegate public InternalCliOptionsParser.JobOptions jobOptions = internalJobOptions; - @Parameter(names = {"-s", "--" + ParamConstants.STUDY_PARAM}, description = "Study [[user@]project:]study.", required = true, + @Parameter(names = {"-s", "--" + ParamConstants.STUDY_PARAM}, description = "Study [[organization@]project:]study.", required = true, arity = 1) public String studyId; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/StudyCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/StudyCommandOptions.java index 7cfdaa6af9d..096981a36c5 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/StudyCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/StudyCommandOptions.java @@ -38,7 +38,7 @@ public class TemplateLoader { @ParametersDelegate public InternalCliOptionsParser.JobOptions jobOptions = internalJobOptions; - @Parameter(names = {"-s", "--" + ParamConstants.STUDY_PARAM}, description = "Study [[user@]project:]study.", arity = 1) + @Parameter(names = {"-s", "--" + ParamConstants.STUDY_PARAM}, description = "Study [[organization@]project:]study.", arity = 1) public String studyId; @Parameter(names = {"-o", "--" + OUTDIR_PARAM_NAME}, description = "Directory where output files will be saved", required = true, diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/VariantCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/VariantCommandOptions.java index aa25fe630e1..46ef28a52f6 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/VariantCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/VariantCommandOptions.java @@ -70,6 +70,8 @@ import static org.opencb.opencga.app.cli.internal.options.VariantCommandOptions.VariantSampleQueryCommandOptions.SAMPLE_QUERY_COMMAND; import static org.opencb.opencga.app.cli.internal.options.VariantCommandOptions.VariantSecondaryIndexCommandOptions.SECONDARY_INDEX_COMMAND; import static org.opencb.opencga.app.cli.internal.options.VariantCommandOptions.VariantSecondaryIndexDeleteCommandOptions.SECONDARY_INDEX_DELETE_COMMAND; +import static org.opencb.opencga.core.api.FieldConstants.EXOMISER_SAMPLE_DESCRIPTION; +import static org.opencb.opencga.core.api.FieldConstants.EXOMISER_VERSION_DESCRIPTION; import static org.opencb.opencga.core.api.ParamConstants.*; import static org.opencb.opencga.storage.app.cli.client.options.StorageVariantCommandOptions.AggregateCommandOptions.AGGREGATE_COMMAND; import static org.opencb.opencga.storage.app.cli.client.options.StorageVariantCommandOptions.AggregateCommandOptions.AGGREGATE_COMMAND_DESCRIPTION; @@ -1842,12 +1844,15 @@ public class ExomiserAnalysisCommandOptions { @Parameter(names = {"--study"}, description = "Study where all the samples belong to.") public String study; - @Parameter(names = {"--sample"}, description = FieldConstants.SAMPLE_ID_DESCRIPTION, required = true) + @Parameter(names = {"--sample"}, description = EXOMISER_SAMPLE_DESCRIPTION, required = true) public String sample; @Parameter(names = {"--clinical-analysis-type"}, description = FieldConstants.EXOMISER_CLINICAL_ANALYSIS_TYPE_DESCRIPTION) public String clinicalAnalysisType = ClinicalAnalysis.Type.SINGLE.name(); + @Parameter(names = {"--exomiser-version"}, description = EXOMISER_VERSION_DESCRIPTION) + public String exomiserVersion; + @Parameter(names = {"-o", "--outdir"}, description = FieldConstants.JOB_OUT_DIR_DESCRIPTION) public String outdir; } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java index b0db2e99473..fa5b3482284 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java @@ -1,19 +1,3 @@ -/* -* Copyright 2015-2024-04-25 OpenCB -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - package org.opencb.opencga.app.cli.main; import org.jline.reader.Candidate; @@ -40,7 +24,7 @@ public abstract class OpenCgaCompleter implements Completer { - protected List commands = asList("login","logout","help","use","variant","projects","panels","clinical","jobs","admin","individuals","families","users","samples","alignments","meta","studies","files","operations","cohorts") + protected List commands = asList("login","logout","help","use","variant","projects","panels","clinical","jobs","admin","individuals","families","users","samples","alignments","meta","organizations","studies","files","operations","cohorts") .stream() .map(Candidate::new) .collect(toList()); @@ -50,7 +34,7 @@ public abstract class OpenCgaCompleter implements Completer { .map(Candidate::new) .collect(toList()); - private List projectsList = asList( "create","search","aggregationstats","info","increlease","studies","update") + private List projectsList = asList( "create","search","info","increlease","studies","update") .stream() .map(Candidate::new) .collect(toList()); @@ -60,37 +44,37 @@ public abstract class OpenCgaCompleter implements Completer { .map(Candidate::new) .collect(toList()); - private List clinicalList = asList( "acl-update","annotation-sets-load","clinical-configuration-update","create","distinct","interpretation-distinct","interpretation-search","interpretation-info","interpreter-cancer-tiering-run","interpreter-exomiser-run","interpreter-team-run","interpreter-tiering-run","interpreter-zetta-run","rga-aggregation-stats","rga-gene-query","rga-gene-summary","rga-index-run","rga-individual-query","rga-individual-summary","rga-variant-query","rga-variant-summary","search","variant-query","acl","delete","update","annotation-sets-annotations-update","info","interpretation-create","interpretation-clear","interpretation-delete","interpretation-revert","interpretation-update","report-update") + private List clinicalList = asList( "acl-update","annotation-sets-load","clinical-configuration-update","create","distinct","interpretation-distinct","interpretation-search","interpretation-info","interpreter-cancer-tiering-run","interpreter-exomiser-run","interpreter-team-run","interpreter-tiering-run","interpreter-zetta-run","load","rga-aggregation-stats","rga-gene-query","rga-gene-summary","rga-index-run","rga-individual-query","rga-individual-summary","rga-variant-query","rga-variant-summary","search","variant-query","acl","delete","update","annotation-sets-annotations-update","info","interpretation-create","interpretation-clear","interpretation-delete","interpretation-revert","interpretation-update","report-update") .stream() .map(Candidate::new) .collect(toList()); - private List jobsList = asList( "acl-update","aggregationstats","create","distinct","retry","search","top","acl","delete","info","update","log-head","log-tail") + private List jobsList = asList( "acl-update","create","distinct","retry","search","top","acl","delete","info","update","kill","log-head","log-tail") .stream() .map(Candidate::new) .collect(toList()); - private List adminList = asList( "audit-group-by","catalog-index-stats","catalog-install","catalog-jwt","users-create","users-import","users-search","users-sync","update-groups-users") + private List adminList = asList( "audit-group-by","catalog-install","catalog-jwt","users-create","users-import","users-permissions","users-search","users-sync","update-groups-users") .stream() .map(Candidate::new) .collect(toList()); - private List individualsList = asList( "acl-update","aggregationstats","annotation-sets-load","create","distinct","search","acl","delete","info","update","annotation-sets-annotations-update","relatives") + private List individualsList = asList( "acl-update","annotation-sets-load","create","distinct","search","acl","delete","info","update","annotation-sets-annotations-update","relatives") .stream() .map(Candidate::new) .collect(toList()); - private List familiesList = asList( "acl-update","aggregationstats","annotation-sets-load","create","distinct","search","acl","delete","info","update","annotation-sets-annotations-update") + private List familiesList = asList( "acl-update","annotation-sets-load","create","distinct","search","acl","delete","info","update","annotation-sets-annotations-update") .stream() .map(Candidate::new) .collect(toList()); - private List usersList = asList( "login","password","info","configs","configs-update","filters","password-reset","projects","update") + private List usersList = asList( "anonymous","create","login","password","search","info","configs","configs-update","filters","password-reset","update") .stream() .map(Candidate::new) .collect(toList()); - private List samplesList = asList( "acl-update","aggregationstats","annotation-sets-load","create","distinct","load","search","acl","delete","info","update","annotation-sets-annotations-update") + private List samplesList = asList( "acl-update","annotation-sets-load","create","distinct","load","search","acl","delete","info","update","annotation-sets-annotations-update") .stream() .map(Candidate::new) .collect(toList()); @@ -105,22 +89,27 @@ public abstract class OpenCgaCompleter implements Completer { .map(Candidate::new) .collect(toList()); - private List studiesList = asList( "acl-update","create","search","acl","aggregationstats","info","audit-search","groups","groups-update","groups-users-update","permissionrules","permission-rules-update","templates-run","templates-upload","templates-delete","update","variablesets","variable-sets-update","variable-sets-variables-update") + private List organizationsList = asList( "create","notes-create","notes-search","notes-delete","notes-update","update-status-user","user-update","configuration-update","info","update") + .stream() + .map(Candidate::new) + .collect(toList()); + + private List studiesList = asList( "acl-update","create","search","acl","info","audit-search","groups","groups-update","groups-users-update","notes-create","notes-search","notes-delete","notes-update","permissionrules","permission-rules-update","templates-run","templates-upload","templates-delete","update","variablesets","variable-sets-update","variable-sets-variables-update") .stream() .map(Candidate::new) .collect(toList()); - private List filesList = asList( "acl-update","aggregationstats","annotation-sets-load","bioformats","create","distinct","fetch","formats","link","link-run","postlink-run","search","upload","acl","delete","info","unlink","update","annotation-sets-annotations-update","download","grep","head","image","move","refresh","tail","list","tree") + private List filesList = asList( "acl-update","annotation-sets-load","bioformats","create","distinct","fetch","formats","link","link-run","postlink-run","search","upload","acl","delete","info","unlink","update","annotation-sets-annotations-update","download","grep","head","image","move","refresh","tail","list","tree") .stream() .map(Candidate::new) .collect(toList()); - private List operationsList = asList( "cellbase-configure","variant-aggregate","variant-annotation-delete","variant-annotation-index","variant-annotation-save","variant-configure","variant-delete","variant-family-aggregate","variant-family-index","variant-index","variant-index-launcher","variant-julie-run","variant-metadata-repair","variant-metadata-synchronize","variant-prune","variant-sample-delete","variant-sample-index","variant-sample-index-configure","variant-score-delete","variant-score-index","variant-secondary-annotation-index","variant-secondary-sample-index","configure-variant-secondary-sample-index","variant-secondary-index","variant-secondary-index-delete","variant-stats-delete","variant-stats-index","variant-study-delete") + private List operationsList = asList( "cellbase-configure","variant-aggregate","variant-annotation-delete","variant-annotation-index","variant-annotation-save","variant-configure","variant-delete","variant-family-aggregate","variant-family-index","variant-index","variant-index-launcher","variant-julie-run","variant-metadata-repair","variant-metadata-synchronize","variant-prune","variant-sample-delete","variant-sample-index","variant-sample-index-configure","variant-score-delete","variant-score-index","variant-secondary-annotation-index","variant-secondary-sample-index","configure-variant-secondary-sample-index","variant-secondary-index","variant-secondary-index-delete","variant-setup","variant-stats-delete","variant-stats-index","variant-study-delete") .stream() .map(Candidate::new) .collect(toList()); - private List cohortsList = asList( "acl-update","aggregationstats","annotation-sets-load","create","distinct","generate","search","acl","delete","info","update","annotation-sets-annotations-update") + private List cohortsList = asList( "acl-update","annotation-sets-load","create","distinct","generate","search","acl","delete","info","update","annotation-sets-annotations-update") .stream() .map(Candidate::new) .collect(toList()); @@ -145,6 +134,7 @@ public void complete(LineReader lineReader, ParsedLine parsedLine, List login(CustomUsersCommandOptions.Logi if (StringUtils.isNotEmpty(user) && StringUtils.isNotEmpty(password)) { AuthenticationResponse response = null; try { - response = openCGAClient.login(user, password); + response = openCGAClient.login(commandOptions.organization, user, password); } catch (Exception e) { logger.debug("Login error", e); Event event = new Event(); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomUsersCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomUsersCommandOptions.java index 2ec514067f7..18f3e29e310 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomUsersCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomUsersCommandOptions.java @@ -54,6 +54,9 @@ public class LoginCommandOptions { @Parameter(names = {"-p", "--password"}, description = "User password", arity = 0, required = true, password = true) public String password; + @Parameter(names = {"-o", "--organization"}, description = "Organization id", arity = 1) + public String organization; + @Parameter(names = {"--refresh-token"}, description = "The body web service refreshToken parameter", arity = 1) public String refreshToken; } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AdminCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AdminCommandExecutor.java index 9126d2a3a7b..99135ede6b0 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AdminCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AdminCommandExecutor.java @@ -14,17 +14,17 @@ import org.opencb.opencga.catalog.utils.ParamUtils.AddRemoveAction; import org.opencb.opencga.client.exceptions.ClientException; import org.opencb.opencga.core.common.JacksonUtils; +import org.opencb.opencga.core.models.Acl; import org.opencb.opencga.core.models.admin.GroupSyncParams; import org.opencb.opencga.core.models.admin.InstallationParams; import org.opencb.opencga.core.models.admin.JWTParams; -import org.opencb.opencga.core.models.admin.UserCreateParams; import org.opencb.opencga.core.models.admin.UserImportParams; import org.opencb.opencga.core.models.admin.UserUpdateGroup; import org.opencb.opencga.core.models.common.Enums.Resource; import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.study.Group; -import org.opencb.opencga.core.models.user.Account; import org.opencb.opencga.core.models.user.User; +import org.opencb.opencga.core.models.user.UserCreateParams; import org.opencb.opencga.core.response.QueryType; import org.opencb.opencga.core.response.RestResponse; @@ -65,9 +65,6 @@ public void execute() throws Exception { case "audit-group-by": queryResponse = groupByAudit(); break; - case "catalog-index-stats": - queryResponse = indexStatsCatalog(); - break; case "catalog-install": queryResponse = installCatalog(); break; @@ -80,6 +77,9 @@ public void execute() throws Exception { case "users-import": queryResponse = importUsers(); break; + case "users-permissions": + queryResponse = permissionsUsers(); + break; case "users-search": queryResponse = searchUsers(); break; @@ -114,17 +114,6 @@ private RestResponse groupByAudit() throws Exception { return openCGAClient.getAdminClient().groupByAudit(commandOptions.fields, commandOptions.entity, queryParams); } - private RestResponse indexStatsCatalog() throws Exception { - logger.debug("Executing indexStatsCatalog in Admin command line"); - - AdminCommandOptions.IndexStatsCatalogCommandOptions commandOptions = adminCommandOptions.indexStatsCatalogCommandOptions; - - ObjectMap queryParams = new ObjectMap(); - queryParams.putIfNotEmpty("collection", commandOptions.collection); - - return openCGAClient.getAdminClient().indexStatsCatalog(queryParams); - } - private RestResponse installCatalog() throws Exception { logger.debug("Executing installCatalog in Admin command line"); @@ -141,10 +130,9 @@ private RestResponse installCatalog() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), InstallationParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "secretKey",commandOptions.secretKey, true); - putNestedIfNotEmpty(beanParams, "password",commandOptions.password, true); - putNestedIfNotEmpty(beanParams, "email",commandOptions.email, true); - putNestedIfNotEmpty(beanParams, "organization",commandOptions.organization, true); + putNestedIfNotEmpty(beanParams, "secretKey", commandOptions.secretKey, true); + putNestedIfNotEmpty(beanParams, "password", commandOptions.password, true); + putNestedIfNotEmpty(beanParams, "email", commandOptions.email, true); installationParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -158,6 +146,10 @@ private RestResponse jwtCatalog() throws Exception { AdminCommandOptions.JwtCatalogCommandOptions commandOptions = adminCommandOptions.jwtCatalogCommandOptions; + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("organization", commandOptions.organization); + + JWTParams jWTParams = null; if (commandOptions.jsonDataModel) { RestResponse res = new RestResponse<>(); @@ -169,13 +161,13 @@ private RestResponse jwtCatalog() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), JWTParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "secretKey",commandOptions.secretKey, true); + putNestedIfNotEmpty(beanParams, "secretKey", commandOptions.secretKey, true); jWTParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) .readValue(beanParams.toJson(), JWTParams.class); } - return openCGAClient.getAdminClient().jwtCatalog(jWTParams); + return openCGAClient.getAdminClient().jwtCatalog(jWTParams, queryParams); } private RestResponse createUsers() throws Exception { @@ -194,12 +186,11 @@ private RestResponse createUsers() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), UserCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "email",commandOptions.email, true); - putNestedIfNotEmpty(beanParams, "password",commandOptions.password, true); - putNestedIfNotEmpty(beanParams, "organization",commandOptions.organization, true); - putNestedIfNotNull(beanParams, "type",commandOptions.type, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "email", commandOptions.email, true); + putNestedIfNotEmpty(beanParams, "password", commandOptions.password, true); + putNestedIfNotEmpty(beanParams, "organization", commandOptions.organization, true); userCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -213,6 +204,10 @@ private RestResponse importUsers() throws Exception { AdminCommandOptions.ImportUsersCommandOptions commandOptions = adminCommandOptions.importUsersCommandOptions; + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("organization", commandOptions.organization); + + UserImportParams userImportParams = null; if (commandOptions.jsonDataModel) { RestResponse res = new RestResponse<>(); @@ -224,17 +219,34 @@ private RestResponse importUsers() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), UserImportParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "authenticationOriginId",commandOptions.authenticationOriginId, true); - putNestedIfNotNull(beanParams, "id",commandOptions.id, true); - putNestedIfNotNull(beanParams, "resourceType",commandOptions.resourceType, true); - putNestedIfNotEmpty(beanParams, "study",commandOptions.study, true); - putNestedIfNotEmpty(beanParams, "studyGroup",commandOptions.studyGroup, true); + putNestedIfNotEmpty(beanParams, "authenticationOriginId", commandOptions.authenticationOriginId, true); + putNestedIfNotNull(beanParams, "id", commandOptions.id, true); + putNestedIfNotNull(beanParams, "resourceType", commandOptions.resourceType, true); + putNestedIfNotEmpty(beanParams, "study", commandOptions.study, true); + putNestedIfNotEmpty(beanParams, "studyGroup", commandOptions.studyGroup, true); userImportParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) .readValue(beanParams.toJson(), UserImportParams.class); } - return openCGAClient.getAdminClient().importUsers(userImportParams); + return openCGAClient.getAdminClient().importUsers(userImportParams, queryParams); + } + + private RestResponse permissionsUsers() throws Exception { + logger.debug("Executing permissionsUsers in Admin command line"); + + AdminCommandOptions.PermissionsUsersCommandOptions commandOptions = adminCommandOptions.permissionsUsersCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("study", commandOptions.study); + queryParams.putIfNotEmpty("entryIds", commandOptions.entryIds); + queryParams.putIfNotEmpty("permissions", commandOptions.permissions); + queryParams.putIfNotEmpty("category", commandOptions.category); + if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { + queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); + } + + return openCGAClient.getAdminClient().permissionsUsers(queryParams); } private RestResponse searchUsers() throws Exception { @@ -248,8 +260,8 @@ private RestResponse searchUsers() throws Exception { queryParams.putIfNotNull("limit", commandOptions.limit); queryParams.putIfNotNull("skip", commandOptions.skip); queryParams.putIfNotNull("count", commandOptions.count); + queryParams.putIfNotEmpty("organization", commandOptions.organization); queryParams.putIfNotEmpty("user", commandOptions.user); - queryParams.putIfNotEmpty("account", commandOptions.account); queryParams.putIfNotEmpty("authenticationId", commandOptions.authenticationId); return openCGAClient.getAdminClient().searchUsers(queryParams); @@ -260,6 +272,10 @@ private RestResponse syncUsers() throws Exception { AdminCommandOptions.SyncUsersCommandOptions commandOptions = adminCommandOptions.syncUsersCommandOptions; + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("organization", commandOptions.organization); + + GroupSyncParams groupSyncParams = null; if (commandOptions.jsonDataModel) { RestResponse res = new RestResponse<>(); @@ -271,19 +287,18 @@ private RestResponse syncUsers() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), GroupSyncParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "authenticationOriginId",commandOptions.authenticationOriginId, true); - putNestedIfNotEmpty(beanParams, "from",commandOptions.from, true); - putNestedIfNotEmpty(beanParams, "to",commandOptions.to, true); - putNestedIfNotEmpty(beanParams, "study",commandOptions.study, true); - putNestedIfNotNull(beanParams, "syncAll",commandOptions.syncAll, true); - putNestedIfNotNull(beanParams, "type",commandOptions.type, true); - putNestedIfNotNull(beanParams, "force",commandOptions.force, true); + putNestedIfNotEmpty(beanParams, "authenticationOriginId", commandOptions.authenticationOriginId, true); + putNestedIfNotEmpty(beanParams, "from", commandOptions.from, true); + putNestedIfNotEmpty(beanParams, "to", commandOptions.to, true); + putNestedIfNotEmpty(beanParams, "study", commandOptions.study, true); + putNestedIfNotNull(beanParams, "syncAll", commandOptions.syncAll, true); + putNestedIfNotNull(beanParams, "force", commandOptions.force, true); groupSyncParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) .readValue(beanParams.toJson(), GroupSyncParams.class); } - return openCGAClient.getAdminClient().syncUsers(groupSyncParams); + return openCGAClient.getAdminClient().syncUsers(groupSyncParams, queryParams); } private RestResponse usersUpdateGroups() throws Exception { @@ -292,6 +307,7 @@ private RestResponse usersUpdateGroups() throws Exception { AdminCommandOptions.UsersUpdateGroupsCommandOptions commandOptions = adminCommandOptions.usersUpdateGroupsCommandOptions; ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("organization", commandOptions.organization); queryParams.putIfNotNull("action", commandOptions.action); @@ -306,8 +322,8 @@ private RestResponse usersUpdateGroups() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), UserUpdateGroup.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "studyIds",commandOptions.studyIds, true); - putNestedIfNotNull(beanParams, "groupIds",commandOptions.groupIds, true); + putNestedIfNotNull(beanParams, "studyIds", commandOptions.studyIds, true); + putNestedIfNotNull(beanParams, "groupIds", commandOptions.groupIds, true); userUpdateGroup = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisAlignmentCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisAlignmentCommandExecutor.java index 88c7345e9da..566389f36b2 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisAlignmentCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisAlignmentCommandExecutor.java @@ -121,6 +121,9 @@ private RestResponse runBwa() throws Exception { queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -137,12 +140,12 @@ private RestResponse runBwa() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), BwaWrapperParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "command",commandOptions.command, true); - putNestedIfNotEmpty(beanParams, "fastaFile",commandOptions.fastaFile, true); - putNestedIfNotEmpty(beanParams, "fastq1File",commandOptions.fastq1File, true); - putNestedIfNotEmpty(beanParams, "fastq2File",commandOptions.fastq2File, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotNull(beanParams, "bwaParams",commandOptions.bwaParams, true); + putNestedIfNotEmpty(beanParams, "command", commandOptions.command, true); + putNestedIfNotEmpty(beanParams, "fastaFile", commandOptions.fastaFile, true); + putNestedIfNotEmpty(beanParams, "fastq1File", commandOptions.fastq1File, true); + putNestedIfNotEmpty(beanParams, "fastq2File", commandOptions.fastq2File, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedMapIfNotEmpty(beanParams, "bwaParams", commandOptions.bwaParams, true); bwaWrapperParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -162,6 +165,9 @@ private RestResponse runCoverageIndex() throws Exception { queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -178,9 +184,9 @@ private RestResponse runCoverageIndex() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), CoverageIndexParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "bamFileId",commandOptions.bamFileId, true); - putNestedIfNotEmpty(beanParams, "baiFileId",commandOptions.baiFileId, true); - putNestedIfNotNull(beanParams, "windowSize",commandOptions.windowSize, true); + putNestedIfNotEmpty(beanParams, "bamFileId", commandOptions.bamFileId, true); + putNestedIfNotEmpty(beanParams, "baiFileId", commandOptions.baiFileId, true); + putNestedIfNotNull(beanParams, "windowSize", commandOptions.windowSize, true); coverageIndexParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -200,6 +206,9 @@ private RestResponse coverageQcGeneCoverageStatsRun() throws Exception { queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -216,9 +225,9 @@ private RestResponse coverageQcGeneCoverageStatsRun() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), AlignmentGeneCoverageStatsParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "bamFile",commandOptions.bamFile, true); - putNestedIfNotNull(beanParams, "genes",commandOptions.genes, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "bamFile", commandOptions.bamFile, true); + putNestedIfNotNull(beanParams, "genes", commandOptions.genes, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); alignmentGeneCoverageStatsParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -295,6 +304,9 @@ private RestResponse runDeeptools() throws Exception { queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -311,9 +323,9 @@ private RestResponse runDeeptools() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), DeeptoolsWrapperParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "command",commandOptions.command, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotNull(beanParams, "deeptoolsParams",commandOptions.deeptoolsParams, true); + putNestedIfNotEmpty(beanParams, "command", commandOptions.command, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedMapIfNotEmpty(beanParams, "deeptoolsParams", commandOptions.deeptoolsParams, true); deeptoolsWrapperParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -333,6 +345,9 @@ private RestResponse runFastqc() throws Exception { queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -349,9 +364,9 @@ private RestResponse runFastqc() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), FastqcWrapperParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "inputFile",commandOptions.inputFile, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotNull(beanParams, "fastqcParams",commandOptions.fastqcParams, true); + putNestedIfNotEmpty(beanParams, "inputFile", commandOptions.inputFile, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedMapIfNotEmpty(beanParams, "fastqcParams", commandOptions.fastqcParams, true); fastqcWrapperParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -371,6 +386,9 @@ private RestResponse runIndex() throws Exception { queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -387,8 +405,8 @@ private RestResponse runIndex() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), AlignmentIndexParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "fileId",commandOptions.fileId, true); - putNestedIfNotNull(beanParams, "overwrite",commandOptions.overwrite, true); + putNestedIfNotEmpty(beanParams, "fileId", commandOptions.fileId, true); + putNestedIfNotNull(beanParams, "overwrite", commandOptions.overwrite, true); alignmentIndexParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -408,6 +426,9 @@ private RestResponse runPicard() throws Exception { queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -424,9 +445,9 @@ private RestResponse runPicard() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), PicardWrapperParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "command",commandOptions.command, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotNull(beanParams, "picardParams",commandOptions.picardParams, true); + putNestedIfNotEmpty(beanParams, "command", commandOptions.command, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedMapIfNotEmpty(beanParams, "picardParams", commandOptions.picardParams, true); picardWrapperParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -446,6 +467,9 @@ private RestResponse runQc() throws Exception { queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -462,10 +486,10 @@ private RestResponse runQc() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), AlignmentQcParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "bamFile",commandOptions.bamFile, true); - putNestedIfNotEmpty(beanParams, "skip",commandOptions.skip, true); - putNestedIfNotNull(beanParams, "overwrite",commandOptions.overwrite, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "bamFile", commandOptions.bamFile, true); + putNestedIfNotEmpty(beanParams, "skip", commandOptions.skip, true); + putNestedIfNotNull(beanParams, "overwrite", commandOptions.overwrite, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); alignmentQcParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -517,6 +541,9 @@ private RestResponse runSamtools() throws Exception { queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -533,10 +560,10 @@ private RestResponse runSamtools() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), SamtoolsWrapperParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "command",commandOptions.command, true); - putNestedIfNotEmpty(beanParams, "inputFile",commandOptions.inputFile, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotNull(beanParams, "samtoolsParams",commandOptions.samtoolsParams, true); + putNestedIfNotEmpty(beanParams, "command", commandOptions.command, true); + putNestedIfNotEmpty(beanParams, "inputFile", commandOptions.inputFile, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedMapIfNotEmpty(beanParams, "samtoolsParams", commandOptions.samtoolsParams, true); samtoolsWrapperParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisClinicalCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisClinicalCommandExecutor.java index 22df7f71de9..07dbf9230f9 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisClinicalCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisClinicalCommandExecutor.java @@ -35,6 +35,7 @@ import org.opencb.opencga.core.models.clinical.ClinicalAnalysisAclEntryList; import org.opencb.opencga.core.models.clinical.ClinicalAnalysisAclUpdateParams; import org.opencb.opencga.core.models.clinical.ClinicalAnalysisCreateParams; +import org.opencb.opencga.core.models.clinical.ClinicalAnalysisLoadParams; import org.opencb.opencga.core.models.clinical.ClinicalAnalysisQualityControl; import org.opencb.opencga.core.models.clinical.ClinicalAnalysisQualityControlUpdateParam; import org.opencb.opencga.core.models.clinical.ClinicalAnalysisUpdateParams; @@ -138,6 +139,9 @@ public void execute() throws Exception { case "interpreter-zetta-run": queryResponse = runInterpreterZetta(); break; + case "load": + queryResponse = load(); + break; case "rga-aggregation-stats": queryResponse = aggregationStatsRga(); break; @@ -234,8 +238,8 @@ private RestResponse updateAcl() throws Exception .readValue(new java.io.File(commandOptions.jsonFile), ClinicalAnalysisAclUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "permissions",commandOptions.permissions, true); - putNestedIfNotEmpty(beanParams, "clinicalAnalysis",commandOptions.clinicalAnalysis, true); + putNestedIfNotEmpty(beanParams, "permissions", commandOptions.permissions, true); + putNestedIfNotEmpty(beanParams, "clinicalAnalysis", commandOptions.clinicalAnalysis, true); clinicalAnalysisAclUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -269,7 +273,7 @@ private RestResponse loadAnnotationSets() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), TsvAnnotationParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "content",commandOptions.content, true); + putNestedIfNotEmpty(beanParams, "content", commandOptions.content, true); tsvAnnotationParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -301,7 +305,7 @@ private RestResponse updateClinicalConfiguration() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), ClinicalAnalysisStudyConfiguration.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "interpretation.defaultFilter",commandOptions.interpretationDefaultFilter, true); + putNestedMapIfNotEmpty(beanParams, "interpretation.defaultFilter", commandOptions.interpretationDefaultFilter, true); clinicalAnalysisStudyConfiguration = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -337,47 +341,48 @@ private RestResponse create() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), ClinicalAnalysisCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotNull(beanParams, "type",commandOptions.type, true); - putNestedIfNotEmpty(beanParams, "disorder.id",commandOptions.disorderId, true); - putNestedIfNotEmpty(beanParams, "proband.id",commandOptions.probandId, true); - putNestedIfNotEmpty(beanParams, "family.id",commandOptions.familyId, true); - putNestedIfNotNull(beanParams, "panelLock",commandOptions.panelLock, true); - putNestedIfNotEmpty(beanParams, "analyst.id",commandOptions.analystId, true); - putNestedIfNotEmpty(beanParams, "report.title",commandOptions.reportTitle, true); - putNestedIfNotEmpty(beanParams, "report.overview",commandOptions.reportOverview, true); - putNestedIfNotEmpty(beanParams, "report.logo",commandOptions.reportLogo, true); - putNestedIfNotEmpty(beanParams, "report.signedBy",commandOptions.reportSignedBy, true); - putNestedIfNotEmpty(beanParams, "report.signature",commandOptions.reportSignature, true); - putNestedIfNotEmpty(beanParams, "report.date",commandOptions.reportDate, true); - putNestedIfNotEmpty(beanParams, "request.id",commandOptions.requestId, true); - putNestedIfNotEmpty(beanParams, "request.justification",commandOptions.requestJustification, true); - putNestedIfNotEmpty(beanParams, "request.date",commandOptions.requestDate, true); - putNestedIfNotNull(beanParams, "request.attributes",commandOptions.requestAttributes, true); - putNestedIfNotEmpty(beanParams, "responsible.id",commandOptions.responsibleId, true); - putNestedIfNotEmpty(beanParams, "responsible.name",commandOptions.responsibleName, true); - putNestedIfNotEmpty(beanParams, "responsible.email",commandOptions.responsibleEmail, true); - putNestedIfNotEmpty(beanParams, "responsible.organization",commandOptions.responsibleOrganization, true); - putNestedIfNotEmpty(beanParams, "responsible.department",commandOptions.responsibleDepartment, true); - putNestedIfNotEmpty(beanParams, "responsible.address",commandOptions.responsibleAddress, true); - putNestedIfNotEmpty(beanParams, "responsible.city",commandOptions.responsibleCity, true); - putNestedIfNotEmpty(beanParams, "responsible.postcode",commandOptions.responsiblePostcode, true); - putNestedIfNotEmpty(beanParams, "interpretation.description",commandOptions.interpretationDescription, true); - putNestedIfNotEmpty(beanParams, "interpretation.clinicalAnalysisId",commandOptions.interpretationClinicalAnalysisId, true); - putNestedIfNotEmpty(beanParams, "interpretation.creationDate",commandOptions.interpretationCreationDate, true); - putNestedIfNotEmpty(beanParams, "interpretation.modificationDate",commandOptions.interpretationModificationDate, true); - putNestedIfNotNull(beanParams, "interpretation.locked",commandOptions.interpretationLocked, true); - putNestedIfNotNull(beanParams, "interpretation.attributes",commandOptions.interpretationAttributes, true); - putNestedIfNotNull(beanParams, "qualityControl.summary",commandOptions.qualityControlSummary, true); - putNestedIfNotNull(beanParams, "qualityControl.comments",commandOptions.qualityControlComments, true); - putNestedIfNotNull(beanParams, "qualityControl.files",commandOptions.qualityControlFiles, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "dueDate",commandOptions.dueDate, true); - putNestedIfNotEmpty(beanParams, "priority.id",commandOptions.priorityId, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotNull(beanParams, "type", commandOptions.type, true); + putNestedIfNotEmpty(beanParams, "disorder.id", commandOptions.disorderId, true); + putNestedIfNotEmpty(beanParams, "proband.id", commandOptions.probandId, true); + putNestedIfNotEmpty(beanParams, "family.id", commandOptions.familyId, true); + putNestedIfNotNull(beanParams, "panelLocked", commandOptions.panelLocked, true); + putNestedIfNotEmpty(beanParams, "analyst.id", commandOptions.analystId, true); + putNestedIfNotEmpty(beanParams, "report.title", commandOptions.reportTitle, true); + putNestedIfNotEmpty(beanParams, "report.overview", commandOptions.reportOverview, true); + putNestedIfNotEmpty(beanParams, "report.logo", commandOptions.reportLogo, true); + putNestedIfNotEmpty(beanParams, "report.signedBy", commandOptions.reportSignedBy, true); + putNestedIfNotEmpty(beanParams, "report.signature", commandOptions.reportSignature, true); + putNestedIfNotEmpty(beanParams, "report.date", commandOptions.reportDate, true); + putNestedIfNotEmpty(beanParams, "request.id", commandOptions.requestId, true); + putNestedIfNotEmpty(beanParams, "request.justification", commandOptions.requestJustification, true); + putNestedIfNotEmpty(beanParams, "request.date", commandOptions.requestDate, true); + putNestedMapIfNotEmpty(beanParams, "request.attributes", commandOptions.requestAttributes, true); + putNestedIfNotEmpty(beanParams, "responsible.id", commandOptions.responsibleId, true); + putNestedIfNotEmpty(beanParams, "responsible.name", commandOptions.responsibleName, true); + putNestedIfNotEmpty(beanParams, "responsible.email", commandOptions.responsibleEmail, true); + putNestedIfNotEmpty(beanParams, "responsible.organization", commandOptions.responsibleOrganization, true); + putNestedIfNotEmpty(beanParams, "responsible.department", commandOptions.responsibleDepartment, true); + putNestedIfNotEmpty(beanParams, "responsible.address", commandOptions.responsibleAddress, true); + putNestedIfNotEmpty(beanParams, "responsible.city", commandOptions.responsibleCity, true); + putNestedIfNotEmpty(beanParams, "responsible.postcode", commandOptions.responsiblePostcode, true); + putNestedIfNotEmpty(beanParams, "interpretation.name", commandOptions.interpretationName, true); + putNestedIfNotEmpty(beanParams, "interpretation.description", commandOptions.interpretationDescription, true); + putNestedIfNotEmpty(beanParams, "interpretation.clinicalAnalysisId", commandOptions.interpretationClinicalAnalysisId, true); + putNestedIfNotEmpty(beanParams, "interpretation.creationDate", commandOptions.interpretationCreationDate, true); + putNestedIfNotEmpty(beanParams, "interpretation.modificationDate", commandOptions.interpretationModificationDate, true); + putNestedIfNotNull(beanParams, "interpretation.locked", commandOptions.interpretationLocked, true); + putNestedMapIfNotEmpty(beanParams, "interpretation.attributes", commandOptions.interpretationAttributes, true); + putNestedIfNotNull(beanParams, "qualityControl.summary", commandOptions.qualityControlSummary, true); + putNestedIfNotNull(beanParams, "qualityControl.comments", commandOptions.qualityControlComments, true); + putNestedIfNotNull(beanParams, "qualityControl.files", commandOptions.qualityControlFiles, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "dueDate", commandOptions.dueDate, true); + putNestedIfNotEmpty(beanParams, "priority.id", commandOptions.priorityId, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); clinicalAnalysisCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -415,6 +420,7 @@ private RestResponse distinct() throws Exception { queryParams.putIfNotEmpty("dueDate", commandOptions.dueDate); queryParams.putIfNotEmpty("qualityControlSummary", commandOptions.qualityControlSummary); queryParams.putIfNotEmpty("release", commandOptions.release); + queryParams.putIfNotNull("snapshot", commandOptions.snapshot); queryParams.putIfNotEmpty("status", commandOptions.status); queryParams.putIfNotEmpty("internalStatus", commandOptions.internalStatus); queryParams.putIfNotEmpty("annotation", commandOptions.annotation); @@ -435,6 +441,7 @@ private RestResponse distinctInterpretation() throws Exception { queryParams.putIfNotEmpty("study", commandOptions.study); queryParams.putIfNotEmpty("id", commandOptions.id); queryParams.putIfNotEmpty("uuid", commandOptions.uuid); + queryParams.putIfNotEmpty("name", commandOptions.name); queryParams.putIfNotEmpty("clinicalAnalysisId", commandOptions.clinicalAnalysisId); queryParams.putIfNotEmpty("analystId", commandOptions.analystId); queryParams.putIfNotEmpty("methodName", commandOptions.methodName); @@ -467,6 +474,7 @@ private RestResponse searchInterpretation() throws Exception { queryParams.putIfNotEmpty("study", commandOptions.study); queryParams.putIfNotEmpty("id", commandOptions.id); queryParams.putIfNotEmpty("uuid", commandOptions.uuid); + queryParams.putIfNotEmpty("name", commandOptions.name); queryParams.putIfNotEmpty("clinicalAnalysisId", commandOptions.clinicalAnalysisId); queryParams.putIfNotEmpty("analystId", commandOptions.analystId); queryParams.putIfNotEmpty("methodName", commandOptions.methodName); @@ -514,6 +522,9 @@ private RestResponse runInterpreterCancerTiering() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -530,9 +541,9 @@ private RestResponse runInterpreterCancerTiering() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), CancerTieringInterpretationAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "clinicalAnalysis",commandOptions.clinicalAnalysis, true); - putNestedIfNotNull(beanParams, "discardedVariants",commandOptions.discardedVariants, true); - putNestedIfNotNull(beanParams, "primary",commandOptions.primary, true); + putNestedIfNotEmpty(beanParams, "clinicalAnalysis", commandOptions.clinicalAnalysis, true); + putNestedIfNotNull(beanParams, "discardedVariants", commandOptions.discardedVariants, true); + putNestedIfNotNull(beanParams, "primary", commandOptions.primary, true); cancerTieringInterpretationAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -552,6 +563,9 @@ private RestResponse runInterpreterExomiser() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -568,7 +582,8 @@ private RestResponse runInterpreterExomiser() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), ExomiserInterpretationAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "clinicalAnalysis",commandOptions.clinicalAnalysis, true); + putNestedIfNotEmpty(beanParams, "clinicalAnalysis", commandOptions.clinicalAnalysis, true); + putNestedIfNotEmpty(beanParams, "exomiserVersion", commandOptions.exomiserVersion, true); exomiserInterpretationAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -588,6 +603,9 @@ private RestResponse runInterpreterTeam() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -604,10 +622,10 @@ private RestResponse runInterpreterTeam() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), TeamInterpretationAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "clinicalAnalysis",commandOptions.clinicalAnalysis, true); - putNestedIfNotNull(beanParams, "panels",commandOptions.panels, true); - putNestedIfNotEmpty(beanParams, "familySegregation",commandOptions.familySegregation, true); - putNestedIfNotNull(beanParams, "primary",commandOptions.primary, true); + putNestedIfNotEmpty(beanParams, "clinicalAnalysis", commandOptions.clinicalAnalysis, true); + putNestedIfNotNull(beanParams, "panels", commandOptions.panels, true); + putNestedIfNotEmpty(beanParams, "familySegregation", commandOptions.familySegregation, true); + putNestedIfNotNull(beanParams, "primary", commandOptions.primary, true); teamInterpretationAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -627,6 +645,9 @@ private RestResponse runInterpreterTiering() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -643,10 +664,10 @@ private RestResponse runInterpreterTiering() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), TieringInterpretationAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "clinicalAnalysis",commandOptions.clinicalAnalysis, true); - putNestedIfNotNull(beanParams, "panels",commandOptions.panels, true); - putNestedIfNotNull(beanParams, "penetrance",commandOptions.penetrance, true); - putNestedIfNotNull(beanParams, "primary",commandOptions.primary, true); + putNestedIfNotEmpty(beanParams, "clinicalAnalysis", commandOptions.clinicalAnalysis, true); + putNestedIfNotNull(beanParams, "panels", commandOptions.panels, true); + putNestedIfNotNull(beanParams, "penetrance", commandOptions.penetrance, true); + putNestedIfNotNull(beanParams, "primary", commandOptions.primary, true); tieringInterpretationAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -666,6 +687,9 @@ private RestResponse runInterpreterZetta() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -682,57 +706,57 @@ private RestResponse runInterpreterZetta() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), ZettaInterpretationAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "clinicalAnalysis",commandOptions.clinicalAnalysis, true); - putNestedIfNotNull(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "region",commandOptions.region, true); - putNestedIfNotEmpty(beanParams, "type",commandOptions.type, true); - putNestedIfNotEmpty(beanParams, "study",commandOptions.bodyStudy, true); - putNestedIfNotEmpty(beanParams, "file",commandOptions.file, true); - putNestedIfNotEmpty(beanParams, "filter",commandOptions.filter, true); - putNestedIfNotEmpty(beanParams, "qual",commandOptions.qual, true); - putNestedIfNotEmpty(beanParams, "fileData",commandOptions.fileData, true); - putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotEmpty(beanParams, "sampleData",commandOptions.sampleData, true); - putNestedIfNotEmpty(beanParams, "sampleAnnotation",commandOptions.sampleAnnotation, true); - putNestedIfNotEmpty(beanParams, "sampleMetadata",commandOptions.sampleMetadata, true); - putNestedIfNotEmpty(beanParams, "cohort",commandOptions.cohort, true); - putNestedIfNotEmpty(beanParams, "cohortStatsRef",commandOptions.cohortStatsRef, true); - putNestedIfNotEmpty(beanParams, "cohortStatsAlt",commandOptions.cohortStatsAlt, true); - putNestedIfNotEmpty(beanParams, "cohortStatsMaf",commandOptions.cohortStatsMaf, true); - putNestedIfNotEmpty(beanParams, "cohortStatsMgf",commandOptions.cohortStatsMgf, true); - putNestedIfNotEmpty(beanParams, "cohortStatsPass",commandOptions.cohortStatsPass, true); - putNestedIfNotEmpty(beanParams, "score",commandOptions.score, true); - putNestedIfNotEmpty(beanParams, "family",commandOptions.family, true); - putNestedIfNotEmpty(beanParams, "familyDisorder",commandOptions.familyDisorder, true); - putNestedIfNotEmpty(beanParams, "familySegregation",commandOptions.familySegregation, true); - putNestedIfNotEmpty(beanParams, "familyMembers",commandOptions.familyMembers, true); - putNestedIfNotEmpty(beanParams, "familyProband",commandOptions.familyProband, true); - putNestedIfNotEmpty(beanParams, "gene",commandOptions.gene, true); - putNestedIfNotEmpty(beanParams, "ct",commandOptions.ct, true); - putNestedIfNotEmpty(beanParams, "xref",commandOptions.xref, true); - putNestedIfNotEmpty(beanParams, "biotype",commandOptions.biotype, true); - putNestedIfNotEmpty(beanParams, "proteinSubstitution",commandOptions.proteinSubstitution, true); - putNestedIfNotEmpty(beanParams, "conservation",commandOptions.conservation, true); - putNestedIfNotEmpty(beanParams, "populationFrequencyAlt",commandOptions.populationFrequencyAlt, true); - putNestedIfNotEmpty(beanParams, "populationFrequencyRef",commandOptions.populationFrequencyRef, true); - putNestedIfNotEmpty(beanParams, "populationFrequencyMaf",commandOptions.populationFrequencyMaf, true); - putNestedIfNotEmpty(beanParams, "transcriptFlag",commandOptions.transcriptFlag, true); - putNestedIfNotEmpty(beanParams, "geneTraitId",commandOptions.geneTraitId, true); - putNestedIfNotEmpty(beanParams, "go",commandOptions.go, true); - putNestedIfNotEmpty(beanParams, "expression",commandOptions.expression, true); - putNestedIfNotEmpty(beanParams, "proteinKeyword",commandOptions.proteinKeyword, true); - putNestedIfNotEmpty(beanParams, "drug",commandOptions.drug, true); - putNestedIfNotEmpty(beanParams, "functionalScore",commandOptions.functionalScore, true); - putNestedIfNotEmpty(beanParams, "clinical",commandOptions.clinical, true); - putNestedIfNotEmpty(beanParams, "clinicalSignificance",commandOptions.clinicalSignificance, true); - putNestedIfNotNull(beanParams, "clinicalConfirmedStatus",commandOptions.clinicalConfirmedStatus, true); - putNestedIfNotEmpty(beanParams, "customAnnotation",commandOptions.customAnnotation, true); - putNestedIfNotEmpty(beanParams, "panel",commandOptions.panel, true); - putNestedIfNotEmpty(beanParams, "panelModeOfInheritance",commandOptions.panelModeOfInheritance, true); - putNestedIfNotEmpty(beanParams, "panelConfidence",commandOptions.panelConfidence, true); - putNestedIfNotEmpty(beanParams, "panelRoleInCancer",commandOptions.panelRoleInCancer, true); - putNestedIfNotEmpty(beanParams, "trait",commandOptions.trait, true); - putNestedIfNotNull(beanParams, "primary",commandOptions.primary, true); + putNestedIfNotEmpty(beanParams, "clinicalAnalysis", commandOptions.clinicalAnalysis, true); + putNestedIfNotNull(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "region", commandOptions.region, true); + putNestedIfNotEmpty(beanParams, "type", commandOptions.type, true); + putNestedIfNotEmpty(beanParams, "study", commandOptions.bodyStudy, true); + putNestedIfNotEmpty(beanParams, "file", commandOptions.file, true); + putNestedIfNotEmpty(beanParams, "filter", commandOptions.filter, true); + putNestedIfNotEmpty(beanParams, "qual", commandOptions.qual, true); + putNestedIfNotEmpty(beanParams, "fileData", commandOptions.fileData, true); + putNestedIfNotEmpty(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotEmpty(beanParams, "sampleData", commandOptions.sampleData, true); + putNestedIfNotEmpty(beanParams, "sampleAnnotation", commandOptions.sampleAnnotation, true); + putNestedIfNotEmpty(beanParams, "sampleMetadata", commandOptions.sampleMetadata, true); + putNestedIfNotEmpty(beanParams, "cohort", commandOptions.cohort, true); + putNestedIfNotEmpty(beanParams, "cohortStatsRef", commandOptions.cohortStatsRef, true); + putNestedIfNotEmpty(beanParams, "cohortStatsAlt", commandOptions.cohortStatsAlt, true); + putNestedIfNotEmpty(beanParams, "cohortStatsMaf", commandOptions.cohortStatsMaf, true); + putNestedIfNotEmpty(beanParams, "cohortStatsMgf", commandOptions.cohortStatsMgf, true); + putNestedIfNotEmpty(beanParams, "cohortStatsPass", commandOptions.cohortStatsPass, true); + putNestedIfNotEmpty(beanParams, "score", commandOptions.score, true); + putNestedIfNotEmpty(beanParams, "family", commandOptions.family, true); + putNestedIfNotEmpty(beanParams, "familyDisorder", commandOptions.familyDisorder, true); + putNestedIfNotEmpty(beanParams, "familySegregation", commandOptions.familySegregation, true); + putNestedIfNotEmpty(beanParams, "familyMembers", commandOptions.familyMembers, true); + putNestedIfNotEmpty(beanParams, "familyProband", commandOptions.familyProband, true); + putNestedIfNotEmpty(beanParams, "gene", commandOptions.gene, true); + putNestedIfNotEmpty(beanParams, "ct", commandOptions.ct, true); + putNestedIfNotEmpty(beanParams, "xref", commandOptions.xref, true); + putNestedIfNotEmpty(beanParams, "biotype", commandOptions.biotype, true); + putNestedIfNotEmpty(beanParams, "proteinSubstitution", commandOptions.proteinSubstitution, true); + putNestedIfNotEmpty(beanParams, "conservation", commandOptions.conservation, true); + putNestedIfNotEmpty(beanParams, "populationFrequencyAlt", commandOptions.populationFrequencyAlt, true); + putNestedIfNotEmpty(beanParams, "populationFrequencyRef", commandOptions.populationFrequencyRef, true); + putNestedIfNotEmpty(beanParams, "populationFrequencyMaf", commandOptions.populationFrequencyMaf, true); + putNestedIfNotEmpty(beanParams, "transcriptFlag", commandOptions.transcriptFlag, true); + putNestedIfNotEmpty(beanParams, "geneTraitId", commandOptions.geneTraitId, true); + putNestedIfNotEmpty(beanParams, "go", commandOptions.go, true); + putNestedIfNotEmpty(beanParams, "expression", commandOptions.expression, true); + putNestedIfNotEmpty(beanParams, "proteinKeyword", commandOptions.proteinKeyword, true); + putNestedIfNotEmpty(beanParams, "drug", commandOptions.drug, true); + putNestedIfNotEmpty(beanParams, "functionalScore", commandOptions.functionalScore, true); + putNestedIfNotEmpty(beanParams, "clinical", commandOptions.clinical, true); + putNestedIfNotEmpty(beanParams, "clinicalSignificance", commandOptions.clinicalSignificance, true); + putNestedIfNotNull(beanParams, "clinicalConfirmedStatus", commandOptions.clinicalConfirmedStatus, true); + putNestedIfNotEmpty(beanParams, "customAnnotation", commandOptions.customAnnotation, true); + putNestedIfNotEmpty(beanParams, "panel", commandOptions.panel, true); + putNestedIfNotEmpty(beanParams, "panelModeOfInheritance", commandOptions.panelModeOfInheritance, true); + putNestedIfNotEmpty(beanParams, "panelConfidence", commandOptions.panelConfidence, true); + putNestedIfNotEmpty(beanParams, "panelRoleInCancer", commandOptions.panelRoleInCancer, true); + putNestedIfNotEmpty(beanParams, "trait", commandOptions.trait, true); + putNestedIfNotNull(beanParams, "primary", commandOptions.primary, true); zettaInterpretationAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -741,6 +765,45 @@ private RestResponse runInterpreterZetta() throws Exception { return openCGAClient.getClinicalAnalysisClient().runInterpreterZetta(zettaInterpretationAnalysisParams, queryParams); } + private RestResponse load() throws Exception { + logger.debug("Executing load in Analysis - Clinical command line"); + + AnalysisClinicalCommandOptions.LoadCommandOptions commandOptions = analysisClinicalCommandOptions.loadCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("study", commandOptions.study); + queryParams.putIfNotEmpty("jobId", commandOptions.jobId); + queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); + queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); + queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); + if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { + queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); + } + + + ClinicalAnalysisLoadParams clinicalAnalysisLoadParams = null; + if (commandOptions.jsonDataModel) { + RestResponse res = new RestResponse<>(); + res.setType(QueryType.VOID); + PrintUtils.println(getObjectAsJSON(categoryName,"/{apiVersion}/analysis/clinical/load")); + return res; + } else if (commandOptions.jsonFile != null) { + clinicalAnalysisLoadParams = JacksonUtils.getDefaultObjectMapper() + .readValue(new java.io.File(commandOptions.jsonFile), ClinicalAnalysisLoadParams.class); + } else { + ObjectMap beanParams = new ObjectMap(); + putNestedIfNotEmpty(beanParams, "file", commandOptions.file, true); + + clinicalAnalysisLoadParams = JacksonUtils.getDefaultObjectMapper().copy() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) + .readValue(beanParams.toJson(), ClinicalAnalysisLoadParams.class); + } + return openCGAClient.getClinicalAnalysisClient().load(clinicalAnalysisLoadParams, queryParams); + } + private RestResponse aggregationStatsRga() throws Exception { logger.debug("Executing aggregationStatsRga in Analysis - Clinical command line"); @@ -867,6 +930,9 @@ private RestResponse runRgaIndex() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotNull("auxiliarIndex", commandOptions.auxiliarIndex); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); @@ -884,7 +950,7 @@ private RestResponse runRgaIndex() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), RgaAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "file",commandOptions.file, true); + putNestedIfNotEmpty(beanParams, "file", commandOptions.file, true); rgaAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1083,6 +1149,7 @@ private RestResponse search() throws Exception { queryParams.putIfNotEmpty("dueDate", commandOptions.dueDate); queryParams.putIfNotEmpty("qualityControlSummary", commandOptions.qualityControlSummary); queryParams.putIfNotEmpty("release", commandOptions.release); + queryParams.putIfNotNull("snapshot", commandOptions.snapshot); queryParams.putIfNotEmpty("status", commandOptions.status); queryParams.putIfNotEmpty("internalStatus", commandOptions.internalStatus); queryParams.putIfNotEmpty("annotation", commandOptions.annotation); @@ -1227,42 +1294,43 @@ private RestResponse update() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), ClinicalAnalysisUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotNull(beanParams, "type",commandOptions.type, true); - putNestedIfNotEmpty(beanParams, "disorder.id",commandOptions.disorderId, true); - putNestedIfNotNull(beanParams, "panelLock",commandOptions.panelLock, true); - putNestedIfNotEmpty(beanParams, "proband.id",commandOptions.probandId, true); - putNestedIfNotEmpty(beanParams, "family.id",commandOptions.familyId, true); - putNestedIfNotNull(beanParams, "locked",commandOptions.locked, true); - putNestedIfNotEmpty(beanParams, "analyst.id",commandOptions.analystId, true); - putNestedIfNotEmpty(beanParams, "report.title",commandOptions.reportTitle, true); - putNestedIfNotEmpty(beanParams, "report.overview",commandOptions.reportOverview, true); - putNestedIfNotEmpty(beanParams, "report.logo",commandOptions.reportLogo, true); - putNestedIfNotEmpty(beanParams, "report.signedBy",commandOptions.reportSignedBy, true); - putNestedIfNotEmpty(beanParams, "report.signature",commandOptions.reportSignature, true); - putNestedIfNotEmpty(beanParams, "report.date",commandOptions.reportDate, true); - putNestedIfNotEmpty(beanParams, "request.id",commandOptions.requestId, true); - putNestedIfNotEmpty(beanParams, "request.justification",commandOptions.requestJustification, true); - putNestedIfNotEmpty(beanParams, "request.date",commandOptions.requestDate, true); - putNestedIfNotNull(beanParams, "request.attributes",commandOptions.requestAttributes, true); - putNestedIfNotEmpty(beanParams, "responsible.id",commandOptions.responsibleId, true); - putNestedIfNotEmpty(beanParams, "responsible.name",commandOptions.responsibleName, true); - putNestedIfNotEmpty(beanParams, "responsible.email",commandOptions.responsibleEmail, true); - putNestedIfNotEmpty(beanParams, "responsible.organization",commandOptions.responsibleOrganization, true); - putNestedIfNotEmpty(beanParams, "responsible.department",commandOptions.responsibleDepartment, true); - putNestedIfNotEmpty(beanParams, "responsible.address",commandOptions.responsibleAddress, true); - putNestedIfNotEmpty(beanParams, "responsible.city",commandOptions.responsibleCity, true); - putNestedIfNotEmpty(beanParams, "responsible.postcode",commandOptions.responsiblePostcode, true); - putNestedIfNotNull(beanParams, "qualityControl.summary",commandOptions.qualityControlSummary, true); - putNestedIfNotNull(beanParams, "qualityControl.comments",commandOptions.qualityControlComments, true); - putNestedIfNotNull(beanParams, "qualityControl.files",commandOptions.qualityControlFiles, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "dueDate",commandOptions.dueDate, true); - putNestedIfNotEmpty(beanParams, "priority.id",commandOptions.priorityId, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotNull(beanParams, "type", commandOptions.type, true); + putNestedIfNotEmpty(beanParams, "disorder.id", commandOptions.disorderId, true); + putNestedIfNotNull(beanParams, "panelLocked", commandOptions.panelLocked, true); + putNestedIfNotEmpty(beanParams, "proband.id", commandOptions.probandId, true); + putNestedIfNotEmpty(beanParams, "family.id", commandOptions.familyId, true); + putNestedIfNotNull(beanParams, "locked", commandOptions.locked, true); + putNestedIfNotEmpty(beanParams, "analyst.id", commandOptions.analystId, true); + putNestedIfNotEmpty(beanParams, "report.title", commandOptions.reportTitle, true); + putNestedIfNotEmpty(beanParams, "report.overview", commandOptions.reportOverview, true); + putNestedIfNotEmpty(beanParams, "report.logo", commandOptions.reportLogo, true); + putNestedIfNotEmpty(beanParams, "report.signedBy", commandOptions.reportSignedBy, true); + putNestedIfNotEmpty(beanParams, "report.signature", commandOptions.reportSignature, true); + putNestedIfNotEmpty(beanParams, "report.date", commandOptions.reportDate, true); + putNestedIfNotEmpty(beanParams, "request.id", commandOptions.requestId, true); + putNestedIfNotEmpty(beanParams, "request.justification", commandOptions.requestJustification, true); + putNestedIfNotEmpty(beanParams, "request.date", commandOptions.requestDate, true); + putNestedMapIfNotEmpty(beanParams, "request.attributes", commandOptions.requestAttributes, true); + putNestedIfNotEmpty(beanParams, "responsible.id", commandOptions.responsibleId, true); + putNestedIfNotEmpty(beanParams, "responsible.name", commandOptions.responsibleName, true); + putNestedIfNotEmpty(beanParams, "responsible.email", commandOptions.responsibleEmail, true); + putNestedIfNotEmpty(beanParams, "responsible.organization", commandOptions.responsibleOrganization, true); + putNestedIfNotEmpty(beanParams, "responsible.department", commandOptions.responsibleDepartment, true); + putNestedIfNotEmpty(beanParams, "responsible.address", commandOptions.responsibleAddress, true); + putNestedIfNotEmpty(beanParams, "responsible.city", commandOptions.responsibleCity, true); + putNestedIfNotEmpty(beanParams, "responsible.postcode", commandOptions.responsiblePostcode, true); + putNestedIfNotNull(beanParams, "qualityControl.summary", commandOptions.qualityControlSummary, true); + putNestedIfNotNull(beanParams, "qualityControl.comments", commandOptions.qualityControlComments, true); + putNestedIfNotNull(beanParams, "qualityControl.files", commandOptions.qualityControlFiles, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "dueDate", commandOptions.dueDate, true); + putNestedIfNotEmpty(beanParams, "priority.id", commandOptions.priorityId, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotNull(beanParams, "panelLock", commandOptions.panelLock, true); clinicalAnalysisUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1307,6 +1375,7 @@ private RestResponse info() throws Exception { queryParams.putIfNotEmpty("exclude", commandOptions.exclude); queryParams.putIfNotNull("flattenAnnotations", commandOptions.flattenAnnotations); queryParams.putIfNotEmpty("study", commandOptions.study); + queryParams.putIfNotEmpty("version", commandOptions.version); queryParams.putIfNotNull("deleted", commandOptions.deleted); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); @@ -1342,17 +1411,18 @@ private RestResponse createInterpretation() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), InterpretationCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "clinicalAnalysisId",commandOptions.clinicalAnalysisId, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "analyst.id",commandOptions.analystId, true); - putNestedIfNotEmpty(beanParams, "method.name",commandOptions.methodName, true); - putNestedIfNotEmpty(beanParams, "method.version",commandOptions.methodVersion, true); - putNestedIfNotEmpty(beanParams, "method.commit",commandOptions.methodCommit, true); - putNestedIfNotNull(beanParams, "locked",commandOptions.locked, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "clinicalAnalysisId", commandOptions.clinicalAnalysisId, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "analyst.id", commandOptions.analystId, true); + putNestedIfNotEmpty(beanParams, "method.name", commandOptions.methodName, true); + putNestedIfNotEmpty(beanParams, "method.version", commandOptions.methodVersion, true); + putNestedIfNotEmpty(beanParams, "method.commit", commandOptions.methodCommit, true); + putNestedIfNotNull(beanParams, "locked", commandOptions.locked, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); interpretationCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1431,16 +1501,17 @@ private RestResponse updateInterpretation() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), InterpretationUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "analyst.id",commandOptions.analystId, true); - putNestedIfNotEmpty(beanParams, "method.name",commandOptions.methodName, true); - putNestedIfNotEmpty(beanParams, "method.version",commandOptions.methodVersion, true); - putNestedIfNotEmpty(beanParams, "method.commit",commandOptions.methodCommit, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotNull(beanParams, "locked",commandOptions.locked, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "analyst.id", commandOptions.analystId, true); + putNestedIfNotEmpty(beanParams, "method.name", commandOptions.methodName, true); + putNestedIfNotEmpty(beanParams, "method.version", commandOptions.methodVersion, true); + putNestedIfNotEmpty(beanParams, "method.commit", commandOptions.methodCommit, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotNull(beanParams, "locked", commandOptions.locked, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); interpretationUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1476,15 +1547,15 @@ private RestResponse updateReport() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), ClinicalReport.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "title",commandOptions.title, true); - putNestedIfNotEmpty(beanParams, "overview",commandOptions.overview, true); - putNestedIfNotEmpty(beanParams, "discussion.author",commandOptions.discussionAuthor, true); - putNestedIfNotEmpty(beanParams, "discussion.date",commandOptions.discussionDate, true); - putNestedIfNotEmpty(beanParams, "discussion.text",commandOptions.discussionText, true); - putNestedIfNotEmpty(beanParams, "logo",commandOptions.logo, true); - putNestedIfNotEmpty(beanParams, "signedBy",commandOptions.signedBy, true); - putNestedIfNotEmpty(beanParams, "signature",commandOptions.signature, true); - putNestedIfNotEmpty(beanParams, "date",commandOptions.date, true); + putNestedIfNotEmpty(beanParams, "title", commandOptions.title, true); + putNestedIfNotEmpty(beanParams, "overview", commandOptions.overview, true); + putNestedIfNotEmpty(beanParams, "discussion.author", commandOptions.discussionAuthor, true); + putNestedIfNotEmpty(beanParams, "discussion.date", commandOptions.discussionDate, true); + putNestedIfNotEmpty(beanParams, "discussion.text", commandOptions.discussionText, true); + putNestedIfNotEmpty(beanParams, "logo", commandOptions.logo, true); + putNestedIfNotEmpty(beanParams, "signedBy", commandOptions.signedBy, true); + putNestedIfNotEmpty(beanParams, "signature", commandOptions.signature, true); + putNestedIfNotEmpty(beanParams, "date", commandOptions.date, true); clinicalReport = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java index de60333d151..57edea737fa 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java @@ -318,10 +318,10 @@ private RestResponse runCircos() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), CircosAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "title",commandOptions.title, true); - putNestedIfNotEmpty(beanParams, "density",commandOptions.density, true); - putNestedIfNotNull(beanParams, "query",commandOptions.query, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "title", commandOptions.title, true); + putNestedIfNotEmpty(beanParams, "density", commandOptions.density, true); + putNestedMapIfNotEmpty(beanParams, "query", commandOptions.query, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); circosAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -370,6 +370,9 @@ private RestResponse runCohortStats() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -386,11 +389,11 @@ private RestResponse runCohortStats() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), CohortVariantStatsAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "cohort",commandOptions.cohort, true); - putNestedIfNotNull(beanParams, "samples",commandOptions.samples, true); - putNestedIfNotNull(beanParams, "index",commandOptions.index, true); - putNestedIfNotEmpty(beanParams, "sampleAnnotation",commandOptions.sampleAnnotation, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "cohort", commandOptions.cohort, true); + putNestedIfNotNull(beanParams, "samples", commandOptions.samples, true); + putNestedIfNotNull(beanParams, "index", commandOptions.index, true); + putNestedIfNotEmpty(beanParams, "sampleAnnotation", commandOptions.sampleAnnotation, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); cohortVariantStatsAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -410,6 +413,9 @@ private RestResponse runExomiser() throws Exception { queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -426,9 +432,10 @@ private RestResponse runExomiser() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), ExomiserWrapperParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotEmpty(beanParams, "clinicalAnalysisType",commandOptions.clinicalAnalysisType, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotEmpty(beanParams, "exomiserVersion", commandOptions.exomiserVersion, true); + putNestedIfNotEmpty(beanParams, "clinicalAnalysisType", commandOptions.clinicalAnalysisType, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); exomiserWrapperParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -451,6 +458,9 @@ private RestResponse runExport() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -467,93 +477,93 @@ private RestResponse runExport() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantExportParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "region",commandOptions.region, true); - putNestedIfNotEmpty(beanParams, "gene",commandOptions.gene, true); - putNestedIfNotEmpty(beanParams, "type",commandOptions.type, true); - putNestedIfNotEmpty(beanParams, "panel",commandOptions.panel, true); - putNestedIfNotEmpty(beanParams, "panelModeOfInheritance",commandOptions.panelModeOfInheritance, true); - putNestedIfNotEmpty(beanParams, "panelConfidence",commandOptions.panelConfidence, true); - putNestedIfNotEmpty(beanParams, "panelRoleInCancer",commandOptions.panelRoleInCancer, true); - putNestedIfNotNull(beanParams, "panelIntersection",commandOptions.panelIntersection, true); - putNestedIfNotEmpty(beanParams, "panelFeatureType",commandOptions.panelFeatureType, true); - putNestedIfNotEmpty(beanParams, "cohortStatsRef",commandOptions.cohortStatsRef, true); - putNestedIfNotEmpty(beanParams, "cohortStatsAlt",commandOptions.cohortStatsAlt, true); - putNestedIfNotEmpty(beanParams, "cohortStatsMaf",commandOptions.cohortStatsMaf, true); - putNestedIfNotEmpty(beanParams, "ct",commandOptions.ct, true); - putNestedIfNotEmpty(beanParams, "xref",commandOptions.xref, true); - putNestedIfNotEmpty(beanParams, "biotype",commandOptions.biotype, true); - putNestedIfNotEmpty(beanParams, "proteinSubstitution",commandOptions.proteinSubstitution, true); - putNestedIfNotEmpty(beanParams, "conservation",commandOptions.conservation, true); - putNestedIfNotEmpty(beanParams, "populationFrequencyMaf",commandOptions.populationFrequencyMaf, true); - putNestedIfNotEmpty(beanParams, "populationFrequencyAlt",commandOptions.populationFrequencyAlt, true); - putNestedIfNotEmpty(beanParams, "populationFrequencyRef",commandOptions.populationFrequencyRef, true); - putNestedIfNotEmpty(beanParams, "transcriptFlag",commandOptions.transcriptFlag, true); - putNestedIfNotEmpty(beanParams, "functionalScore",commandOptions.functionalScore, true); - putNestedIfNotEmpty(beanParams, "clinical",commandOptions.clinical, true); - putNestedIfNotEmpty(beanParams, "clinicalSignificance",commandOptions.clinicalSignificance, true); - putNestedIfNotNull(beanParams, "clinicalConfirmedStatus",commandOptions.clinicalConfirmedStatus, true); - putNestedIfNotEmpty(beanParams, "project",commandOptions.bodyProject, true); - putNestedIfNotEmpty(beanParams, "study",commandOptions.bodyStudy, true); - putNestedIfNotEmpty(beanParams, "savedFilter",commandOptions.savedFilter, true); - putNestedIfNotEmpty(beanParams, "chromosome",commandOptions.chromosome, true); - putNestedIfNotEmpty(beanParams, "reference",commandOptions.reference, true); - putNestedIfNotEmpty(beanParams, "alternate",commandOptions.alternate, true); - putNestedIfNotEmpty(beanParams, "release",commandOptions.release, true); - putNestedIfNotEmpty(beanParams, "includeStudy",commandOptions.includeStudy, true); - putNestedIfNotEmpty(beanParams, "includeSample",commandOptions.includeSample, true); - putNestedIfNotEmpty(beanParams, "includeFile",commandOptions.includeFile, true); - putNestedIfNotEmpty(beanParams, "includeSampleData",commandOptions.includeSampleData, true); - putNestedIfNotNull(beanParams, "includeSampleId",commandOptions.includeSampleId, true); - putNestedIfNotNull(beanParams, "includeGenotype",commandOptions.includeGenotype, true); - putNestedIfNotEmpty(beanParams, "file",commandOptions.file, true); - putNestedIfNotEmpty(beanParams, "qual",commandOptions.qual, true); - putNestedIfNotEmpty(beanParams, "filter",commandOptions.filter, true); - putNestedIfNotEmpty(beanParams, "fileData",commandOptions.fileData, true); - putNestedIfNotEmpty(beanParams, "genotype",commandOptions.genotype, true); - putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotNull(beanParams, "sampleLimit",commandOptions.sampleLimit, true); - putNestedIfNotNull(beanParams, "sampleSkip",commandOptions.sampleSkip, true); - putNestedIfNotEmpty(beanParams, "sampleData",commandOptions.sampleData, true); - putNestedIfNotEmpty(beanParams, "sampleAnnotation",commandOptions.sampleAnnotation, true); - putNestedIfNotEmpty(beanParams, "family",commandOptions.family, true); - putNestedIfNotEmpty(beanParams, "familyMembers",commandOptions.familyMembers, true); - putNestedIfNotEmpty(beanParams, "familyDisorder",commandOptions.familyDisorder, true); - putNestedIfNotEmpty(beanParams, "familyProband",commandOptions.familyProband, true); - putNestedIfNotEmpty(beanParams, "familySegregation",commandOptions.familySegregation, true); - putNestedIfNotEmpty(beanParams, "cohort",commandOptions.cohort, true); - putNestedIfNotEmpty(beanParams, "cohortStatsPass",commandOptions.cohortStatsPass, true); - putNestedIfNotEmpty(beanParams, "cohortStatsMgf",commandOptions.cohortStatsMgf, true); - putNestedIfNotEmpty(beanParams, "missingAlleles",commandOptions.missingAlleles, true); - putNestedIfNotEmpty(beanParams, "missingGenotypes",commandOptions.missingGenotypes, true); - putNestedIfNotNull(beanParams, "annotationExists",commandOptions.annotationExists, true); - putNestedIfNotEmpty(beanParams, "score",commandOptions.score, true); - putNestedIfNotEmpty(beanParams, "polyphen",commandOptions.polyphen, true); - putNestedIfNotEmpty(beanParams, "sift",commandOptions.sift, true); - putNestedIfNotEmpty(beanParams, "geneRoleInCancer",commandOptions.geneRoleInCancer, true); - putNestedIfNotEmpty(beanParams, "geneTraitId",commandOptions.geneTraitId, true); - putNestedIfNotEmpty(beanParams, "geneTraitName",commandOptions.geneTraitName, true); - putNestedIfNotEmpty(beanParams, "trait",commandOptions.trait, true); - putNestedIfNotEmpty(beanParams, "cosmic",commandOptions.cosmic, true); - putNestedIfNotEmpty(beanParams, "clinvar",commandOptions.clinvar, true); - putNestedIfNotEmpty(beanParams, "hpo",commandOptions.hpo, true); - putNestedIfNotEmpty(beanParams, "go",commandOptions.go, true); - putNestedIfNotEmpty(beanParams, "expression",commandOptions.expression, true); - putNestedIfNotEmpty(beanParams, "proteinKeyword",commandOptions.proteinKeyword, true); - putNestedIfNotEmpty(beanParams, "drug",commandOptions.drug, true); - putNestedIfNotEmpty(beanParams, "customAnnotation",commandOptions.customAnnotation, true); - putNestedIfNotEmpty(beanParams, "unknownGenotype",commandOptions.unknownGenotype, true); - putNestedIfNotNull(beanParams, "sampleMetadata",commandOptions.sampleMetadata, true); - putNestedIfNotNull(beanParams, "sort",commandOptions.sort, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotEmpty(beanParams, "outputFileName",commandOptions.outputFileName, true); - putNestedIfNotEmpty(beanParams, "outputFileFormat",commandOptions.outputFileFormat, true); - putNestedIfNotEmpty(beanParams, "variantsFile",commandOptions.variantsFile, true); - putNestedIfNotEmpty(beanParams, "include",commandOptions.bodyInclude, true); - putNestedIfNotEmpty(beanParams, "exclude",commandOptions.bodyExclude, true); - putNestedIfNotNull(beanParams, "limit",commandOptions.limit, true); - putNestedIfNotNull(beanParams, "skip",commandOptions.skip, true); - putNestedIfNotNull(beanParams, "summary",commandOptions.summary, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "region", commandOptions.region, true); + putNestedIfNotEmpty(beanParams, "gene", commandOptions.gene, true); + putNestedIfNotEmpty(beanParams, "type", commandOptions.type, true); + putNestedIfNotEmpty(beanParams, "panel", commandOptions.panel, true); + putNestedIfNotEmpty(beanParams, "panelModeOfInheritance", commandOptions.panelModeOfInheritance, true); + putNestedIfNotEmpty(beanParams, "panelConfidence", commandOptions.panelConfidence, true); + putNestedIfNotEmpty(beanParams, "panelRoleInCancer", commandOptions.panelRoleInCancer, true); + putNestedIfNotNull(beanParams, "panelIntersection", commandOptions.panelIntersection, true); + putNestedIfNotEmpty(beanParams, "panelFeatureType", commandOptions.panelFeatureType, true); + putNestedIfNotEmpty(beanParams, "cohortStatsRef", commandOptions.cohortStatsRef, true); + putNestedIfNotEmpty(beanParams, "cohortStatsAlt", commandOptions.cohortStatsAlt, true); + putNestedIfNotEmpty(beanParams, "cohortStatsMaf", commandOptions.cohortStatsMaf, true); + putNestedIfNotEmpty(beanParams, "ct", commandOptions.ct, true); + putNestedIfNotEmpty(beanParams, "xref", commandOptions.xref, true); + putNestedIfNotEmpty(beanParams, "biotype", commandOptions.biotype, true); + putNestedIfNotEmpty(beanParams, "proteinSubstitution", commandOptions.proteinSubstitution, true); + putNestedIfNotEmpty(beanParams, "conservation", commandOptions.conservation, true); + putNestedIfNotEmpty(beanParams, "populationFrequencyMaf", commandOptions.populationFrequencyMaf, true); + putNestedIfNotEmpty(beanParams, "populationFrequencyAlt", commandOptions.populationFrequencyAlt, true); + putNestedIfNotEmpty(beanParams, "populationFrequencyRef", commandOptions.populationFrequencyRef, true); + putNestedIfNotEmpty(beanParams, "transcriptFlag", commandOptions.transcriptFlag, true); + putNestedIfNotEmpty(beanParams, "functionalScore", commandOptions.functionalScore, true); + putNestedIfNotEmpty(beanParams, "clinical", commandOptions.clinical, true); + putNestedIfNotEmpty(beanParams, "clinicalSignificance", commandOptions.clinicalSignificance, true); + putNestedIfNotNull(beanParams, "clinicalConfirmedStatus", commandOptions.clinicalConfirmedStatus, true); + putNestedIfNotEmpty(beanParams, "project", commandOptions.bodyProject, true); + putNestedIfNotEmpty(beanParams, "study", commandOptions.bodyStudy, true); + putNestedIfNotEmpty(beanParams, "savedFilter", commandOptions.savedFilter, true); + putNestedIfNotEmpty(beanParams, "chromosome", commandOptions.chromosome, true); + putNestedIfNotEmpty(beanParams, "reference", commandOptions.reference, true); + putNestedIfNotEmpty(beanParams, "alternate", commandOptions.alternate, true); + putNestedIfNotEmpty(beanParams, "release", commandOptions.release, true); + putNestedIfNotEmpty(beanParams, "includeStudy", commandOptions.includeStudy, true); + putNestedIfNotEmpty(beanParams, "includeSample", commandOptions.includeSample, true); + putNestedIfNotEmpty(beanParams, "includeFile", commandOptions.includeFile, true); + putNestedIfNotEmpty(beanParams, "includeSampleData", commandOptions.includeSampleData, true); + putNestedIfNotNull(beanParams, "includeSampleId", commandOptions.includeSampleId, true); + putNestedIfNotNull(beanParams, "includeGenotype", commandOptions.includeGenotype, true); + putNestedIfNotEmpty(beanParams, "file", commandOptions.file, true); + putNestedIfNotEmpty(beanParams, "qual", commandOptions.qual, true); + putNestedIfNotEmpty(beanParams, "filter", commandOptions.filter, true); + putNestedIfNotEmpty(beanParams, "fileData", commandOptions.fileData, true); + putNestedIfNotEmpty(beanParams, "genotype", commandOptions.genotype, true); + putNestedIfNotEmpty(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotNull(beanParams, "sampleLimit", commandOptions.sampleLimit, true); + putNestedIfNotNull(beanParams, "sampleSkip", commandOptions.sampleSkip, true); + putNestedIfNotEmpty(beanParams, "sampleData", commandOptions.sampleData, true); + putNestedIfNotEmpty(beanParams, "sampleAnnotation", commandOptions.sampleAnnotation, true); + putNestedIfNotEmpty(beanParams, "family", commandOptions.family, true); + putNestedIfNotEmpty(beanParams, "familyMembers", commandOptions.familyMembers, true); + putNestedIfNotEmpty(beanParams, "familyDisorder", commandOptions.familyDisorder, true); + putNestedIfNotEmpty(beanParams, "familyProband", commandOptions.familyProband, true); + putNestedIfNotEmpty(beanParams, "familySegregation", commandOptions.familySegregation, true); + putNestedIfNotEmpty(beanParams, "cohort", commandOptions.cohort, true); + putNestedIfNotEmpty(beanParams, "cohortStatsPass", commandOptions.cohortStatsPass, true); + putNestedIfNotEmpty(beanParams, "cohortStatsMgf", commandOptions.cohortStatsMgf, true); + putNestedIfNotEmpty(beanParams, "missingAlleles", commandOptions.missingAlleles, true); + putNestedIfNotEmpty(beanParams, "missingGenotypes", commandOptions.missingGenotypes, true); + putNestedIfNotNull(beanParams, "annotationExists", commandOptions.annotationExists, true); + putNestedIfNotEmpty(beanParams, "score", commandOptions.score, true); + putNestedIfNotEmpty(beanParams, "polyphen", commandOptions.polyphen, true); + putNestedIfNotEmpty(beanParams, "sift", commandOptions.sift, true); + putNestedIfNotEmpty(beanParams, "geneRoleInCancer", commandOptions.geneRoleInCancer, true); + putNestedIfNotEmpty(beanParams, "geneTraitId", commandOptions.geneTraitId, true); + putNestedIfNotEmpty(beanParams, "geneTraitName", commandOptions.geneTraitName, true); + putNestedIfNotEmpty(beanParams, "trait", commandOptions.trait, true); + putNestedIfNotEmpty(beanParams, "cosmic", commandOptions.cosmic, true); + putNestedIfNotEmpty(beanParams, "clinvar", commandOptions.clinvar, true); + putNestedIfNotEmpty(beanParams, "hpo", commandOptions.hpo, true); + putNestedIfNotEmpty(beanParams, "go", commandOptions.go, true); + putNestedIfNotEmpty(beanParams, "expression", commandOptions.expression, true); + putNestedIfNotEmpty(beanParams, "proteinKeyword", commandOptions.proteinKeyword, true); + putNestedIfNotEmpty(beanParams, "drug", commandOptions.drug, true); + putNestedIfNotEmpty(beanParams, "customAnnotation", commandOptions.customAnnotation, true); + putNestedIfNotEmpty(beanParams, "unknownGenotype", commandOptions.unknownGenotype, true); + putNestedIfNotNull(beanParams, "sampleMetadata", commandOptions.sampleMetadata, true); + putNestedIfNotNull(beanParams, "sort", commandOptions.sort, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "outputFileName", commandOptions.outputFileName, true); + putNestedIfNotEmpty(beanParams, "outputFileFormat", commandOptions.outputFileFormat, true); + putNestedIfNotEmpty(beanParams, "variantsFile", commandOptions.variantsFile, true); + putNestedIfNotEmpty(beanParams, "include", commandOptions.bodyInclude, true); + putNestedIfNotEmpty(beanParams, "exclude", commandOptions.bodyExclude, true); + putNestedIfNotNull(beanParams, "limit", commandOptions.limit, true); + putNestedIfNotNull(beanParams, "skip", commandOptions.skip, true); + putNestedIfNotNull(beanParams, "summary", commandOptions.summary, true); variantExportParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -591,6 +601,9 @@ private RestResponse runFamilyQc() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -607,10 +620,10 @@ private RestResponse runFamilyQc() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), FamilyQcAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "family",commandOptions.family, true); - putNestedIfNotEmpty(beanParams, "relatednessMethod",commandOptions.relatednessMethod, true); - putNestedIfNotEmpty(beanParams, "relatednessMaf",commandOptions.relatednessMaf, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "family", commandOptions.family, true); + putNestedIfNotEmpty(beanParams, "relatednessMethod", commandOptions.relatednessMethod, true); + putNestedIfNotEmpty(beanParams, "relatednessMaf", commandOptions.relatednessMaf, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); familyQcAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -629,6 +642,9 @@ private RestResponse deleteFile() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("study", commandOptions.study); queryParams.putIfNotEmpty("file", commandOptions.file); queryParams.putIfNotNull("resume", commandOptions.resume); @@ -650,6 +666,9 @@ private RestResponse runGatk() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -666,9 +685,9 @@ private RestResponse runGatk() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), GatkWrapperParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "command",commandOptions.command, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotNull(beanParams, "gatkParams",commandOptions.gatkParams, true); + putNestedIfNotEmpty(beanParams, "command", commandOptions.command, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedMapIfNotEmpty(beanParams, "gatkParams", commandOptions.gatkParams, true); gatkWrapperParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -688,6 +707,9 @@ private RestResponse runGenomePlot() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -704,11 +726,11 @@ private RestResponse runGenomePlot() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), GenomePlotAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "configFile",commandOptions.configFile, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "configFile", commandOptions.configFile, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); genomePlotAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -728,6 +750,9 @@ private RestResponse runGwas() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -744,18 +769,18 @@ private RestResponse runGwas() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), GwasAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "phenotype",commandOptions.phenotype, true); - putNestedIfNotNull(beanParams, "index",commandOptions.index, true); - putNestedIfNotEmpty(beanParams, "indexScoreId",commandOptions.indexScoreId, true); - putNestedIfNotNull(beanParams, "method",commandOptions.method, true); - putNestedIfNotNull(beanParams, "fisherMode",commandOptions.fisherMode, true); - putNestedIfNotEmpty(beanParams, "caseCohort",commandOptions.caseCohort, true); - putNestedIfNotEmpty(beanParams, "caseCohortSamplesAnnotation",commandOptions.caseCohortSamplesAnnotation, true); - putNestedIfNotNull(beanParams, "caseCohortSamples",commandOptions.caseCohortSamples, true); - putNestedIfNotEmpty(beanParams, "controlCohort",commandOptions.controlCohort, true); - putNestedIfNotEmpty(beanParams, "controlCohortSamplesAnnotation",commandOptions.controlCohortSamplesAnnotation, true); - putNestedIfNotNull(beanParams, "controlCohortSamples",commandOptions.controlCohortSamples, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "phenotype", commandOptions.phenotype, true); + putNestedIfNotNull(beanParams, "index", commandOptions.index, true); + putNestedIfNotEmpty(beanParams, "indexScoreId", commandOptions.indexScoreId, true); + putNestedIfNotNull(beanParams, "method", commandOptions.method, true); + putNestedIfNotNull(beanParams, "fisherMode", commandOptions.fisherMode, true); + putNestedIfNotEmpty(beanParams, "caseCohort", commandOptions.caseCohort, true); + putNestedIfNotEmpty(beanParams, "caseCohortSamplesAnnotation", commandOptions.caseCohortSamplesAnnotation, true); + putNestedIfNotNull(beanParams, "caseCohortSamples", commandOptions.caseCohortSamples, true); + putNestedIfNotEmpty(beanParams, "controlCohort", commandOptions.controlCohort, true); + putNestedIfNotEmpty(beanParams, "controlCohortSamplesAnnotation", commandOptions.controlCohortSamplesAnnotation, true); + putNestedIfNotNull(beanParams, "controlCohortSamples", commandOptions.controlCohortSamples, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); gwasAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -775,6 +800,9 @@ private RestResponse runHrDetect() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -791,19 +819,19 @@ private RestResponse runHrDetect() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), HRDetectAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "sampleId",commandOptions.sampleId, true); - putNestedIfNotEmpty(beanParams, "snvFittingId",commandOptions.snvFittingId, true); - putNestedIfNotEmpty(beanParams, "svFittingId",commandOptions.svFittingId, true); - putNestedIfNotEmpty(beanParams, "cnvQuery",commandOptions.cnvQuery, true); - putNestedIfNotEmpty(beanParams, "indelQuery",commandOptions.indelQuery, true); - putNestedIfNotEmpty(beanParams, "snv3CustomName",commandOptions.snv3CustomName, true); - putNestedIfNotEmpty(beanParams, "snv8CustomName",commandOptions.snv8CustomName, true); - putNestedIfNotEmpty(beanParams, "sv3CustomName",commandOptions.sv3CustomName, true); - putNestedIfNotEmpty(beanParams, "sv8CustomName",commandOptions.sv8CustomName, true); - putNestedIfNotNull(beanParams, "bootstrap",commandOptions.bootstrap, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "sampleId", commandOptions.sampleId, true); + putNestedIfNotEmpty(beanParams, "snvFittingId", commandOptions.snvFittingId, true); + putNestedIfNotEmpty(beanParams, "svFittingId", commandOptions.svFittingId, true); + putNestedIfNotEmpty(beanParams, "cnvQuery", commandOptions.cnvQuery, true); + putNestedIfNotEmpty(beanParams, "indelQuery", commandOptions.indelQuery, true); + putNestedIfNotEmpty(beanParams, "snv3CustomName", commandOptions.snv3CustomName, true); + putNestedIfNotEmpty(beanParams, "snv8CustomName", commandOptions.snv8CustomName, true); + putNestedIfNotEmpty(beanParams, "sv3CustomName", commandOptions.sv3CustomName, true); + putNestedIfNotEmpty(beanParams, "sv8CustomName", commandOptions.sv8CustomName, true); + putNestedIfNotNull(beanParams, "bootstrap", commandOptions.bootstrap, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); hRDetectAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -823,6 +851,9 @@ private RestResponse runIndex() throws Exception { queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -839,36 +870,36 @@ private RestResponse runIndex() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantIndexParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "file",commandOptions.file, true); - putNestedIfNotNull(beanParams, "resume",commandOptions.resume, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotNull(beanParams, "transform",commandOptions.transform, true); - putNestedIfNotNull(beanParams, "gvcf",commandOptions.gvcf, true); - putNestedIfNotNull(beanParams, "normalizationSkip",commandOptions.normalizationSkip, true); - putNestedIfNotEmpty(beanParams, "referenceGenome",commandOptions.referenceGenome, true); - putNestedIfNotEmpty(beanParams, "failOnMalformedLines",commandOptions.failOnMalformedLines, true); - putNestedIfNotNull(beanParams, "family",commandOptions.family, true); - putNestedIfNotNull(beanParams, "somatic",commandOptions.somatic, true); - putNestedIfNotNull(beanParams, "load",commandOptions.load, true); - putNestedIfNotNull(beanParams, "forceReload",commandOptions.forceReload, true); - putNestedIfNotEmpty(beanParams, "loadSplitData",commandOptions.loadSplitData, true); - putNestedIfNotNull(beanParams, "loadMultiFileData",commandOptions.loadMultiFileData, true); - putNestedIfNotEmpty(beanParams, "loadSampleIndex",commandOptions.loadSampleIndex, true); - putNestedIfNotEmpty(beanParams, "loadArchive",commandOptions.loadArchive, true); - putNestedIfNotEmpty(beanParams, "loadHomRef",commandOptions.loadHomRef, true); - putNestedIfNotEmpty(beanParams, "postLoadCheck",commandOptions.postLoadCheck, true); - putNestedIfNotEmpty(beanParams, "includeGenotypes",commandOptions.includeGenotypes, true); - putNestedIfNotEmpty(beanParams, "includeSampleData",commandOptions.includeSampleData, true); - putNestedIfNotEmpty(beanParams, "merge",commandOptions.merge, true); - putNestedIfNotEmpty(beanParams, "deduplicationPolicy",commandOptions.deduplicationPolicy, true); - putNestedIfNotNull(beanParams, "calculateStats",commandOptions.calculateStats, true); - putNestedIfNotNull(beanParams, "aggregated",commandOptions.aggregated, true); - putNestedIfNotEmpty(beanParams, "aggregationMappingFile",commandOptions.aggregationMappingFile, true); - putNestedIfNotNull(beanParams, "annotate",commandOptions.annotate, true); - putNestedIfNotEmpty(beanParams, "annotator",commandOptions.annotator, true); - putNestedIfNotNull(beanParams, "overwriteAnnotations",commandOptions.overwriteAnnotations, true); - putNestedIfNotNull(beanParams, "indexSearch",commandOptions.indexSearch, true); - putNestedIfNotNull(beanParams, "skipIndexedFiles",commandOptions.skipIndexedFiles, true); + putNestedIfNotEmpty(beanParams, "file", commandOptions.file, true); + putNestedIfNotNull(beanParams, "resume", commandOptions.resume, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedIfNotNull(beanParams, "transform", commandOptions.transform, true); + putNestedIfNotNull(beanParams, "gvcf", commandOptions.gvcf, true); + putNestedIfNotNull(beanParams, "normalizationSkip", commandOptions.normalizationSkip, true); + putNestedIfNotEmpty(beanParams, "referenceGenome", commandOptions.referenceGenome, true); + putNestedIfNotEmpty(beanParams, "failOnMalformedLines", commandOptions.failOnMalformedLines, true); + putNestedIfNotNull(beanParams, "family", commandOptions.family, true); + putNestedIfNotNull(beanParams, "somatic", commandOptions.somatic, true); + putNestedIfNotNull(beanParams, "load", commandOptions.load, true); + putNestedIfNotNull(beanParams, "forceReload", commandOptions.forceReload, true); + putNestedIfNotEmpty(beanParams, "loadSplitData", commandOptions.loadSplitData, true); + putNestedIfNotNull(beanParams, "loadMultiFileData", commandOptions.loadMultiFileData, true); + putNestedIfNotEmpty(beanParams, "loadSampleIndex", commandOptions.loadSampleIndex, true); + putNestedIfNotEmpty(beanParams, "loadArchive", commandOptions.loadArchive, true); + putNestedIfNotEmpty(beanParams, "loadHomRef", commandOptions.loadHomRef, true); + putNestedIfNotEmpty(beanParams, "postLoadCheck", commandOptions.postLoadCheck, true); + putNestedIfNotEmpty(beanParams, "includeGenotypes", commandOptions.includeGenotypes, true); + putNestedIfNotEmpty(beanParams, "includeSampleData", commandOptions.includeSampleData, true); + putNestedIfNotEmpty(beanParams, "merge", commandOptions.merge, true); + putNestedIfNotEmpty(beanParams, "deduplicationPolicy", commandOptions.deduplicationPolicy, true); + putNestedIfNotNull(beanParams, "calculateStats", commandOptions.calculateStats, true); + putNestedIfNotNull(beanParams, "aggregated", commandOptions.aggregated, true); + putNestedIfNotEmpty(beanParams, "aggregationMappingFile", commandOptions.aggregationMappingFile, true); + putNestedIfNotNull(beanParams, "annotate", commandOptions.annotate, true); + putNestedIfNotEmpty(beanParams, "annotator", commandOptions.annotator, true); + putNestedIfNotNull(beanParams, "overwriteAnnotations", commandOptions.overwriteAnnotations, true); + putNestedIfNotNull(beanParams, "indexSearch", commandOptions.indexSearch, true); + putNestedIfNotNull(beanParams, "skipIndexedFiles", commandOptions.skipIndexedFiles, true); variantIndexParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -888,6 +919,9 @@ private RestResponse runIndividualQc() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -904,10 +938,10 @@ private RestResponse runIndividualQc() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), IndividualQcAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "individual",commandOptions.individual, true); - putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotEmpty(beanParams, "inferredSexMethod",commandOptions.inferredSexMethod, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "individual", commandOptions.individual, true); + putNestedIfNotEmpty(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotEmpty(beanParams, "inferredSexMethod", commandOptions.inferredSexMethod, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); individualQcAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -927,6 +961,9 @@ private RestResponse runInferredSex() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -943,9 +980,9 @@ private RestResponse runInferredSex() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), InferredSexAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "individual",commandOptions.individual, true); - putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "individual", commandOptions.individual, true); + putNestedIfNotEmpty(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); inferredSexAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -999,6 +1036,9 @@ private RestResponse runKnockout() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -1015,16 +1055,16 @@ private RestResponse runKnockout() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), KnockoutAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotNull(beanParams, "gene",commandOptions.gene, true); - putNestedIfNotNull(beanParams, "panel",commandOptions.panel, true); - putNestedIfNotEmpty(beanParams, "biotype",commandOptions.biotype, true); - putNestedIfNotEmpty(beanParams, "consequenceType",commandOptions.consequenceType, true); - putNestedIfNotEmpty(beanParams, "filter",commandOptions.filter, true); - putNestedIfNotEmpty(beanParams, "qual",commandOptions.qual, true); - putNestedIfNotNull(beanParams, "skipGenesFile",commandOptions.skipGenesFile, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotNull(beanParams, "index",commandOptions.index, true); + putNestedIfNotNull(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotNull(beanParams, "gene", commandOptions.gene, true); + putNestedIfNotNull(beanParams, "panel", commandOptions.panel, true); + putNestedIfNotEmpty(beanParams, "biotype", commandOptions.biotype, true); + putNestedIfNotEmpty(beanParams, "consequenceType", commandOptions.consequenceType, true); + putNestedIfNotEmpty(beanParams, "filter", commandOptions.filter, true); + putNestedIfNotEmpty(beanParams, "qual", commandOptions.qual, true); + putNestedIfNotNull(beanParams, "skipGenesFile", commandOptions.skipGenesFile, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedIfNotNull(beanParams, "index", commandOptions.index, true); knockoutAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1044,6 +1084,9 @@ private RestResponse runMendelianError() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -1060,10 +1103,10 @@ private RestResponse runMendelianError() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), MendelianErrorAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "family",commandOptions.family, true); - putNestedIfNotEmpty(beanParams, "individual",commandOptions.individual, true); - putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "family", commandOptions.family, true); + putNestedIfNotEmpty(beanParams, "individual", commandOptions.individual, true); + putNestedIfNotEmpty(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); mendelianErrorAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1136,6 +1179,9 @@ private RestResponse runMutationalSignature() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -1152,22 +1198,22 @@ private RestResponse runMutationalSignature() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), MutationalSignatureAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotEmpty(beanParams, "query",commandOptions.query, true); - putNestedIfNotEmpty(beanParams, "fitId",commandOptions.fitId, true); - putNestedIfNotEmpty(beanParams, "fitMethod",commandOptions.fitMethod, true); - putNestedIfNotNull(beanParams, "fitNBoot",commandOptions.fitNBoot, true); - putNestedIfNotEmpty(beanParams, "fitSigVersion",commandOptions.fitSigVersion, true); - putNestedIfNotEmpty(beanParams, "fitOrgan",commandOptions.fitOrgan, true); - putNestedIfNotNull(beanParams, "fitThresholdPerc",commandOptions.fitThresholdPerc, true); - putNestedIfNotNull(beanParams, "fitThresholdPval",commandOptions.fitThresholdPval, true); - putNestedIfNotNull(beanParams, "fitMaxRareSigs",commandOptions.fitMaxRareSigs, true); - putNestedIfNotEmpty(beanParams, "fitSignaturesFile",commandOptions.fitSignaturesFile, true); - putNestedIfNotEmpty(beanParams, "fitRareSignaturesFile",commandOptions.fitRareSignaturesFile, true); - putNestedIfNotEmpty(beanParams, "skip",commandOptions.skip, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotEmpty(beanParams, "query", commandOptions.query, true); + putNestedIfNotEmpty(beanParams, "fitId", commandOptions.fitId, true); + putNestedIfNotEmpty(beanParams, "fitMethod", commandOptions.fitMethod, true); + putNestedIfNotNull(beanParams, "fitNBoot", commandOptions.fitNBoot, true); + putNestedIfNotEmpty(beanParams, "fitSigVersion", commandOptions.fitSigVersion, true); + putNestedIfNotEmpty(beanParams, "fitOrgan", commandOptions.fitOrgan, true); + putNestedIfNotNull(beanParams, "fitThresholdPerc", commandOptions.fitThresholdPerc, true); + putNestedIfNotNull(beanParams, "fitThresholdPval", commandOptions.fitThresholdPval, true); + putNestedIfNotNull(beanParams, "fitMaxRareSigs", commandOptions.fitMaxRareSigs, true); + putNestedIfNotEmpty(beanParams, "fitSignaturesFile", commandOptions.fitSignaturesFile, true); + putNestedIfNotEmpty(beanParams, "fitRareSignaturesFile", commandOptions.fitRareSignaturesFile, true); + putNestedIfNotEmpty(beanParams, "skip", commandOptions.skip, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); mutationalSignatureAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1187,6 +1233,9 @@ private RestResponse runPlink() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -1203,8 +1252,8 @@ private RestResponse runPlink() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), PlinkWrapperParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotNull(beanParams, "plinkParams",commandOptions.plinkParams, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedMapIfNotEmpty(beanParams, "plinkParams", commandOptions.plinkParams, true); plinkWrapperParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1314,6 +1363,9 @@ private RestResponse runRelatedness() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -1330,11 +1382,11 @@ private RestResponse runRelatedness() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), RelatednessAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "individuals",commandOptions.individuals, true); - putNestedIfNotNull(beanParams, "samples",commandOptions.samples, true); - putNestedIfNotEmpty(beanParams, "minorAlleleFreq",commandOptions.minorAlleleFreq, true); - putNestedIfNotEmpty(beanParams, "method",commandOptions.method, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotNull(beanParams, "individuals", commandOptions.individuals, true); + putNestedIfNotNull(beanParams, "samples", commandOptions.samples, true); + putNestedIfNotEmpty(beanParams, "minorAlleleFreq", commandOptions.minorAlleleFreq, true); + putNestedIfNotEmpty(beanParams, "method", commandOptions.method, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); relatednessAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1354,6 +1406,9 @@ private RestResponse runRvtests() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -1370,9 +1425,9 @@ private RestResponse runRvtests() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), RvtestsWrapperParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "command",commandOptions.command, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotNull(beanParams, "rvtestsParams",commandOptions.rvtestsParams, true); + putNestedIfNotEmpty(beanParams, "command", commandOptions.command, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedMapIfNotEmpty(beanParams, "rvtestsParams", commandOptions.rvtestsParams, true); rvtestsWrapperParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1427,6 +1482,9 @@ private RestResponse runSampleEligibility() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -1443,9 +1501,9 @@ private RestResponse runSampleEligibility() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), SampleEligibilityAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "query",commandOptions.query, true); - putNestedIfNotNull(beanParams, "index",commandOptions.index, true); - putNestedIfNotEmpty(beanParams, "cohortId",commandOptions.cohortId, true); + putNestedIfNotEmpty(beanParams, "query", commandOptions.query, true); + putNestedIfNotNull(beanParams, "index", commandOptions.index, true); + putNestedIfNotEmpty(beanParams, "cohortId", commandOptions.cohortId, true); sampleEligibilityAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1465,6 +1523,9 @@ private RestResponse runSampleQc() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -1481,53 +1542,53 @@ private RestResponse runSampleQc() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), SampleQcAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotEmpty(beanParams, "vsId",commandOptions.vsId, true); - putNestedIfNotEmpty(beanParams, "vsDescription",commandOptions.vsDescription, true); - putNestedIfNotEmpty(beanParams, "vsQuery.id",commandOptions.vsQueryId, true); - putNestedIfNotEmpty(beanParams, "vsQuery.region",commandOptions.vsQueryRegion, true); - putNestedIfNotEmpty(beanParams, "vsQuery.gene",commandOptions.vsQueryGene, true); - putNestedIfNotEmpty(beanParams, "vsQuery.type",commandOptions.vsQueryType, true); - putNestedIfNotEmpty(beanParams, "vsQuery.panel",commandOptions.vsQueryPanel, true); - putNestedIfNotEmpty(beanParams, "vsQuery.panelModeOfInheritance",commandOptions.vsQueryPanelModeOfInheritance, true); - putNestedIfNotEmpty(beanParams, "vsQuery.panelConfidence",commandOptions.vsQueryPanelConfidence, true); - putNestedIfNotEmpty(beanParams, "vsQuery.panelRoleInCancer",commandOptions.vsQueryPanelRoleInCancer, true); - putNestedIfNotNull(beanParams, "vsQuery.panelIntersection",commandOptions.vsQueryPanelIntersection, true); - putNestedIfNotEmpty(beanParams, "vsQuery.panelFeatureType",commandOptions.vsQueryPanelFeatureType, true); - putNestedIfNotEmpty(beanParams, "vsQuery.cohortStatsRef",commandOptions.vsQueryCohortStatsRef, true); - putNestedIfNotEmpty(beanParams, "vsQuery.cohortStatsAlt",commandOptions.vsQueryCohortStatsAlt, true); - putNestedIfNotEmpty(beanParams, "vsQuery.cohortStatsMaf",commandOptions.vsQueryCohortStatsMaf, true); - putNestedIfNotEmpty(beanParams, "vsQuery.ct",commandOptions.vsQueryCt, true); - putNestedIfNotEmpty(beanParams, "vsQuery.xref",commandOptions.vsQueryXref, true); - putNestedIfNotEmpty(beanParams, "vsQuery.biotype",commandOptions.vsQueryBiotype, true); - putNestedIfNotEmpty(beanParams, "vsQuery.proteinSubstitution",commandOptions.vsQueryProteinSubstitution, true); - putNestedIfNotEmpty(beanParams, "vsQuery.conservation",commandOptions.vsQueryConservation, true); - putNestedIfNotEmpty(beanParams, "vsQuery.populationFrequencyMaf",commandOptions.vsQueryPopulationFrequencyMaf, true); - putNestedIfNotEmpty(beanParams, "vsQuery.populationFrequencyAlt",commandOptions.vsQueryPopulationFrequencyAlt, true); - putNestedIfNotEmpty(beanParams, "vsQuery.populationFrequencyRef",commandOptions.vsQueryPopulationFrequencyRef, true); - putNestedIfNotEmpty(beanParams, "vsQuery.transcriptFlag",commandOptions.vsQueryTranscriptFlag, true); - putNestedIfNotEmpty(beanParams, "vsQuery.functionalScore",commandOptions.vsQueryFunctionalScore, true); - putNestedIfNotEmpty(beanParams, "vsQuery.clinical",commandOptions.vsQueryClinical, true); - putNestedIfNotEmpty(beanParams, "vsQuery.clinicalSignificance",commandOptions.vsQueryClinicalSignificance, true); - putNestedIfNotNull(beanParams, "vsQuery.clinicalConfirmedStatus",commandOptions.vsQueryClinicalConfirmedStatus, true); - putNestedIfNotEmpty(beanParams, "msId",commandOptions.msId, true); - putNestedIfNotEmpty(beanParams, "msDescription",commandOptions.msDescription, true); - putNestedIfNotEmpty(beanParams, "msQuery",commandOptions.msQuery, true); - putNestedIfNotEmpty(beanParams, "msFitId",commandOptions.msFitId, true); - putNestedIfNotEmpty(beanParams, "msFitMethod",commandOptions.msFitMethod, true); - putNestedIfNotNull(beanParams, "msFitNBoot",commandOptions.msFitNBoot, true); - putNestedIfNotEmpty(beanParams, "msFitSigVersion",commandOptions.msFitSigVersion, true); - putNestedIfNotEmpty(beanParams, "msFitOrgan",commandOptions.msFitOrgan, true); - putNestedIfNotNull(beanParams, "msFitThresholdPerc",commandOptions.msFitThresholdPerc, true); - putNestedIfNotNull(beanParams, "msFitThresholdPval",commandOptions.msFitThresholdPval, true); - putNestedIfNotNull(beanParams, "msFitMaxRareSigs",commandOptions.msFitMaxRareSigs, true); - putNestedIfNotEmpty(beanParams, "msFitSignaturesFile",commandOptions.msFitSignaturesFile, true); - putNestedIfNotEmpty(beanParams, "msFitRareSignaturesFile",commandOptions.msFitRareSignaturesFile, true); - putNestedIfNotEmpty(beanParams, "gpId",commandOptions.gpId, true); - putNestedIfNotEmpty(beanParams, "gpDescription",commandOptions.gpDescription, true); - putNestedIfNotEmpty(beanParams, "gpConfigFile",commandOptions.gpConfigFile, true); - putNestedIfNotEmpty(beanParams, "skip",commandOptions.skip, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotEmpty(beanParams, "vsId", commandOptions.vsId, true); + putNestedIfNotEmpty(beanParams, "vsDescription", commandOptions.vsDescription, true); + putNestedIfNotEmpty(beanParams, "vsQuery.id", commandOptions.vsQueryId, true); + putNestedIfNotEmpty(beanParams, "vsQuery.region", commandOptions.vsQueryRegion, true); + putNestedIfNotEmpty(beanParams, "vsQuery.gene", commandOptions.vsQueryGene, true); + putNestedIfNotEmpty(beanParams, "vsQuery.type", commandOptions.vsQueryType, true); + putNestedIfNotEmpty(beanParams, "vsQuery.panel", commandOptions.vsQueryPanel, true); + putNestedIfNotEmpty(beanParams, "vsQuery.panelModeOfInheritance", commandOptions.vsQueryPanelModeOfInheritance, true); + putNestedIfNotEmpty(beanParams, "vsQuery.panelConfidence", commandOptions.vsQueryPanelConfidence, true); + putNestedIfNotEmpty(beanParams, "vsQuery.panelRoleInCancer", commandOptions.vsQueryPanelRoleInCancer, true); + putNestedIfNotNull(beanParams, "vsQuery.panelIntersection", commandOptions.vsQueryPanelIntersection, true); + putNestedIfNotEmpty(beanParams, "vsQuery.panelFeatureType", commandOptions.vsQueryPanelFeatureType, true); + putNestedIfNotEmpty(beanParams, "vsQuery.cohortStatsRef", commandOptions.vsQueryCohortStatsRef, true); + putNestedIfNotEmpty(beanParams, "vsQuery.cohortStatsAlt", commandOptions.vsQueryCohortStatsAlt, true); + putNestedIfNotEmpty(beanParams, "vsQuery.cohortStatsMaf", commandOptions.vsQueryCohortStatsMaf, true); + putNestedIfNotEmpty(beanParams, "vsQuery.ct", commandOptions.vsQueryCt, true); + putNestedIfNotEmpty(beanParams, "vsQuery.xref", commandOptions.vsQueryXref, true); + putNestedIfNotEmpty(beanParams, "vsQuery.biotype", commandOptions.vsQueryBiotype, true); + putNestedIfNotEmpty(beanParams, "vsQuery.proteinSubstitution", commandOptions.vsQueryProteinSubstitution, true); + putNestedIfNotEmpty(beanParams, "vsQuery.conservation", commandOptions.vsQueryConservation, true); + putNestedIfNotEmpty(beanParams, "vsQuery.populationFrequencyMaf", commandOptions.vsQueryPopulationFrequencyMaf, true); + putNestedIfNotEmpty(beanParams, "vsQuery.populationFrequencyAlt", commandOptions.vsQueryPopulationFrequencyAlt, true); + putNestedIfNotEmpty(beanParams, "vsQuery.populationFrequencyRef", commandOptions.vsQueryPopulationFrequencyRef, true); + putNestedIfNotEmpty(beanParams, "vsQuery.transcriptFlag", commandOptions.vsQueryTranscriptFlag, true); + putNestedIfNotEmpty(beanParams, "vsQuery.functionalScore", commandOptions.vsQueryFunctionalScore, true); + putNestedIfNotEmpty(beanParams, "vsQuery.clinical", commandOptions.vsQueryClinical, true); + putNestedIfNotEmpty(beanParams, "vsQuery.clinicalSignificance", commandOptions.vsQueryClinicalSignificance, true); + putNestedIfNotNull(beanParams, "vsQuery.clinicalConfirmedStatus", commandOptions.vsQueryClinicalConfirmedStatus, true); + putNestedIfNotEmpty(beanParams, "msId", commandOptions.msId, true); + putNestedIfNotEmpty(beanParams, "msDescription", commandOptions.msDescription, true); + putNestedIfNotEmpty(beanParams, "msQuery", commandOptions.msQuery, true); + putNestedIfNotEmpty(beanParams, "msFitId", commandOptions.msFitId, true); + putNestedIfNotEmpty(beanParams, "msFitMethod", commandOptions.msFitMethod, true); + putNestedIfNotNull(beanParams, "msFitNBoot", commandOptions.msFitNBoot, true); + putNestedIfNotEmpty(beanParams, "msFitSigVersion", commandOptions.msFitSigVersion, true); + putNestedIfNotEmpty(beanParams, "msFitOrgan", commandOptions.msFitOrgan, true); + putNestedIfNotNull(beanParams, "msFitThresholdPerc", commandOptions.msFitThresholdPerc, true); + putNestedIfNotNull(beanParams, "msFitThresholdPval", commandOptions.msFitThresholdPval, true); + putNestedIfNotNull(beanParams, "msFitMaxRareSigs", commandOptions.msFitMaxRareSigs, true); + putNestedIfNotEmpty(beanParams, "msFitSignaturesFile", commandOptions.msFitSignaturesFile, true); + putNestedIfNotEmpty(beanParams, "msFitRareSignaturesFile", commandOptions.msFitRareSignaturesFile, true); + putNestedIfNotEmpty(beanParams, "gpId", commandOptions.gpId, true); + putNestedIfNotEmpty(beanParams, "gpDescription", commandOptions.gpDescription, true); + putNestedIfNotEmpty(beanParams, "gpConfigFile", commandOptions.gpConfigFile, true); + putNestedIfNotEmpty(beanParams, "skip", commandOptions.skip, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); sampleQcAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1565,6 +1626,9 @@ private RestResponse runSample() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -1581,36 +1645,36 @@ private RestResponse runSample() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), SampleVariantFilterParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "region",commandOptions.region, true); - putNestedIfNotEmpty(beanParams, "gene",commandOptions.gene, true); - putNestedIfNotEmpty(beanParams, "type",commandOptions.type, true); - putNestedIfNotEmpty(beanParams, "panel",commandOptions.panel, true); - putNestedIfNotEmpty(beanParams, "panelModeOfInheritance",commandOptions.panelModeOfInheritance, true); - putNestedIfNotEmpty(beanParams, "panelConfidence",commandOptions.panelConfidence, true); - putNestedIfNotEmpty(beanParams, "panelRoleInCancer",commandOptions.panelRoleInCancer, true); - putNestedIfNotNull(beanParams, "panelIntersection",commandOptions.panelIntersection, true); - putNestedIfNotEmpty(beanParams, "panelFeatureType",commandOptions.panelFeatureType, true); - putNestedIfNotEmpty(beanParams, "cohortStatsRef",commandOptions.cohortStatsRef, true); - putNestedIfNotEmpty(beanParams, "cohortStatsAlt",commandOptions.cohortStatsAlt, true); - putNestedIfNotEmpty(beanParams, "cohortStatsMaf",commandOptions.cohortStatsMaf, true); - putNestedIfNotEmpty(beanParams, "ct",commandOptions.ct, true); - putNestedIfNotEmpty(beanParams, "xref",commandOptions.xref, true); - putNestedIfNotEmpty(beanParams, "biotype",commandOptions.biotype, true); - putNestedIfNotEmpty(beanParams, "proteinSubstitution",commandOptions.proteinSubstitution, true); - putNestedIfNotEmpty(beanParams, "conservation",commandOptions.conservation, true); - putNestedIfNotEmpty(beanParams, "populationFrequencyMaf",commandOptions.populationFrequencyMaf, true); - putNestedIfNotEmpty(beanParams, "populationFrequencyAlt",commandOptions.populationFrequencyAlt, true); - putNestedIfNotEmpty(beanParams, "populationFrequencyRef",commandOptions.populationFrequencyRef, true); - putNestedIfNotEmpty(beanParams, "transcriptFlag",commandOptions.transcriptFlag, true); - putNestedIfNotEmpty(beanParams, "functionalScore",commandOptions.functionalScore, true); - putNestedIfNotEmpty(beanParams, "clinical",commandOptions.clinical, true); - putNestedIfNotEmpty(beanParams, "clinicalSignificance",commandOptions.clinicalSignificance, true); - putNestedIfNotNull(beanParams, "clinicalConfirmedStatus",commandOptions.clinicalConfirmedStatus, true); - putNestedIfNotNull(beanParams, "genotypes",commandOptions.genotypes, true); - putNestedIfNotNull(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotNull(beanParams, "samplesInAllVariants",commandOptions.samplesInAllVariants, true); - putNestedIfNotNull(beanParams, "maxVariants",commandOptions.maxVariants, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "region", commandOptions.region, true); + putNestedIfNotEmpty(beanParams, "gene", commandOptions.gene, true); + putNestedIfNotEmpty(beanParams, "type", commandOptions.type, true); + putNestedIfNotEmpty(beanParams, "panel", commandOptions.panel, true); + putNestedIfNotEmpty(beanParams, "panelModeOfInheritance", commandOptions.panelModeOfInheritance, true); + putNestedIfNotEmpty(beanParams, "panelConfidence", commandOptions.panelConfidence, true); + putNestedIfNotEmpty(beanParams, "panelRoleInCancer", commandOptions.panelRoleInCancer, true); + putNestedIfNotNull(beanParams, "panelIntersection", commandOptions.panelIntersection, true); + putNestedIfNotEmpty(beanParams, "panelFeatureType", commandOptions.panelFeatureType, true); + putNestedIfNotEmpty(beanParams, "cohortStatsRef", commandOptions.cohortStatsRef, true); + putNestedIfNotEmpty(beanParams, "cohortStatsAlt", commandOptions.cohortStatsAlt, true); + putNestedIfNotEmpty(beanParams, "cohortStatsMaf", commandOptions.cohortStatsMaf, true); + putNestedIfNotEmpty(beanParams, "ct", commandOptions.ct, true); + putNestedIfNotEmpty(beanParams, "xref", commandOptions.xref, true); + putNestedIfNotEmpty(beanParams, "biotype", commandOptions.biotype, true); + putNestedIfNotEmpty(beanParams, "proteinSubstitution", commandOptions.proteinSubstitution, true); + putNestedIfNotEmpty(beanParams, "conservation", commandOptions.conservation, true); + putNestedIfNotEmpty(beanParams, "populationFrequencyMaf", commandOptions.populationFrequencyMaf, true); + putNestedIfNotEmpty(beanParams, "populationFrequencyAlt", commandOptions.populationFrequencyAlt, true); + putNestedIfNotEmpty(beanParams, "populationFrequencyRef", commandOptions.populationFrequencyRef, true); + putNestedIfNotEmpty(beanParams, "transcriptFlag", commandOptions.transcriptFlag, true); + putNestedIfNotEmpty(beanParams, "functionalScore", commandOptions.functionalScore, true); + putNestedIfNotEmpty(beanParams, "clinical", commandOptions.clinical, true); + putNestedIfNotEmpty(beanParams, "clinicalSignificance", commandOptions.clinicalSignificance, true); + putNestedIfNotNull(beanParams, "clinicalConfirmedStatus", commandOptions.clinicalConfirmedStatus, true); + putNestedIfNotNull(beanParams, "genotypes", commandOptions.genotypes, true); + putNestedIfNotNull(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotNull(beanParams, "samplesInAllVariants", commandOptions.samplesInAllVariants, true); + putNestedIfNotNull(beanParams, "maxVariants", commandOptions.maxVariants, true); sampleVariantFilterParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1658,6 +1722,9 @@ private RestResponse runSampleStats() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -1674,42 +1741,42 @@ private RestResponse runSampleStats() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), SampleVariantStatsAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotNull(beanParams, "individual",commandOptions.individual, true); - putNestedIfNotEmpty(beanParams, "variantQuery.id",commandOptions.variantQueryId, true); - putNestedIfNotEmpty(beanParams, "variantQuery.region",commandOptions.variantQueryRegion, true); - putNestedIfNotEmpty(beanParams, "variantQuery.gene",commandOptions.variantQueryGene, true); - putNestedIfNotEmpty(beanParams, "variantQuery.type",commandOptions.variantQueryType, true); - putNestedIfNotEmpty(beanParams, "variantQuery.panel",commandOptions.variantQueryPanel, true); - putNestedIfNotEmpty(beanParams, "variantQuery.panelModeOfInheritance",commandOptions.variantQueryPanelModeOfInheritance, true); - putNestedIfNotEmpty(beanParams, "variantQuery.panelConfidence",commandOptions.variantQueryPanelConfidence, true); - putNestedIfNotEmpty(beanParams, "variantQuery.panelRoleInCancer",commandOptions.variantQueryPanelRoleInCancer, true); - putNestedIfNotNull(beanParams, "variantQuery.panelIntersection",commandOptions.variantQueryPanelIntersection, true); - putNestedIfNotEmpty(beanParams, "variantQuery.panelFeatureType",commandOptions.variantQueryPanelFeatureType, true); - putNestedIfNotEmpty(beanParams, "variantQuery.cohortStatsRef",commandOptions.variantQueryCohortStatsRef, true); - putNestedIfNotEmpty(beanParams, "variantQuery.cohortStatsAlt",commandOptions.variantQueryCohortStatsAlt, true); - putNestedIfNotEmpty(beanParams, "variantQuery.cohortStatsMaf",commandOptions.variantQueryCohortStatsMaf, true); - putNestedIfNotEmpty(beanParams, "variantQuery.ct",commandOptions.variantQueryCt, true); - putNestedIfNotEmpty(beanParams, "variantQuery.xref",commandOptions.variantQueryXref, true); - putNestedIfNotEmpty(beanParams, "variantQuery.biotype",commandOptions.variantQueryBiotype, true); - putNestedIfNotEmpty(beanParams, "variantQuery.proteinSubstitution",commandOptions.variantQueryProteinSubstitution, true); - putNestedIfNotEmpty(beanParams, "variantQuery.conservation",commandOptions.variantQueryConservation, true); - putNestedIfNotEmpty(beanParams, "variantQuery.populationFrequencyMaf",commandOptions.variantQueryPopulationFrequencyMaf, true); - putNestedIfNotEmpty(beanParams, "variantQuery.populationFrequencyAlt",commandOptions.variantQueryPopulationFrequencyAlt, true); - putNestedIfNotEmpty(beanParams, "variantQuery.populationFrequencyRef",commandOptions.variantQueryPopulationFrequencyRef, true); - putNestedIfNotEmpty(beanParams, "variantQuery.transcriptFlag",commandOptions.variantQueryTranscriptFlag, true); - putNestedIfNotEmpty(beanParams, "variantQuery.functionalScore",commandOptions.variantQueryFunctionalScore, true); - putNestedIfNotEmpty(beanParams, "variantQuery.clinical",commandOptions.variantQueryClinical, true); - putNestedIfNotEmpty(beanParams, "variantQuery.clinicalSignificance",commandOptions.variantQueryClinicalSignificance, true); - putNestedIfNotNull(beanParams, "variantQuery.clinicalConfirmedStatus",commandOptions.variantQueryClinicalConfirmedStatus, true); - putNestedIfNotEmpty(beanParams, "variantQuery.sampleData",commandOptions.variantQuerySampleData, true); - putNestedIfNotEmpty(beanParams, "variantQuery.fileData",commandOptions.variantQueryFileData, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotNull(beanParams, "index",commandOptions.index, true); - putNestedIfNotNull(beanParams, "indexOverwrite",commandOptions.indexOverwrite, true); - putNestedIfNotEmpty(beanParams, "indexId",commandOptions.indexId, true); - putNestedIfNotEmpty(beanParams, "indexDescription",commandOptions.indexDescription, true); - putNestedIfNotNull(beanParams, "batchSize",commandOptions.batchSize, true); + putNestedIfNotNull(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotNull(beanParams, "individual", commandOptions.individual, true); + putNestedIfNotEmpty(beanParams, "variantQuery.id", commandOptions.variantQueryId, true); + putNestedIfNotEmpty(beanParams, "variantQuery.region", commandOptions.variantQueryRegion, true); + putNestedIfNotEmpty(beanParams, "variantQuery.gene", commandOptions.variantQueryGene, true); + putNestedIfNotEmpty(beanParams, "variantQuery.type", commandOptions.variantQueryType, true); + putNestedIfNotEmpty(beanParams, "variantQuery.panel", commandOptions.variantQueryPanel, true); + putNestedIfNotEmpty(beanParams, "variantQuery.panelModeOfInheritance", commandOptions.variantQueryPanelModeOfInheritance, true); + putNestedIfNotEmpty(beanParams, "variantQuery.panelConfidence", commandOptions.variantQueryPanelConfidence, true); + putNestedIfNotEmpty(beanParams, "variantQuery.panelRoleInCancer", commandOptions.variantQueryPanelRoleInCancer, true); + putNestedIfNotNull(beanParams, "variantQuery.panelIntersection", commandOptions.variantQueryPanelIntersection, true); + putNestedIfNotEmpty(beanParams, "variantQuery.panelFeatureType", commandOptions.variantQueryPanelFeatureType, true); + putNestedIfNotEmpty(beanParams, "variantQuery.cohortStatsRef", commandOptions.variantQueryCohortStatsRef, true); + putNestedIfNotEmpty(beanParams, "variantQuery.cohortStatsAlt", commandOptions.variantQueryCohortStatsAlt, true); + putNestedIfNotEmpty(beanParams, "variantQuery.cohortStatsMaf", commandOptions.variantQueryCohortStatsMaf, true); + putNestedIfNotEmpty(beanParams, "variantQuery.ct", commandOptions.variantQueryCt, true); + putNestedIfNotEmpty(beanParams, "variantQuery.xref", commandOptions.variantQueryXref, true); + putNestedIfNotEmpty(beanParams, "variantQuery.biotype", commandOptions.variantQueryBiotype, true); + putNestedIfNotEmpty(beanParams, "variantQuery.proteinSubstitution", commandOptions.variantQueryProteinSubstitution, true); + putNestedIfNotEmpty(beanParams, "variantQuery.conservation", commandOptions.variantQueryConservation, true); + putNestedIfNotEmpty(beanParams, "variantQuery.populationFrequencyMaf", commandOptions.variantQueryPopulationFrequencyMaf, true); + putNestedIfNotEmpty(beanParams, "variantQuery.populationFrequencyAlt", commandOptions.variantQueryPopulationFrequencyAlt, true); + putNestedIfNotEmpty(beanParams, "variantQuery.populationFrequencyRef", commandOptions.variantQueryPopulationFrequencyRef, true); + putNestedIfNotEmpty(beanParams, "variantQuery.transcriptFlag", commandOptions.variantQueryTranscriptFlag, true); + putNestedIfNotEmpty(beanParams, "variantQuery.functionalScore", commandOptions.variantQueryFunctionalScore, true); + putNestedIfNotEmpty(beanParams, "variantQuery.clinical", commandOptions.variantQueryClinical, true); + putNestedIfNotEmpty(beanParams, "variantQuery.clinicalSignificance", commandOptions.variantQueryClinicalSignificance, true); + putNestedIfNotNull(beanParams, "variantQuery.clinicalConfirmedStatus", commandOptions.variantQueryClinicalConfirmedStatus, true); + putNestedIfNotEmpty(beanParams, "variantQuery.sampleData", commandOptions.variantQuerySampleData, true); + putNestedIfNotEmpty(beanParams, "variantQuery.fileData", commandOptions.variantQueryFileData, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedIfNotNull(beanParams, "index", commandOptions.index, true); + putNestedIfNotNull(beanParams, "indexOverwrite", commandOptions.indexOverwrite, true); + putNestedIfNotEmpty(beanParams, "indexId", commandOptions.indexId, true); + putNestedIfNotEmpty(beanParams, "indexDescription", commandOptions.indexDescription, true); + putNestedIfNotNull(beanParams, "batchSize", commandOptions.batchSize, true); sampleVariantStatsAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1730,6 +1797,9 @@ private RestResponse runStatsExport() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -1746,11 +1816,11 @@ private RestResponse runStatsExport() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantStatsExportParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "cohorts",commandOptions.cohorts, true); - putNestedIfNotEmpty(beanParams, "output",commandOptions.output, true); - putNestedIfNotEmpty(beanParams, "region",commandOptions.region, true); - putNestedIfNotEmpty(beanParams, "gene",commandOptions.gene, true); - putNestedIfNotEmpty(beanParams, "outputFileFormat",commandOptions.outputFileFormat, true); + putNestedIfNotNull(beanParams, "cohorts", commandOptions.cohorts, true); + putNestedIfNotEmpty(beanParams, "output", commandOptions.output, true); + putNestedIfNotEmpty(beanParams, "region", commandOptions.region, true); + putNestedIfNotEmpty(beanParams, "gene", commandOptions.gene, true); + putNestedIfNotEmpty(beanParams, "outputFileFormat", commandOptions.outputFileFormat, true); variantStatsExportParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1770,6 +1840,9 @@ private RestResponse runStats() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -1786,14 +1859,14 @@ private RestResponse runStats() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantStatsAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "cohort",commandOptions.cohort, true); - putNestedIfNotNull(beanParams, "samples",commandOptions.samples, true); - putNestedIfNotEmpty(beanParams, "region",commandOptions.region, true); - putNestedIfNotEmpty(beanParams, "gene",commandOptions.gene, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotEmpty(beanParams, "outputFileName",commandOptions.outputFileName, true); - putNestedIfNotNull(beanParams, "aggregated",commandOptions.aggregated, true); - putNestedIfNotEmpty(beanParams, "aggregationMappingFile",commandOptions.aggregationMappingFile, true); + putNestedIfNotNull(beanParams, "cohort", commandOptions.cohort, true); + putNestedIfNotNull(beanParams, "samples", commandOptions.samples, true); + putNestedIfNotEmpty(beanParams, "region", commandOptions.region, true); + putNestedIfNotEmpty(beanParams, "gene", commandOptions.gene, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "outputFileName", commandOptions.outputFileName, true); + putNestedIfNotNull(beanParams, "aggregated", commandOptions.aggregated, true); + putNestedIfNotEmpty(beanParams, "aggregationMappingFile", commandOptions.aggregationMappingFile, true); variantStatsAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/CohortsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/CohortsCommandExecutor.java index f2988c96834..287c2223a00 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/CohortsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/CohortsCommandExecutor.java @@ -5,7 +5,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import org.opencb.commons.datastore.core.FacetField; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.utils.PrintUtils; import org.opencb.opencga.app.cli.main.*; @@ -67,9 +66,6 @@ public void execute() throws Exception { case "acl-update": queryResponse = updateAcl(); break; - case "aggregationstats": - queryResponse = aggregationStats(); - break; case "annotation-sets-load": queryResponse = loadAnnotationSets(); break; @@ -132,8 +128,8 @@ private RestResponse updateAcl() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), CohortAclUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "permissions",commandOptions.permissions, true); - putNestedIfNotEmpty(beanParams, "cohort",commandOptions.cohort, true); + putNestedIfNotEmpty(beanParams, "permissions", commandOptions.permissions, true); + putNestedIfNotEmpty(beanParams, "cohort", commandOptions.cohort, true); cohortAclUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -142,31 +138,6 @@ private RestResponse updateAcl() throws Exception { return openCGAClient.getCohortClient().updateAcl(commandOptions.members, commandOptions.action, cohortAclUpdateParams, queryParams); } - private RestResponse aggregationStats() throws Exception { - logger.debug("Executing aggregationStats in Cohorts command line"); - - CohortsCommandOptions.AggregationStatsCommandOptions commandOptions = cohortsCommandOptions.aggregationStatsCommandOptions; - - ObjectMap queryParams = new ObjectMap(); - queryParams.putIfNotEmpty("study", commandOptions.study); - queryParams.putIfNotEmpty("type", commandOptions.type); - queryParams.putIfNotEmpty("creationYear", commandOptions.creationYear); - queryParams.putIfNotEmpty("creationMonth", commandOptions.creationMonth); - queryParams.putIfNotEmpty("creationDay", commandOptions.creationDay); - queryParams.putIfNotEmpty("creationDayOfWeek", commandOptions.creationDayOfWeek); - queryParams.putIfNotEmpty("numSamples", commandOptions.numSamples); - queryParams.putIfNotEmpty("status", commandOptions.status); - queryParams.putIfNotEmpty("release", commandOptions.release); - queryParams.putIfNotEmpty("annotation", commandOptions.annotation); - queryParams.putIfNotNull("default_values", commandOptions.default_values); - queryParams.putIfNotEmpty("field", commandOptions.field); - if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { - queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); - } - - return openCGAClient.getCohortClient().aggregationStats(queryParams); - } - private RestResponse loadAnnotationSets() throws Exception { logger.debug("Executing loadAnnotationSets in Cohorts command line"); @@ -192,7 +163,7 @@ private RestResponse loadAnnotationSets() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), TsvAnnotationParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "content",commandOptions.content, true); + putNestedIfNotEmpty(beanParams, "content", commandOptions.content, true); tsvAnnotationParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -229,16 +200,16 @@ private RestResponse create() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), CohortCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotNull(beanParams, "type",commandOptions.type, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotNull(beanParams, "type", commandOptions.type, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); cohortCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -314,16 +285,16 @@ private RestResponse generate() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), CohortGenerateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.bodyId, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotNull(beanParams, "type",commandOptions.type, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.bodyCreationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.bodyModificationDate, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.bodyId, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotNull(beanParams, "type", commandOptions.type, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.bodyCreationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.bodyModificationDate, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); cohortGenerateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -440,16 +411,16 @@ private RestResponse update() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), CohortUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotNull(beanParams, "type",commandOptions.type, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotNull(beanParams, "type", commandOptions.type, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); cohortUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/DiseasePanelsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/DiseasePanelsCommandExecutor.java index 033aff423b5..d2fde239e71 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/DiseasePanelsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/DiseasePanelsCommandExecutor.java @@ -117,8 +117,8 @@ private RestResponse updateAcl() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), PanelAclUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "permissions",commandOptions.permissions, true); - putNestedIfNotEmpty(beanParams, "panel",commandOptions.panel, true); + putNestedIfNotEmpty(beanParams, "permissions", commandOptions.permissions, true); + putNestedIfNotEmpty(beanParams, "panel", commandOptions.panel, true); panelAclUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -153,17 +153,17 @@ private RestResponse create() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), PanelCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "author",commandOptions.author, true); - putNestedIfNotEmpty(beanParams, "source.id",commandOptions.sourceId, true); - putNestedIfNotEmpty(beanParams, "source.name",commandOptions.sourceName, true); - putNestedIfNotEmpty(beanParams, "source.version",commandOptions.sourceVersion, true); - putNestedIfNotEmpty(beanParams, "source.author",commandOptions.sourceAuthor, true); - putNestedIfNotEmpty(beanParams, "source.project",commandOptions.sourceProject, true); - putNestedIfNotNull(beanParams, "tags",commandOptions.tags, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "author", commandOptions.author, true); + putNestedIfNotEmpty(beanParams, "source.id", commandOptions.sourceId, true); + putNestedIfNotEmpty(beanParams, "source.name", commandOptions.sourceName, true); + putNestedIfNotEmpty(beanParams, "source.version", commandOptions.sourceVersion, true); + putNestedIfNotEmpty(beanParams, "source.author", commandOptions.sourceAuthor, true); + putNestedIfNotEmpty(beanParams, "source.project", commandOptions.sourceProject, true); + putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); panelCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -215,6 +215,9 @@ private RestResponse importPanels() throws Exception { queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -231,8 +234,8 @@ private RestResponse importPanels() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), PanelImportParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "source",commandOptions.source, true); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "source", commandOptions.source, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); panelImportParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -352,17 +355,17 @@ private RestResponse update() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), PanelUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "author",commandOptions.author, true); - putNestedIfNotEmpty(beanParams, "source.id",commandOptions.sourceId, true); - putNestedIfNotEmpty(beanParams, "source.name",commandOptions.sourceName, true); - putNestedIfNotEmpty(beanParams, "source.version",commandOptions.sourceVersion, true); - putNestedIfNotEmpty(beanParams, "source.author",commandOptions.sourceAuthor, true); - putNestedIfNotEmpty(beanParams, "source.project",commandOptions.sourceProject, true); - putNestedIfNotNull(beanParams, "tags",commandOptions.tags, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "author", commandOptions.author, true); + putNestedIfNotEmpty(beanParams, "source.id", commandOptions.sourceId, true); + putNestedIfNotEmpty(beanParams, "source.name", commandOptions.sourceName, true); + putNestedIfNotEmpty(beanParams, "source.version", commandOptions.sourceVersion, true); + putNestedIfNotEmpty(beanParams, "source.author", commandOptions.sourceAuthor, true); + putNestedIfNotEmpty(beanParams, "source.project", commandOptions.sourceProject, true); + putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); panelUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/ExecutorProvider.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/ExecutorProvider.java index b7db12a28db..ebdc2ce4796 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/ExecutorProvider.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/ExecutorProvider.java @@ -10,6 +10,9 @@ public class ExecutorProvider { public static OpencgaCommandExecutor getOpencgaCommandExecutor(OpencgaCliOptionsParser cliOptionsParser, String parsedCommand) throws CatalogAuthenticationException { OpencgaCommandExecutor commandExecutor = null; switch (parsedCommand) { + case "organizations": + commandExecutor = new OrganizationsCommandExecutor(cliOptionsParser.getOrganizationsCommandOptions()); + break; case "users": commandExecutor = new UsersCommandExecutor(cliOptionsParser.getUsersCommandOptions()); break; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FamiliesCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FamiliesCommandExecutor.java index c069ca9c896..dc5a1878e40 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FamiliesCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FamiliesCommandExecutor.java @@ -5,7 +5,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import org.opencb.commons.datastore.core.FacetField; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.utils.PrintUtils; import org.opencb.opencga.app.cli.main.*; @@ -67,9 +66,6 @@ public void execute() throws Exception { case "acl-update": queryResponse = updateAcl(); break; - case "aggregationstats": - queryResponse = aggregationStats(); - break; case "annotation-sets-load": queryResponse = loadAnnotationSets(); break; @@ -130,10 +126,10 @@ private RestResponse updateAcl() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), FamilyAclUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "permissions",commandOptions.permissions, true); - putNestedIfNotEmpty(beanParams, "family",commandOptions.family, true); - putNestedIfNotEmpty(beanParams, "individual",commandOptions.individual, true); - putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); + putNestedIfNotEmpty(beanParams, "permissions", commandOptions.permissions, true); + putNestedIfNotEmpty(beanParams, "family", commandOptions.family, true); + putNestedIfNotEmpty(beanParams, "individual", commandOptions.individual, true); + putNestedIfNotEmpty(beanParams, "sample", commandOptions.sample, true); familyAclUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -142,33 +138,6 @@ private RestResponse updateAcl() throws Exception { return openCGAClient.getFamilyClient().updateAcl(commandOptions.members, commandOptions.action, familyAclUpdateParams, queryParams); } - private RestResponse aggregationStats() throws Exception { - logger.debug("Executing aggregationStats in Families command line"); - - FamiliesCommandOptions.AggregationStatsCommandOptions commandOptions = familiesCommandOptions.aggregationStatsCommandOptions; - - ObjectMap queryParams = new ObjectMap(); - queryParams.putIfNotEmpty("study", commandOptions.study); - queryParams.putIfNotEmpty("creationYear", commandOptions.creationYear); - queryParams.putIfNotEmpty("creationMonth", commandOptions.creationMonth); - queryParams.putIfNotEmpty("creationDay", commandOptions.creationDay); - queryParams.putIfNotEmpty("creationDayOfWeek", commandOptions.creationDayOfWeek); - queryParams.putIfNotEmpty("status", commandOptions.status); - queryParams.putIfNotEmpty("phenotypes", commandOptions.phenotypes); - queryParams.putIfNotEmpty("release", commandOptions.release); - queryParams.putIfNotEmpty("version", commandOptions.version); - queryParams.putIfNotEmpty("numMembers", commandOptions.numMembers); - queryParams.putIfNotEmpty("expectedSize", commandOptions.expectedSize); - queryParams.putIfNotEmpty("annotation", commandOptions.annotation); - queryParams.putIfNotNull("default_values", commandOptions.default_values); - queryParams.putIfNotEmpty("field", commandOptions.field); - if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { - queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); - } - - return openCGAClient.getFamilyClient().aggregationStats(queryParams); - } - private RestResponse loadAnnotationSets() throws Exception { logger.debug("Executing loadAnnotationSets in Families command line"); @@ -194,7 +163,7 @@ private RestResponse loadAnnotationSets() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), TsvAnnotationParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "content",commandOptions.content, true); + putNestedIfNotEmpty(beanParams, "content", commandOptions.content, true); tsvAnnotationParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -230,16 +199,16 @@ private RestResponse create() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), FamilyCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotNull(beanParams, "expectedSize",commandOptions.expectedSize, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotNull(beanParams, "expectedSize", commandOptions.expectedSize, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); familyCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -393,17 +362,17 @@ private RestResponse update() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), FamilyUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotNull(beanParams, "expectedSize",commandOptions.expectedSize, true); - putNestedIfNotNull(beanParams, "qualityControl.files",commandOptions.qualityControlFiles, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotNull(beanParams, "expectedSize", commandOptions.expectedSize, true); + putNestedIfNotNull(beanParams, "qualityControl.files", commandOptions.qualityControlFiles, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); familyUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FilesCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FilesCommandExecutor.java index 0fbaab39b92..873a16baac3 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FilesCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FilesCommandExecutor.java @@ -8,7 +8,6 @@ import java.util.List; import java.util.Map; import org.opencb.biodata.models.clinical.interpretation.Software; -import org.opencb.commons.datastore.core.FacetField; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.utils.PrintUtils; import org.opencb.opencga.app.cli.main.*; @@ -88,9 +87,6 @@ public void execute() throws Exception { case "acl-update": queryResponse = updateAcl(); break; - case "aggregationstats": - queryResponse = aggregationStats(); - break; case "annotation-sets-load": queryResponse = loadAnnotationSets(); break; @@ -201,9 +197,9 @@ private RestResponse updateAcl() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), FileAclUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "permissions",commandOptions.permissions, true); - putNestedIfNotEmpty(beanParams, "file",commandOptions.file, true); - putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); + putNestedIfNotEmpty(beanParams, "permissions", commandOptions.permissions, true); + putNestedIfNotEmpty(beanParams, "file", commandOptions.file, true); + putNestedIfNotEmpty(beanParams, "sample", commandOptions.sample, true); fileAclUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -212,39 +208,6 @@ private RestResponse updateAcl() throws Exception { return openCGAClient.getFileClient().updateAcl(commandOptions.members, commandOptions.action, fileAclUpdateParams, queryParams); } - private RestResponse aggregationStats() throws Exception { - logger.debug("Executing aggregationStats in Files command line"); - - FilesCommandOptions.AggregationStatsCommandOptions commandOptions = filesCommandOptions.aggregationStatsCommandOptions; - - ObjectMap queryParams = new ObjectMap(); - queryParams.putIfNotEmpty("study", commandOptions.study); - queryParams.putIfNotEmpty("name", commandOptions.name); - queryParams.putIfNotEmpty("type", commandOptions.type); - queryParams.putIfNotEmpty("format", commandOptions.format); - queryParams.putIfNotEmpty("bioformat", commandOptions.bioformat); - queryParams.putIfNotEmpty("creationYear", commandOptions.creationYear); - queryParams.putIfNotEmpty("creationMonth", commandOptions.creationMonth); - queryParams.putIfNotEmpty("creationDay", commandOptions.creationDay); - queryParams.putIfNotEmpty("creationDayOfWeek", commandOptions.creationDayOfWeek); - queryParams.putIfNotEmpty("status", commandOptions.status); - queryParams.putIfNotEmpty("release", commandOptions.release); - queryParams.putIfNotNull("external", commandOptions.external); - queryParams.putIfNotEmpty("size", commandOptions.size); - queryParams.putIfNotEmpty("software", commandOptions.software); - queryParams.putIfNotEmpty("experiment", commandOptions.experiment); - queryParams.putIfNotEmpty("numSamples", commandOptions.numSamples); - queryParams.putIfNotEmpty("numRelatedFiles", commandOptions.numRelatedFiles); - queryParams.putIfNotEmpty("annotation", commandOptions.annotation); - queryParams.putIfNotNull("default_values", commandOptions.default_values); - queryParams.putIfNotEmpty("field", commandOptions.field); - if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { - queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); - } - - return openCGAClient.getFileClient().aggregationStats(queryParams); - } - private RestResponse loadAnnotationSets() throws Exception { logger.debug("Executing loadAnnotationSets in Files command line"); @@ -270,7 +233,7 @@ private RestResponse loadAnnotationSets() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), TsvAnnotationParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "content",commandOptions.content, true); + putNestedIfNotEmpty(beanParams, "content", commandOptions.content, true); tsvAnnotationParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -310,27 +273,27 @@ private RestResponse create() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), FileCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "content",commandOptions.content, true); - putNestedIfNotEmpty(beanParams, "path",commandOptions.path, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotNull(beanParams, "type",commandOptions.type, true); - putNestedIfNotNull(beanParams, "format",commandOptions.format, true); - putNestedIfNotNull(beanParams, "bioformat",commandOptions.bioformat, true); - putNestedIfNotNull(beanParams, "sampleIds",commandOptions.sampleIds, true); - putNestedIfNotEmpty(beanParams, "software.name",commandOptions.softwareName, true); - putNestedIfNotEmpty(beanParams, "software.version",commandOptions.softwareVersion, true); - putNestedIfNotEmpty(beanParams, "software.repository",commandOptions.softwareRepository, true); - putNestedIfNotEmpty(beanParams, "software.commit",commandOptions.softwareCommit, true); - putNestedIfNotEmpty(beanParams, "software.website",commandOptions.softwareWebsite, true); - putNestedIfNotNull(beanParams, "software.params",commandOptions.softwareParams, true); - putNestedIfNotNull(beanParams, "tags",commandOptions.tags, true); - putNestedIfNotEmpty(beanParams, "jobId",commandOptions.jobId, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "content", commandOptions.content, true); + putNestedIfNotEmpty(beanParams, "path", commandOptions.path, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotNull(beanParams, "type", commandOptions.type, true); + putNestedIfNotNull(beanParams, "format", commandOptions.format, true); + putNestedIfNotNull(beanParams, "bioformat", commandOptions.bioformat, true); + putNestedIfNotNull(beanParams, "sampleIds", commandOptions.sampleIds, true); + putNestedIfNotEmpty(beanParams, "software.name", commandOptions.softwareName, true); + putNestedIfNotEmpty(beanParams, "software.version", commandOptions.softwareVersion, true); + putNestedIfNotEmpty(beanParams, "software.repository", commandOptions.softwareRepository, true); + putNestedIfNotEmpty(beanParams, "software.commit", commandOptions.softwareCommit, true); + putNestedIfNotEmpty(beanParams, "software.website", commandOptions.softwareWebsite, true); + putNestedMapIfNotEmpty(beanParams, "software.params", commandOptions.softwareParams, true); + putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); + putNestedIfNotEmpty(beanParams, "jobId", commandOptions.jobId, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); fileCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -388,6 +351,9 @@ private RestResponse fetch() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("study", commandOptions.study); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); @@ -405,8 +371,8 @@ private RestResponse fetch() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), FileFetch.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "url",commandOptions.url, true); - putNestedIfNotEmpty(beanParams, "path",commandOptions.path, true); + putNestedIfNotEmpty(beanParams, "url", commandOptions.url, true); + putNestedIfNotEmpty(beanParams, "path", commandOptions.path, true); fileFetch = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -446,16 +412,16 @@ private RestResponse link() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), FileLinkParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "uri",commandOptions.uri, true); - putNestedIfNotEmpty(beanParams, "path",commandOptions.path, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "virtualFileName",commandOptions.virtualFileName, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); - putNestedIfNotNull(beanParams, "internal.sampleMap",commandOptions.internalSampleMap, true); + putNestedIfNotEmpty(beanParams, "uri", commandOptions.uri, true); + putNestedIfNotEmpty(beanParams, "path", commandOptions.path, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "virtualFileName", commandOptions.virtualFileName, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); + putNestedMapIfNotEmpty(beanParams, "internal.sampleMap", commandOptions.internalSampleMap, true); fileLinkParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -475,6 +441,9 @@ private RestResponse runLink() throws Exception { queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -491,12 +460,12 @@ private RestResponse runLink() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), FileLinkToolParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "uri",commandOptions.uri, true); - putNestedIfNotEmpty(beanParams, "path",commandOptions.path, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "virtualFileName",commandOptions.virtualFileName, true); - putNestedIfNotNull(beanParams, "parents",commandOptions.parents, true); - putNestedIfNotNull(beanParams, "skipPostLink",commandOptions.skipPostLink, true); + putNestedIfNotNull(beanParams, "uri", commandOptions.uri, true); + putNestedIfNotEmpty(beanParams, "path", commandOptions.path, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "virtualFileName", commandOptions.virtualFileName, true); + putNestedIfNotNull(beanParams, "parents", commandOptions.parents, true); + putNestedIfNotNull(beanParams, "skipPostLink", commandOptions.skipPostLink, true); fileLinkToolParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -516,6 +485,9 @@ private RestResponse runPostlink() throws Exception { queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -532,8 +504,8 @@ private RestResponse runPostlink() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), PostLinkToolParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "files",commandOptions.files, true); - putNestedIfNotNull(beanParams, "batchSize",commandOptions.batchSize, true); + putNestedIfNotNull(beanParams, "files", commandOptions.files, true); + putNestedIfNotNull(beanParams, "batchSize", commandOptions.batchSize, true); postLinkToolParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -697,37 +669,37 @@ private RestResponse update() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), FileUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotNull(beanParams, "sampleIds",commandOptions.sampleIds, true); - putNestedIfNotNull(beanParams, "format",commandOptions.format, true); - putNestedIfNotNull(beanParams, "bioformat",commandOptions.bioformat, true); - putNestedIfNotEmpty(beanParams, "software.name",commandOptions.softwareName, true); - putNestedIfNotEmpty(beanParams, "software.version",commandOptions.softwareVersion, true); - putNestedIfNotEmpty(beanParams, "software.repository",commandOptions.softwareRepository, true); - putNestedIfNotEmpty(beanParams, "software.commit",commandOptions.softwareCommit, true); - putNestedIfNotEmpty(beanParams, "software.website",commandOptions.softwareWebsite, true); - putNestedIfNotNull(beanParams, "software.params",commandOptions.softwareParams, true); - putNestedIfNotNull(beanParams, "experiment.technology",commandOptions.experimentTechnology, true); - putNestedIfNotNull(beanParams, "experiment.method",commandOptions.experimentMethod, true); - putNestedIfNotNull(beanParams, "experiment.nucleicAcidType",commandOptions.experimentNucleicAcidType, true); - putNestedIfNotEmpty(beanParams, "experiment.manufacturer",commandOptions.experimentManufacturer, true); - putNestedIfNotEmpty(beanParams, "experiment.platform",commandOptions.experimentPlatform, true); - putNestedIfNotEmpty(beanParams, "experiment.library",commandOptions.experimentLibrary, true); - putNestedIfNotEmpty(beanParams, "experiment.date",commandOptions.experimentDate, true); - putNestedIfNotEmpty(beanParams, "experiment.center",commandOptions.experimentCenter, true); - putNestedIfNotEmpty(beanParams, "experiment.lab",commandOptions.experimentLab, true); - putNestedIfNotEmpty(beanParams, "experiment.responsible",commandOptions.experimentResponsible, true); - putNestedIfNotEmpty(beanParams, "experiment.description",commandOptions.experimentDescription, true); - putNestedIfNotNull(beanParams, "experiment.attributes",commandOptions.experimentAttributes, true); - putNestedIfNotNull(beanParams, "tags",commandOptions.tags, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); - putNestedIfNotNull(beanParams, "qualityControl.files",commandOptions.qualityControlFiles, true); - putNestedIfNotNull(beanParams, "stats",commandOptions.stats, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotNull(beanParams, "sampleIds", commandOptions.sampleIds, true); + putNestedIfNotNull(beanParams, "format", commandOptions.format, true); + putNestedIfNotNull(beanParams, "bioformat", commandOptions.bioformat, true); + putNestedIfNotEmpty(beanParams, "software.name", commandOptions.softwareName, true); + putNestedIfNotEmpty(beanParams, "software.version", commandOptions.softwareVersion, true); + putNestedIfNotEmpty(beanParams, "software.repository", commandOptions.softwareRepository, true); + putNestedIfNotEmpty(beanParams, "software.commit", commandOptions.softwareCommit, true); + putNestedIfNotEmpty(beanParams, "software.website", commandOptions.softwareWebsite, true); + putNestedMapIfNotEmpty(beanParams, "software.params", commandOptions.softwareParams, true); + putNestedIfNotNull(beanParams, "experiment.technology", commandOptions.experimentTechnology, true); + putNestedIfNotNull(beanParams, "experiment.method", commandOptions.experimentMethod, true); + putNestedIfNotNull(beanParams, "experiment.nucleicAcidType", commandOptions.experimentNucleicAcidType, true); + putNestedIfNotEmpty(beanParams, "experiment.manufacturer", commandOptions.experimentManufacturer, true); + putNestedIfNotEmpty(beanParams, "experiment.platform", commandOptions.experimentPlatform, true); + putNestedIfNotEmpty(beanParams, "experiment.library", commandOptions.experimentLibrary, true); + putNestedIfNotEmpty(beanParams, "experiment.date", commandOptions.experimentDate, true); + putNestedIfNotEmpty(beanParams, "experiment.center", commandOptions.experimentCenter, true); + putNestedIfNotEmpty(beanParams, "experiment.lab", commandOptions.experimentLab, true); + putNestedIfNotEmpty(beanParams, "experiment.responsible", commandOptions.experimentResponsible, true); + putNestedIfNotEmpty(beanParams, "experiment.description", commandOptions.experimentDescription, true); + putNestedMapIfNotEmpty(beanParams, "experiment.attributes", commandOptions.experimentAttributes, true); + putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); + putNestedIfNotNull(beanParams, "qualityControl.files", commandOptions.qualityControlFiles, true); + putNestedMapIfNotEmpty(beanParams, "stats", commandOptions.stats, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); fileUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -848,7 +820,7 @@ private RestResponse move() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), FileMoveParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "path",commandOptions.path, true); + putNestedIfNotEmpty(beanParams, "path", commandOptions.path, true); fileMoveParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/IndividualsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/IndividualsCommandExecutor.java index 504ff0043b9..dcf24216d8d 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/IndividualsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/IndividualsCommandExecutor.java @@ -9,7 +9,6 @@ import org.opencb.biodata.models.core.OntologyTermAnnotation; import org.opencb.biodata.models.core.SexOntologyTermAnnotation; import org.opencb.biodata.models.pedigree.IndividualProperty; -import org.opencb.commons.datastore.core.FacetField; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.utils.PrintUtils; import org.opencb.opencga.app.cli.main.*; @@ -73,9 +72,6 @@ public void execute() throws Exception { case "acl-update": queryResponse = updateAcl(); break; - case "aggregationstats": - queryResponse = aggregationStats(); - break; case "annotation-sets-load": queryResponse = loadAnnotationSets(); break; @@ -139,9 +135,9 @@ private RestResponse updateAcl() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), IndividualAclUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "permissions",commandOptions.permissions, true); - putNestedIfNotEmpty(beanParams, "individual",commandOptions.individual, true); - putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); + putNestedIfNotEmpty(beanParams, "permissions", commandOptions.permissions, true); + putNestedIfNotEmpty(beanParams, "individual", commandOptions.individual, true); + putNestedIfNotEmpty(beanParams, "sample", commandOptions.sample, true); individualAclUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -150,40 +146,6 @@ private RestResponse updateAcl() throws Exception { return openCGAClient.getIndividualClient().updateAcl(commandOptions.members, commandOptions.action, individualAclUpdateParams, queryParams); } - private RestResponse aggregationStats() throws Exception { - logger.debug("Executing aggregationStats in Individuals command line"); - - IndividualsCommandOptions.AggregationStatsCommandOptions commandOptions = individualsCommandOptions.aggregationStatsCommandOptions; - - ObjectMap queryParams = new ObjectMap(); - queryParams.putIfNotEmpty("study", commandOptions.study); - queryParams.putIfNotNull("hasFather", commandOptions.hasFather); - queryParams.putIfNotNull("hasMother", commandOptions.hasMother); - queryParams.putIfNotEmpty("sex", commandOptions.sex); - queryParams.putIfNotEmpty("karyotypicSex", commandOptions.karyotypicSex); - queryParams.putIfNotEmpty("ethnicity", commandOptions.ethnicity); - queryParams.putIfNotEmpty("population", commandOptions.population); - queryParams.putIfNotEmpty("creationYear", commandOptions.creationYear); - queryParams.putIfNotEmpty("creationMonth", commandOptions.creationMonth); - queryParams.putIfNotEmpty("creationDay", commandOptions.creationDay); - queryParams.putIfNotEmpty("creationDayOfWeek", commandOptions.creationDayOfWeek); - queryParams.putIfNotEmpty("status", commandOptions.status); - queryParams.putIfNotEmpty("lifeStatus", commandOptions.lifeStatus); - queryParams.putIfNotEmpty("phenotypes", commandOptions.phenotypes); - queryParams.putIfNotEmpty("numSamples", commandOptions.numSamples); - queryParams.putIfNotNull("parentalConsanguinity", commandOptions.parentalConsanguinity); - queryParams.putIfNotEmpty("release", commandOptions.release); - queryParams.putIfNotEmpty("version", commandOptions.version); - queryParams.putIfNotEmpty("annotation", commandOptions.annotation); - queryParams.putIfNotNull("default_values", commandOptions.default_values); - queryParams.putIfNotEmpty("field", commandOptions.field); - if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { - queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); - } - - return openCGAClient.getIndividualClient().aggregationStats(queryParams); - } - private RestResponse loadAnnotationSets() throws Exception { logger.debug("Executing loadAnnotationSets in Individuals command line"); @@ -209,7 +171,7 @@ private RestResponse loadAnnotationSets() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), TsvAnnotationParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "content",commandOptions.content, true); + putNestedIfNotEmpty(beanParams, "content", commandOptions.content, true); tsvAnnotationParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -245,42 +207,42 @@ private RestResponse create() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), IndividualCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "father.id",commandOptions.fatherId, true); - putNestedIfNotEmpty(beanParams, "father.uuid",commandOptions.fatherUuid, true); - putNestedIfNotEmpty(beanParams, "mother.id",commandOptions.motherId, true); - putNestedIfNotEmpty(beanParams, "mother.uuid",commandOptions.motherUuid, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "location.address",commandOptions.locationAddress, true); - putNestedIfNotEmpty(beanParams, "location.postalCode",commandOptions.locationPostalCode, true); - putNestedIfNotEmpty(beanParams, "location.city",commandOptions.locationCity, true); - putNestedIfNotEmpty(beanParams, "location.state",commandOptions.locationState, true); - putNestedIfNotEmpty(beanParams, "location.country",commandOptions.locationCountry, true); - putNestedIfNotEmpty(beanParams, "sex.id",commandOptions.sexId, true); - putNestedIfNotEmpty(beanParams, "sex.name",commandOptions.sexName, true); - putNestedIfNotEmpty(beanParams, "sex.description",commandOptions.sexDescription, true); - putNestedIfNotEmpty(beanParams, "sex.source",commandOptions.sexSource, true); - putNestedIfNotEmpty(beanParams, "sex.url",commandOptions.sexUrl, true); - putNestedIfNotNull(beanParams, "sex.attributes",commandOptions.sexAttributes, true); - putNestedIfNotEmpty(beanParams, "ethnicity.id",commandOptions.ethnicityId, true); - putNestedIfNotEmpty(beanParams, "ethnicity.name",commandOptions.ethnicityName, true); - putNestedIfNotEmpty(beanParams, "ethnicity.description",commandOptions.ethnicityDescription, true); - putNestedIfNotEmpty(beanParams, "ethnicity.source",commandOptions.ethnicitySource, true); - putNestedIfNotEmpty(beanParams, "ethnicity.url",commandOptions.ethnicityUrl, true); - putNestedIfNotNull(beanParams, "ethnicity.attributes",commandOptions.ethnicityAttributes, true); - putNestedIfNotNull(beanParams, "parentalConsanguinity",commandOptions.parentalConsanguinity, true); - putNestedIfNotEmpty(beanParams, "population.name",commandOptions.populationName, true); - putNestedIfNotEmpty(beanParams, "population.subpopulation",commandOptions.populationSubpopulation, true); - putNestedIfNotEmpty(beanParams, "population.description",commandOptions.populationDescription, true); - putNestedIfNotEmpty(beanParams, "dateOfBirth",commandOptions.dateOfBirth, true); - putNestedIfNotNull(beanParams, "karyotypicSex",commandOptions.karyotypicSex, true); - putNestedIfNotNull(beanParams, "lifeStatus",commandOptions.lifeStatus, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "father.id", commandOptions.fatherId, true); + putNestedIfNotEmpty(beanParams, "father.uuid", commandOptions.fatherUuid, true); + putNestedIfNotEmpty(beanParams, "mother.id", commandOptions.motherId, true); + putNestedIfNotEmpty(beanParams, "mother.uuid", commandOptions.motherUuid, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "location.address", commandOptions.locationAddress, true); + putNestedIfNotEmpty(beanParams, "location.postalCode", commandOptions.locationPostalCode, true); + putNestedIfNotEmpty(beanParams, "location.city", commandOptions.locationCity, true); + putNestedIfNotEmpty(beanParams, "location.state", commandOptions.locationState, true); + putNestedIfNotEmpty(beanParams, "location.country", commandOptions.locationCountry, true); + putNestedIfNotEmpty(beanParams, "sex.id", commandOptions.sexId, true); + putNestedIfNotEmpty(beanParams, "sex.name", commandOptions.sexName, true); + putNestedIfNotEmpty(beanParams, "sex.description", commandOptions.sexDescription, true); + putNestedIfNotEmpty(beanParams, "sex.source", commandOptions.sexSource, true); + putNestedIfNotEmpty(beanParams, "sex.url", commandOptions.sexUrl, true); + putNestedMapIfNotEmpty(beanParams, "sex.attributes", commandOptions.sexAttributes, true); + putNestedIfNotEmpty(beanParams, "ethnicity.id", commandOptions.ethnicityId, true); + putNestedIfNotEmpty(beanParams, "ethnicity.name", commandOptions.ethnicityName, true); + putNestedIfNotEmpty(beanParams, "ethnicity.description", commandOptions.ethnicityDescription, true); + putNestedIfNotEmpty(beanParams, "ethnicity.source", commandOptions.ethnicitySource, true); + putNestedIfNotEmpty(beanParams, "ethnicity.url", commandOptions.ethnicityUrl, true); + putNestedMapIfNotEmpty(beanParams, "ethnicity.attributes", commandOptions.ethnicityAttributes, true); + putNestedIfNotNull(beanParams, "parentalConsanguinity", commandOptions.parentalConsanguinity, true); + putNestedIfNotEmpty(beanParams, "population.name", commandOptions.populationName, true); + putNestedIfNotEmpty(beanParams, "population.subpopulation", commandOptions.populationSubpopulation, true); + putNestedIfNotEmpty(beanParams, "population.description", commandOptions.populationDescription, true); + putNestedIfNotEmpty(beanParams, "dateOfBirth", commandOptions.dateOfBirth, true); + putNestedIfNotNull(beanParams, "karyotypicSex", commandOptions.karyotypicSex, true); + putNestedIfNotNull(beanParams, "lifeStatus", commandOptions.lifeStatus, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); individualCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -449,43 +411,43 @@ private RestResponse update() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), IndividualUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "father.id",commandOptions.fatherId, true); - putNestedIfNotEmpty(beanParams, "father.uuid",commandOptions.fatherUuid, true); - putNestedIfNotEmpty(beanParams, "mother.id",commandOptions.motherId, true); - putNestedIfNotEmpty(beanParams, "mother.uuid",commandOptions.motherUuid, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotNull(beanParams, "parentalConsanguinity",commandOptions.parentalConsanguinity, true); - putNestedIfNotEmpty(beanParams, "location.address",commandOptions.locationAddress, true); - putNestedIfNotEmpty(beanParams, "location.postalCode",commandOptions.locationPostalCode, true); - putNestedIfNotEmpty(beanParams, "location.city",commandOptions.locationCity, true); - putNestedIfNotEmpty(beanParams, "location.state",commandOptions.locationState, true); - putNestedIfNotEmpty(beanParams, "location.country",commandOptions.locationCountry, true); - putNestedIfNotEmpty(beanParams, "sex.id",commandOptions.sexId, true); - putNestedIfNotEmpty(beanParams, "sex.name",commandOptions.sexName, true); - putNestedIfNotEmpty(beanParams, "sex.description",commandOptions.sexDescription, true); - putNestedIfNotEmpty(beanParams, "sex.source",commandOptions.sexSource, true); - putNestedIfNotEmpty(beanParams, "sex.url",commandOptions.sexUrl, true); - putNestedIfNotNull(beanParams, "sex.attributes",commandOptions.sexAttributes, true); - putNestedIfNotEmpty(beanParams, "ethnicity.id",commandOptions.ethnicityId, true); - putNestedIfNotEmpty(beanParams, "ethnicity.name",commandOptions.ethnicityName, true); - putNestedIfNotEmpty(beanParams, "ethnicity.description",commandOptions.ethnicityDescription, true); - putNestedIfNotEmpty(beanParams, "ethnicity.source",commandOptions.ethnicitySource, true); - putNestedIfNotEmpty(beanParams, "ethnicity.url",commandOptions.ethnicityUrl, true); - putNestedIfNotNull(beanParams, "ethnicity.attributes",commandOptions.ethnicityAttributes, true); - putNestedIfNotEmpty(beanParams, "population.name",commandOptions.populationName, true); - putNestedIfNotEmpty(beanParams, "population.subpopulation",commandOptions.populationSubpopulation, true); - putNestedIfNotEmpty(beanParams, "population.description",commandOptions.populationDescription, true); - putNestedIfNotEmpty(beanParams, "dateOfBirth",commandOptions.dateOfBirth, true); - putNestedIfNotNull(beanParams, "karyotypicSex",commandOptions.karyotypicSex, true); - putNestedIfNotNull(beanParams, "lifeStatus",commandOptions.lifeStatus, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); - putNestedIfNotNull(beanParams, "qualityControl.files",commandOptions.qualityControlFiles, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "father.id", commandOptions.fatherId, true); + putNestedIfNotEmpty(beanParams, "father.uuid", commandOptions.fatherUuid, true); + putNestedIfNotEmpty(beanParams, "mother.id", commandOptions.motherId, true); + putNestedIfNotEmpty(beanParams, "mother.uuid", commandOptions.motherUuid, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotNull(beanParams, "parentalConsanguinity", commandOptions.parentalConsanguinity, true); + putNestedIfNotEmpty(beanParams, "location.address", commandOptions.locationAddress, true); + putNestedIfNotEmpty(beanParams, "location.postalCode", commandOptions.locationPostalCode, true); + putNestedIfNotEmpty(beanParams, "location.city", commandOptions.locationCity, true); + putNestedIfNotEmpty(beanParams, "location.state", commandOptions.locationState, true); + putNestedIfNotEmpty(beanParams, "location.country", commandOptions.locationCountry, true); + putNestedIfNotEmpty(beanParams, "sex.id", commandOptions.sexId, true); + putNestedIfNotEmpty(beanParams, "sex.name", commandOptions.sexName, true); + putNestedIfNotEmpty(beanParams, "sex.description", commandOptions.sexDescription, true); + putNestedIfNotEmpty(beanParams, "sex.source", commandOptions.sexSource, true); + putNestedIfNotEmpty(beanParams, "sex.url", commandOptions.sexUrl, true); + putNestedMapIfNotEmpty(beanParams, "sex.attributes", commandOptions.sexAttributes, true); + putNestedIfNotEmpty(beanParams, "ethnicity.id", commandOptions.ethnicityId, true); + putNestedIfNotEmpty(beanParams, "ethnicity.name", commandOptions.ethnicityName, true); + putNestedIfNotEmpty(beanParams, "ethnicity.description", commandOptions.ethnicityDescription, true); + putNestedIfNotEmpty(beanParams, "ethnicity.source", commandOptions.ethnicitySource, true); + putNestedIfNotEmpty(beanParams, "ethnicity.url", commandOptions.ethnicityUrl, true); + putNestedMapIfNotEmpty(beanParams, "ethnicity.attributes", commandOptions.ethnicityAttributes, true); + putNestedIfNotEmpty(beanParams, "population.name", commandOptions.populationName, true); + putNestedIfNotEmpty(beanParams, "population.subpopulation", commandOptions.populationSubpopulation, true); + putNestedIfNotEmpty(beanParams, "population.description", commandOptions.populationDescription, true); + putNestedIfNotEmpty(beanParams, "dateOfBirth", commandOptions.dateOfBirth, true); + putNestedIfNotNull(beanParams, "karyotypicSex", commandOptions.karyotypicSex, true); + putNestedIfNotNull(beanParams, "lifeStatus", commandOptions.lifeStatus, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); + putNestedIfNotNull(beanParams, "qualityControl.files", commandOptions.qualityControlFiles, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); individualUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/JobsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/JobsCommandExecutor.java index 586f631b19d..ffc4c7a61a1 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/JobsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/JobsCommandExecutor.java @@ -5,7 +5,6 @@ import java.util.Date; import java.util.HashMap; import java.util.List; -import org.opencb.commons.datastore.core.FacetField; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.utils.PrintUtils; import org.opencb.opencga.app.cli.main.*; @@ -71,9 +70,6 @@ public void execute() throws Exception { case "acl-update": queryResponse = updateAcl(); break; - case "aggregationstats": - queryResponse = aggregationStats(); - break; case "create": queryResponse = create(); break; @@ -101,6 +97,9 @@ public void execute() throws Exception { case "update": queryResponse = update(); break; + case "kill": + queryResponse = kill(); + break; case "log-head": queryResponse = headLog(); break; @@ -138,8 +137,8 @@ private RestResponse updateAcl() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), JobAclUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "permissions",commandOptions.permissions, true); - putNestedIfNotEmpty(beanParams, "job",commandOptions.job, true); + putNestedIfNotEmpty(beanParams, "permissions", commandOptions.permissions, true); + putNestedIfNotEmpty(beanParams, "job", commandOptions.job, true); jobAclUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -148,37 +147,6 @@ private RestResponse updateAcl() throws Exception { return openCGAClient.getJobClient().updateAcl(commandOptions.members, commandOptions.action, jobAclUpdateParams); } - private RestResponse aggregationStats() throws Exception { - logger.debug("Executing aggregationStats in Jobs command line"); - - JobsCommandOptions.AggregationStatsCommandOptions commandOptions = jobsCommandOptions.aggregationStatsCommandOptions; - - ObjectMap queryParams = new ObjectMap(); - queryParams.putIfNotEmpty("study", commandOptions.study); - queryParams.putIfNotEmpty("toolId", commandOptions.toolId); - queryParams.putIfNotEmpty("toolScope", commandOptions.toolScope); - queryParams.putIfNotEmpty("toolType", commandOptions.toolType); - queryParams.putIfNotEmpty("toolResource", commandOptions.toolResource); - queryParams.putIfNotEmpty("userId", commandOptions.userId); - queryParams.putIfNotEmpty("priority", commandOptions.priority); - queryParams.putIfNotEmpty("tags", commandOptions.tags); - queryParams.putIfNotEmpty("executorId", commandOptions.executorId); - queryParams.putIfNotEmpty("executorFramework", commandOptions.executorFramework); - queryParams.putIfNotEmpty("creationYear", commandOptions.creationYear); - queryParams.putIfNotEmpty("creationMonth", commandOptions.creationMonth); - queryParams.putIfNotEmpty("creationDay", commandOptions.creationDay); - queryParams.putIfNotEmpty("creationDayOfWeek", commandOptions.creationDayOfWeek); - queryParams.putIfNotEmpty("status", commandOptions.status); - queryParams.putIfNotEmpty("release", commandOptions.release); - queryParams.putIfNotNull("default_values", commandOptions.default_values); - queryParams.putIfNotEmpty("field", commandOptions.field); - if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { - queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); - } - - return openCGAClient.getJobClient().aggregationStats(queryParams); - } - private RestResponse create() throws Exception { logger.debug("Executing create in Jobs command line"); @@ -202,25 +170,25 @@ private RestResponse create() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), JobCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "tool.id",commandOptions.toolId, true); - putNestedIfNotEmpty(beanParams, "tool.description",commandOptions.toolDescription, true); - putNestedIfNotNull(beanParams, "tool.scope",commandOptions.toolScope, true); - putNestedIfNotNull(beanParams, "tool.type",commandOptions.toolType, true); - putNestedIfNotNull(beanParams, "tool.resource",commandOptions.toolResource, true); - putNestedIfNotNull(beanParams, "priority",commandOptions.priority, true); - putNestedIfNotEmpty(beanParams, "commandLine",commandOptions.commandLine, true); - putNestedIfNotNull(beanParams, "params",commandOptions.params, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "outDir.path",commandOptions.outDirPath, true); - putNestedIfNotNull(beanParams, "tags",commandOptions.tags, true); - putNestedIfNotEmpty(beanParams, "result.id",commandOptions.resultId, true); - putNestedIfNotNull(beanParams, "result.attributes",commandOptions.resultAttributes, true); - putNestedIfNotEmpty(beanParams, "stdout.path",commandOptions.stdoutPath, true); - putNestedIfNotEmpty(beanParams, "stderr.path",commandOptions.stderrPath, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "tool.id", commandOptions.toolId, true); + putNestedIfNotEmpty(beanParams, "tool.description", commandOptions.toolDescription, true); + putNestedIfNotNull(beanParams, "tool.scope", commandOptions.toolScope, true); + putNestedIfNotNull(beanParams, "tool.type", commandOptions.toolType, true); + putNestedIfNotNull(beanParams, "tool.resource", commandOptions.toolResource, true); + putNestedIfNotNull(beanParams, "priority", commandOptions.priority, true); + putNestedIfNotEmpty(beanParams, "commandLine", commandOptions.commandLine, true); + putNestedMapIfNotEmpty(beanParams, "params", commandOptions.params, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "outDir.path", commandOptions.outDirPath, true); + putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); + putNestedIfNotEmpty(beanParams, "result.id", commandOptions.resultId, true); + putNestedMapIfNotEmpty(beanParams, "result.attributes", commandOptions.resultAttributes, true); + putNestedIfNotEmpty(beanParams, "stdout.path", commandOptions.stdoutPath, true); + putNestedIfNotEmpty(beanParams, "stderr.path", commandOptions.stderrPath, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); jobCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -271,6 +239,7 @@ private RestResponse retry() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); queryParams.putIfNotEmpty("study", commandOptions.study); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); @@ -288,9 +257,9 @@ private RestResponse retry() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), JobRetryParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "job",commandOptions.job, true); - putNestedIfNotNull(beanParams, "force",commandOptions.force, true); - putNestedIfNotNull(beanParams, "params",commandOptions.params, true); + putNestedIfNotEmpty(beanParams, "job", commandOptions.job, true); + putNestedIfNotNull(beanParams, "force", commandOptions.force, true); + putNestedMapIfNotEmpty(beanParams, "params", commandOptions.params, true); jobRetryParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -423,10 +392,10 @@ private RestResponse update() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), JobUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotNull(beanParams, "tags",commandOptions.tags, true); - putNestedIfNotNull(beanParams, "visited",commandOptions.visited, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); + putNestedIfNotNull(beanParams, "visited", commandOptions.visited, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); jobUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -435,6 +404,20 @@ private RestResponse update() throws Exception { return openCGAClient.getJobClient().update(commandOptions.jobs, jobUpdateParams, queryParams); } + private RestResponse kill() throws Exception { + logger.debug("Executing kill in Jobs command line"); + + JobsCommandOptions.KillCommandOptions commandOptions = jobsCommandOptions.killCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("study", commandOptions.study); + if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { + queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); + } + + return openCGAClient.getJobClient().kill(commandOptions.job, queryParams); + } + private RestResponse headLog() throws Exception { logger.debug("Executing headLog in Jobs command line"); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OpencgaCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OpencgaCommandExecutor.java index 72f9bdd0e8c..f1e0c47ad70 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OpencgaCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OpencgaCommandExecutor.java @@ -17,8 +17,8 @@ package org.opencb.opencga.app.cli.main.executors; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; import org.opencb.commons.datastore.core.ObjectMap; @@ -36,19 +36,13 @@ import org.opencb.opencga.core.models.user.AuthenticationResponse; import org.opencb.opencga.core.response.QueryType; import org.opencb.opencga.core.response.RestResponse; -import org.opencb.opencga.server.generator.models.RestCategory; -import org.opencb.opencga.server.generator.models.RestEndpoint; -import org.opencb.opencga.server.generator.models.RestParameter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.lang.reflect.Method; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Base64; -import java.util.Date; -import java.util.List; +import java.util.*; /** * Created on 27/05/16. @@ -266,18 +260,22 @@ public RestResponse refreshToken(AuthenticationResponse return res; } - public Object putNestedIfNotNull(ObjectMap map, String key, Object value, boolean parents) { + public void putNestedIfNotNull(ObjectMap map, String key, Object value, boolean parents) { if (value != null) { map.putNested(key, value, parents); } - return null; } - public Object putNestedIfNotEmpty(ObjectMap map, String key, String value, boolean parents) { + public void putNestedIfNotEmpty(ObjectMap map, String key, String value, boolean parents) { if (StringUtils.isNotEmpty(value)) { map.putNested(key, value, parents); } - return null; + } + + public void putNestedMapIfNotEmpty(ObjectMap map, String key, Map value, boolean parents) { + if (MapUtils.isNotEmpty(value)) { + map.putNested(key, value, parents); + } } public boolean checkExpiredSession(String[] args) { diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OperationsVariantStorageCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OperationsVariantStorageCommandExecutor.java index 0b3a28d11b3..35f0c49c312 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OperationsVariantStorageCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OperationsVariantStorageCommandExecutor.java @@ -37,6 +37,8 @@ import org.opencb.opencga.core.models.operations.variant.VariantStorageMetadataRepairToolParams; import org.opencb.opencga.core.models.operations.variant.VariantStorageMetadataSynchronizeParams; import org.opencb.opencga.core.models.operations.variant.VariantStudyDeleteParams; +import org.opencb.opencga.core.models.study.VariantSetupResult; +import org.opencb.opencga.core.models.variant.VariantSetupParams; import org.opencb.opencga.core.response.QueryType; import org.opencb.opencga.core.response.RestResponse; @@ -149,6 +151,9 @@ public void execute() throws Exception { case "variant-secondary-index-delete": queryResponse = deleteVariantSecondaryIndex(); break; + case "variant-setup": + queryResponse = setupVariant(); + break; case "variant-stats-delete": queryResponse = deleteVariantStats(); break; @@ -189,10 +194,10 @@ private RestResponse configureCellbase() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), CellBaseConfiguration.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "url",commandOptions.url, true); - putNestedIfNotEmpty(beanParams, "version",commandOptions.version, true); - putNestedIfNotEmpty(beanParams, "dataRelease",commandOptions.dataRelease, true); - putNestedIfNotEmpty(beanParams, "apiKey",commandOptions.apiKey, true); + putNestedIfNotEmpty(beanParams, "url", commandOptions.url, true); + putNestedIfNotEmpty(beanParams, "version", commandOptions.version, true); + putNestedIfNotEmpty(beanParams, "dataRelease", commandOptions.dataRelease, true); + putNestedIfNotEmpty(beanParams, "apiKey", commandOptions.apiKey, true); cellBaseConfiguration = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -211,6 +216,9 @@ private RestResponse aggregateVariant() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("study", commandOptions.study); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); @@ -228,8 +236,8 @@ private RestResponse aggregateVariant() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantAggregateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "overwrite",commandOptions.overwrite, true); - putNestedIfNotNull(beanParams, "resume",commandOptions.resume, true); + putNestedIfNotNull(beanParams, "overwrite", commandOptions.overwrite, true); + putNestedIfNotNull(beanParams, "resume", commandOptions.resume, true); variantAggregateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -248,6 +256,9 @@ private RestResponse deleteVariantAnnotation() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("project", commandOptions.project); queryParams.putIfNotEmpty("annotationId", commandOptions.annotationId); @@ -264,6 +275,9 @@ private RestResponse indexVariantAnnotation() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("project", commandOptions.project); queryParams.putIfNotEmpty("study", commandOptions.study); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { @@ -282,15 +296,15 @@ private RestResponse indexVariantAnnotation() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantAnnotationIndexParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotEmpty(beanParams, "outputFileName",commandOptions.outputFileName, true); - putNestedIfNotEmpty(beanParams, "annotator",commandOptions.annotator, true); - putNestedIfNotNull(beanParams, "overwriteAnnotations",commandOptions.overwriteAnnotations, true); - putNestedIfNotEmpty(beanParams, "region",commandOptions.region, true); - putNestedIfNotNull(beanParams, "create",commandOptions.create, true); - putNestedIfNotEmpty(beanParams, "load",commandOptions.load, true); - putNestedIfNotEmpty(beanParams, "customName",commandOptions.customName, true); - putNestedIfNotNull(beanParams, "sampleIndexAnnotation",commandOptions.sampleIndexAnnotation, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedIfNotEmpty(beanParams, "outputFileName", commandOptions.outputFileName, true); + putNestedIfNotEmpty(beanParams, "annotator", commandOptions.annotator, true); + putNestedIfNotNull(beanParams, "overwriteAnnotations", commandOptions.overwriteAnnotations, true); + putNestedIfNotEmpty(beanParams, "region", commandOptions.region, true); + putNestedIfNotNull(beanParams, "create", commandOptions.create, true); + putNestedIfNotEmpty(beanParams, "load", commandOptions.load, true); + putNestedIfNotEmpty(beanParams, "customName", commandOptions.customName, true); + putNestedIfNotNull(beanParams, "sampleIndexAnnotation", commandOptions.sampleIndexAnnotation, true); variantAnnotationIndexParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -309,6 +323,9 @@ private RestResponse saveVariantAnnotation() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("project", commandOptions.project); @@ -323,7 +340,7 @@ private RestResponse saveVariantAnnotation() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantAnnotationSaveParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "annotationId",commandOptions.annotationId, true); + putNestedIfNotEmpty(beanParams, "annotationId", commandOptions.annotationId, true); variantAnnotationSaveParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -356,7 +373,7 @@ private RestResponse configureVariant() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantConfigureParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "configuration",commandOptions.configuration, true); + putNestedMapIfNotEmpty(beanParams, "configuration", commandOptions.configuration, true); variantConfigureParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -375,6 +392,9 @@ private RestResponse deleteVariant() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("study", commandOptions.study); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); @@ -392,8 +412,9 @@ private RestResponse deleteVariant() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantFileDeleteParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "file",commandOptions.file, true); - putNestedIfNotNull(beanParams, "resume",commandOptions.resume, true); + putNestedIfNotNull(beanParams, "file", commandOptions.file, true); + putNestedIfNotNull(beanParams, "resume", commandOptions.resume, true); + putNestedIfNotNull(beanParams, "force", commandOptions.force, true); variantFileDeleteParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -412,6 +433,9 @@ private RestResponse aggregateVariantFamily() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("study", commandOptions.study); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); @@ -429,9 +453,9 @@ private RestResponse aggregateVariantFamily() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantAggregateFamilyParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "samples",commandOptions.samples, true); - putNestedIfNotEmpty(beanParams, "gapsGenotype",commandOptions.gapsGenotype, true); - putNestedIfNotNull(beanParams, "resume",commandOptions.resume, true); + putNestedIfNotNull(beanParams, "samples", commandOptions.samples, true); + putNestedIfNotEmpty(beanParams, "gapsGenotype", commandOptions.gapsGenotype, true); + putNestedIfNotNull(beanParams, "resume", commandOptions.resume, true); variantAggregateFamilyParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -450,6 +474,9 @@ private RestResponse indexVariantFamily() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("study", commandOptions.study); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); @@ -467,10 +494,10 @@ private RestResponse indexVariantFamily() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantFamilyIndexParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "family",commandOptions.family, true); - putNestedIfNotNull(beanParams, "overwrite",commandOptions.overwrite, true); - putNestedIfNotNull(beanParams, "updateIndex",commandOptions.updateIndex, true); - putNestedIfNotNull(beanParams, "skipIncompleteFamilies",commandOptions.skipIncompleteFamilies, true); + putNestedIfNotNull(beanParams, "family", commandOptions.family, true); + putNestedIfNotNull(beanParams, "overwrite", commandOptions.overwrite, true); + putNestedIfNotNull(beanParams, "updateIndex", commandOptions.updateIndex, true); + putNestedIfNotNull(beanParams, "skipIncompleteFamilies", commandOptions.skipIncompleteFamilies, true); variantFamilyIndexParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -489,6 +516,9 @@ private RestResponse indexVariant() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("study", commandOptions.study); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); @@ -506,36 +536,36 @@ private RestResponse indexVariant() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantIndexParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "file",commandOptions.file, true); - putNestedIfNotNull(beanParams, "resume",commandOptions.resume, true); - putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); - putNestedIfNotNull(beanParams, "transform",commandOptions.transform, true); - putNestedIfNotNull(beanParams, "gvcf",commandOptions.gvcf, true); - putNestedIfNotNull(beanParams, "normalizationSkip",commandOptions.normalizationSkip, true); - putNestedIfNotEmpty(beanParams, "referenceGenome",commandOptions.referenceGenome, true); - putNestedIfNotEmpty(beanParams, "failOnMalformedLines",commandOptions.failOnMalformedLines, true); - putNestedIfNotNull(beanParams, "family",commandOptions.family, true); - putNestedIfNotNull(beanParams, "somatic",commandOptions.somatic, true); - putNestedIfNotNull(beanParams, "load",commandOptions.load, true); - putNestedIfNotNull(beanParams, "forceReload",commandOptions.forceReload, true); - putNestedIfNotEmpty(beanParams, "loadSplitData",commandOptions.loadSplitData, true); - putNestedIfNotNull(beanParams, "loadMultiFileData",commandOptions.loadMultiFileData, true); - putNestedIfNotEmpty(beanParams, "loadSampleIndex",commandOptions.loadSampleIndex, true); - putNestedIfNotEmpty(beanParams, "loadArchive",commandOptions.loadArchive, true); - putNestedIfNotEmpty(beanParams, "loadHomRef",commandOptions.loadHomRef, true); - putNestedIfNotEmpty(beanParams, "postLoadCheck",commandOptions.postLoadCheck, true); - putNestedIfNotEmpty(beanParams, "includeGenotypes",commandOptions.includeGenotypes, true); - putNestedIfNotEmpty(beanParams, "includeSampleData",commandOptions.includeSampleData, true); - putNestedIfNotEmpty(beanParams, "merge",commandOptions.merge, true); - putNestedIfNotEmpty(beanParams, "deduplicationPolicy",commandOptions.deduplicationPolicy, true); - putNestedIfNotNull(beanParams, "calculateStats",commandOptions.calculateStats, true); - putNestedIfNotNull(beanParams, "aggregated",commandOptions.aggregated, true); - putNestedIfNotEmpty(beanParams, "aggregationMappingFile",commandOptions.aggregationMappingFile, true); - putNestedIfNotNull(beanParams, "annotate",commandOptions.annotate, true); - putNestedIfNotEmpty(beanParams, "annotator",commandOptions.annotator, true); - putNestedIfNotNull(beanParams, "overwriteAnnotations",commandOptions.overwriteAnnotations, true); - putNestedIfNotNull(beanParams, "indexSearch",commandOptions.indexSearch, true); - putNestedIfNotNull(beanParams, "skipIndexedFiles",commandOptions.skipIndexedFiles, true); + putNestedIfNotEmpty(beanParams, "file", commandOptions.file, true); + putNestedIfNotNull(beanParams, "resume", commandOptions.resume, true); + putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); + putNestedIfNotNull(beanParams, "transform", commandOptions.transform, true); + putNestedIfNotNull(beanParams, "gvcf", commandOptions.gvcf, true); + putNestedIfNotNull(beanParams, "normalizationSkip", commandOptions.normalizationSkip, true); + putNestedIfNotEmpty(beanParams, "referenceGenome", commandOptions.referenceGenome, true); + putNestedIfNotEmpty(beanParams, "failOnMalformedLines", commandOptions.failOnMalformedLines, true); + putNestedIfNotNull(beanParams, "family", commandOptions.family, true); + putNestedIfNotNull(beanParams, "somatic", commandOptions.somatic, true); + putNestedIfNotNull(beanParams, "load", commandOptions.load, true); + putNestedIfNotNull(beanParams, "forceReload", commandOptions.forceReload, true); + putNestedIfNotEmpty(beanParams, "loadSplitData", commandOptions.loadSplitData, true); + putNestedIfNotNull(beanParams, "loadMultiFileData", commandOptions.loadMultiFileData, true); + putNestedIfNotEmpty(beanParams, "loadSampleIndex", commandOptions.loadSampleIndex, true); + putNestedIfNotEmpty(beanParams, "loadArchive", commandOptions.loadArchive, true); + putNestedIfNotEmpty(beanParams, "loadHomRef", commandOptions.loadHomRef, true); + putNestedIfNotEmpty(beanParams, "postLoadCheck", commandOptions.postLoadCheck, true); + putNestedIfNotEmpty(beanParams, "includeGenotypes", commandOptions.includeGenotypes, true); + putNestedIfNotEmpty(beanParams, "includeSampleData", commandOptions.includeSampleData, true); + putNestedIfNotEmpty(beanParams, "merge", commandOptions.merge, true); + putNestedIfNotEmpty(beanParams, "deduplicationPolicy", commandOptions.deduplicationPolicy, true); + putNestedIfNotNull(beanParams, "calculateStats", commandOptions.calculateStats, true); + putNestedIfNotNull(beanParams, "aggregated", commandOptions.aggregated, true); + putNestedIfNotEmpty(beanParams, "aggregationMappingFile", commandOptions.aggregationMappingFile, true); + putNestedIfNotNull(beanParams, "annotate", commandOptions.annotate, true); + putNestedIfNotEmpty(beanParams, "annotator", commandOptions.annotator, true); + putNestedIfNotNull(beanParams, "overwriteAnnotations", commandOptions.overwriteAnnotations, true); + putNestedIfNotNull(beanParams, "indexSearch", commandOptions.indexSearch, true); + putNestedIfNotNull(beanParams, "skipIndexedFiles", commandOptions.skipIndexedFiles, true); variantIndexParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -554,6 +584,9 @@ private RestResponse launcherVariantIndex() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("study", commandOptions.study); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); @@ -571,41 +604,41 @@ private RestResponse launcherVariantIndex() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantFileIndexJobLauncherParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "directory",commandOptions.directory, true); - putNestedIfNotNull(beanParams, "resumeFailed",commandOptions.resumeFailed, true); - putNestedIfNotNull(beanParams, "ignoreFailed",commandOptions.ignoreFailed, true); - putNestedIfNotNull(beanParams, "maxJobs",commandOptions.maxJobs, true); - putNestedIfNotEmpty(beanParams, "indexParams.file",commandOptions.indexParamsFile, true); - putNestedIfNotNull(beanParams, "indexParams.resume",commandOptions.indexParamsResume, true); - putNestedIfNotEmpty(beanParams, "indexParams.outdir",commandOptions.indexParamsOutdir, true); - putNestedIfNotNull(beanParams, "indexParams.transform",commandOptions.indexParamsTransform, true); - putNestedIfNotNull(beanParams, "indexParams.gvcf",commandOptions.indexParamsGvcf, true); - putNestedIfNotNull(beanParams, "indexParams.normalizationSkip",commandOptions.indexParamsNormalizationSkip, true); - putNestedIfNotEmpty(beanParams, "indexParams.referenceGenome",commandOptions.indexParamsReferenceGenome, true); - putNestedIfNotEmpty(beanParams, "indexParams.failOnMalformedLines",commandOptions.indexParamsFailOnMalformedLines, true); - putNestedIfNotNull(beanParams, "indexParams.family",commandOptions.indexParamsFamily, true); - putNestedIfNotNull(beanParams, "indexParams.somatic",commandOptions.indexParamsSomatic, true); - putNestedIfNotNull(beanParams, "indexParams.load",commandOptions.indexParamsLoad, true); - putNestedIfNotNull(beanParams, "indexParams.forceReload",commandOptions.indexParamsForceReload, true); - putNestedIfNotEmpty(beanParams, "indexParams.loadSplitData",commandOptions.indexParamsLoadSplitData, true); - putNestedIfNotNull(beanParams, "indexParams.loadMultiFileData",commandOptions.indexParamsLoadMultiFileData, true); - putNestedIfNotEmpty(beanParams, "indexParams.loadSampleIndex",commandOptions.indexParamsLoadSampleIndex, true); - putNestedIfNotEmpty(beanParams, "indexParams.loadArchive",commandOptions.indexParamsLoadArchive, true); - putNestedIfNotEmpty(beanParams, "indexParams.loadHomRef",commandOptions.indexParamsLoadHomRef, true); - putNestedIfNotEmpty(beanParams, "indexParams.postLoadCheck",commandOptions.indexParamsPostLoadCheck, true); - putNestedIfNotEmpty(beanParams, "indexParams.includeGenotypes",commandOptions.indexParamsIncludeGenotypes, true); - putNestedIfNotEmpty(beanParams, "indexParams.includeSampleData",commandOptions.indexParamsIncludeSampleData, true); - putNestedIfNotEmpty(beanParams, "indexParams.merge",commandOptions.indexParamsMerge, true); - putNestedIfNotEmpty(beanParams, "indexParams.deduplicationPolicy",commandOptions.indexParamsDeduplicationPolicy, true); - putNestedIfNotNull(beanParams, "indexParams.calculateStats",commandOptions.indexParamsCalculateStats, true); - putNestedIfNotNull(beanParams, "indexParams.aggregated",commandOptions.indexParamsAggregated, true); - putNestedIfNotEmpty(beanParams, "indexParams.aggregationMappingFile",commandOptions.indexParamsAggregationMappingFile, true); - putNestedIfNotNull(beanParams, "indexParams.annotate",commandOptions.indexParamsAnnotate, true); - putNestedIfNotEmpty(beanParams, "indexParams.annotator",commandOptions.indexParamsAnnotator, true); - putNestedIfNotNull(beanParams, "indexParams.overwriteAnnotations",commandOptions.indexParamsOverwriteAnnotations, true); - putNestedIfNotNull(beanParams, "indexParams.indexSearch",commandOptions.indexParamsIndexSearch, true); - putNestedIfNotNull(beanParams, "indexParams.skipIndexedFiles",commandOptions.indexParamsSkipIndexedFiles, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "directory", commandOptions.directory, true); + putNestedIfNotNull(beanParams, "resumeFailed", commandOptions.resumeFailed, true); + putNestedIfNotNull(beanParams, "ignoreFailed", commandOptions.ignoreFailed, true); + putNestedIfNotNull(beanParams, "maxJobs", commandOptions.maxJobs, true); + putNestedIfNotEmpty(beanParams, "indexParams.file", commandOptions.indexParamsFile, true); + putNestedIfNotNull(beanParams, "indexParams.resume", commandOptions.indexParamsResume, true); + putNestedIfNotEmpty(beanParams, "indexParams.outdir", commandOptions.indexParamsOutdir, true); + putNestedIfNotNull(beanParams, "indexParams.transform", commandOptions.indexParamsTransform, true); + putNestedIfNotNull(beanParams, "indexParams.gvcf", commandOptions.indexParamsGvcf, true); + putNestedIfNotNull(beanParams, "indexParams.normalizationSkip", commandOptions.indexParamsNormalizationSkip, true); + putNestedIfNotEmpty(beanParams, "indexParams.referenceGenome", commandOptions.indexParamsReferenceGenome, true); + putNestedIfNotEmpty(beanParams, "indexParams.failOnMalformedLines", commandOptions.indexParamsFailOnMalformedLines, true); + putNestedIfNotNull(beanParams, "indexParams.family", commandOptions.indexParamsFamily, true); + putNestedIfNotNull(beanParams, "indexParams.somatic", commandOptions.indexParamsSomatic, true); + putNestedIfNotNull(beanParams, "indexParams.load", commandOptions.indexParamsLoad, true); + putNestedIfNotNull(beanParams, "indexParams.forceReload", commandOptions.indexParamsForceReload, true); + putNestedIfNotEmpty(beanParams, "indexParams.loadSplitData", commandOptions.indexParamsLoadSplitData, true); + putNestedIfNotNull(beanParams, "indexParams.loadMultiFileData", commandOptions.indexParamsLoadMultiFileData, true); + putNestedIfNotEmpty(beanParams, "indexParams.loadSampleIndex", commandOptions.indexParamsLoadSampleIndex, true); + putNestedIfNotEmpty(beanParams, "indexParams.loadArchive", commandOptions.indexParamsLoadArchive, true); + putNestedIfNotEmpty(beanParams, "indexParams.loadHomRef", commandOptions.indexParamsLoadHomRef, true); + putNestedIfNotEmpty(beanParams, "indexParams.postLoadCheck", commandOptions.indexParamsPostLoadCheck, true); + putNestedIfNotEmpty(beanParams, "indexParams.includeGenotypes", commandOptions.indexParamsIncludeGenotypes, true); + putNestedIfNotEmpty(beanParams, "indexParams.includeSampleData", commandOptions.indexParamsIncludeSampleData, true); + putNestedIfNotEmpty(beanParams, "indexParams.merge", commandOptions.indexParamsMerge, true); + putNestedIfNotEmpty(beanParams, "indexParams.deduplicationPolicy", commandOptions.indexParamsDeduplicationPolicy, true); + putNestedIfNotNull(beanParams, "indexParams.calculateStats", commandOptions.indexParamsCalculateStats, true); + putNestedIfNotNull(beanParams, "indexParams.aggregated", commandOptions.indexParamsAggregated, true); + putNestedIfNotEmpty(beanParams, "indexParams.aggregationMappingFile", commandOptions.indexParamsAggregationMappingFile, true); + putNestedIfNotNull(beanParams, "indexParams.annotate", commandOptions.indexParamsAnnotate, true); + putNestedIfNotEmpty(beanParams, "indexParams.annotator", commandOptions.indexParamsAnnotator, true); + putNestedIfNotNull(beanParams, "indexParams.overwriteAnnotations", commandOptions.indexParamsOverwriteAnnotations, true); + putNestedIfNotNull(beanParams, "indexParams.indexSearch", commandOptions.indexParamsIndexSearch, true); + putNestedIfNotNull(beanParams, "indexParams.skipIndexedFiles", commandOptions.indexParamsSkipIndexedFiles, true); variantFileIndexJobLauncherParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -624,6 +657,9 @@ private RestResponse runVariantJulie() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("project", commandOptions.project); @@ -638,9 +674,9 @@ private RestResponse runVariantJulie() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), JulieParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "cohorts",commandOptions.cohorts, true); - putNestedIfNotEmpty(beanParams, "region",commandOptions.region, true); - putNestedIfNotNull(beanParams, "overwrite",commandOptions.overwrite, true); + putNestedIfNotNull(beanParams, "cohorts", commandOptions.cohorts, true); + putNestedIfNotEmpty(beanParams, "region", commandOptions.region, true); + putNestedIfNotNull(beanParams, "overwrite", commandOptions.overwrite, true); julieParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -659,6 +695,9 @@ private RestResponse repairVariantMetadata() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); VariantStorageMetadataRepairToolParams variantStorageMetadataRepairToolParams = null; @@ -672,8 +711,8 @@ private RestResponse repairVariantMetadata() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantStorageMetadataRepairToolParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "studies",commandOptions.studies, true); - putNestedIfNotNull(beanParams, "samplesBatchSize",commandOptions.samplesBatchSize, true); + putNestedIfNotNull(beanParams, "studies", commandOptions.studies, true); + putNestedIfNotNull(beanParams, "samplesBatchSize", commandOptions.samplesBatchSize, true); variantStorageMetadataRepairToolParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -692,6 +731,9 @@ private RestResponse synchronizeVariantMetadata() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("study", commandOptions.study); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); @@ -709,8 +751,8 @@ private RestResponse synchronizeVariantMetadata() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantStorageMetadataSynchronizeParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "study",commandOptions.bodyStudy, true); - putNestedIfNotNull(beanParams, "files",commandOptions.files, true); + putNestedIfNotEmpty(beanParams, "study", commandOptions.bodyStudy, true); + putNestedIfNotNull(beanParams, "files", commandOptions.files, true); variantStorageMetadataSynchronizeParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -729,6 +771,9 @@ private RestResponse pruneVariant() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); VariantPruneParams variantPruneParams = null; @@ -742,9 +787,9 @@ private RestResponse pruneVariant() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantPruneParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "project",commandOptions.project, true); - putNestedIfNotNull(beanParams, "dryRun",commandOptions.dryRun, true); - putNestedIfNotNull(beanParams, "resume",commandOptions.resume, true); + putNestedIfNotEmpty(beanParams, "project", commandOptions.project, true); + putNestedIfNotNull(beanParams, "dryRun", commandOptions.dryRun, true); + putNestedIfNotNull(beanParams, "resume", commandOptions.resume, true); variantPruneParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -763,6 +808,9 @@ private RestResponse deleteVariantSample() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("study", commandOptions.study); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); @@ -780,9 +828,9 @@ private RestResponse deleteVariantSample() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantSampleDeleteParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotNull(beanParams, "force",commandOptions.force, true); - putNestedIfNotNull(beanParams, "resume",commandOptions.resume, true); + putNestedIfNotNull(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotNull(beanParams, "force", commandOptions.force, true); + putNestedIfNotNull(beanParams, "resume", commandOptions.resume, true); variantSampleDeleteParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -801,6 +849,9 @@ private RestResponse indexVariantSample() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("study", commandOptions.study); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); @@ -818,11 +869,11 @@ private RestResponse indexVariantSample() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantSecondarySampleIndexParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotNull(beanParams, "buildIndex",commandOptions.buildIndex, true); - putNestedIfNotNull(beanParams, "annotate",commandOptions.annotate, true); - putNestedIfNotNull(beanParams, "familyIndex",commandOptions.familyIndex, true); - putNestedIfNotNull(beanParams, "overwrite",commandOptions.overwrite, true); + putNestedIfNotNull(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotNull(beanParams, "buildIndex", commandOptions.buildIndex, true); + putNestedIfNotNull(beanParams, "annotate", commandOptions.annotate, true); + putNestedIfNotNull(beanParams, "familyIndex", commandOptions.familyIndex, true); + putNestedIfNotNull(beanParams, "overwrite", commandOptions.overwrite, true); variantSecondarySampleIndexParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -867,6 +918,9 @@ private RestResponse deleteVariantScore() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("study", commandOptions.study); queryParams.putIfNotEmpty("name", commandOptions.name); queryParams.putIfNotNull("resume", commandOptions.resume); @@ -888,6 +942,9 @@ private RestResponse indexVariantScore() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("study", commandOptions.study); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); @@ -905,12 +962,12 @@ private RestResponse indexVariantScore() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantScoreIndexParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "scoreName",commandOptions.scoreName, true); - putNestedIfNotEmpty(beanParams, "cohort1",commandOptions.cohort1, true); - putNestedIfNotEmpty(beanParams, "cohort2",commandOptions.cohort2, true); - putNestedIfNotEmpty(beanParams, "input",commandOptions.input, true); - putNestedIfNotEmpty(beanParams, "inputColumns",commandOptions.inputColumns, true); - putNestedIfNotNull(beanParams, "resume",commandOptions.resume, true); + putNestedIfNotEmpty(beanParams, "scoreName", commandOptions.scoreName, true); + putNestedIfNotEmpty(beanParams, "cohort1", commandOptions.cohort1, true); + putNestedIfNotEmpty(beanParams, "cohort2", commandOptions.cohort2, true); + putNestedIfNotEmpty(beanParams, "input", commandOptions.input, true); + putNestedIfNotEmpty(beanParams, "inputColumns", commandOptions.inputColumns, true); + putNestedIfNotNull(beanParams, "resume", commandOptions.resume, true); variantScoreIndexParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -929,6 +986,9 @@ private RestResponse variantSecondaryAnnotationIndex() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("project", commandOptions.project); queryParams.putIfNotEmpty("study", commandOptions.study); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { @@ -947,9 +1007,9 @@ private RestResponse variantSecondaryAnnotationIndex() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantSecondaryAnnotationIndexParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "region",commandOptions.region, true); - putNestedIfNotNull(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotNull(beanParams, "overwrite",commandOptions.overwrite, true); + putNestedIfNotEmpty(beanParams, "region", commandOptions.region, true); + putNestedIfNotNull(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotNull(beanParams, "overwrite", commandOptions.overwrite, true); variantSecondaryAnnotationIndexParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -968,6 +1028,9 @@ private RestResponse variantSecondarySampleIndex() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("study", commandOptions.study); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); @@ -985,11 +1048,11 @@ private RestResponse variantSecondarySampleIndex() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantSecondarySampleIndexParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotNull(beanParams, "buildIndex",commandOptions.buildIndex, true); - putNestedIfNotNull(beanParams, "annotate",commandOptions.annotate, true); - putNestedIfNotNull(beanParams, "familyIndex",commandOptions.familyIndex, true); - putNestedIfNotNull(beanParams, "overwrite",commandOptions.overwrite, true); + putNestedIfNotNull(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotNull(beanParams, "buildIndex", commandOptions.buildIndex, true); + putNestedIfNotNull(beanParams, "annotate", commandOptions.annotate, true); + putNestedIfNotNull(beanParams, "familyIndex", commandOptions.familyIndex, true); + putNestedIfNotNull(beanParams, "overwrite", commandOptions.overwrite, true); variantSecondarySampleIndexParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1034,6 +1097,9 @@ private RestResponse secondaryIndexVariant() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("project", commandOptions.project); queryParams.putIfNotEmpty("study", commandOptions.study); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { @@ -1052,9 +1118,9 @@ private RestResponse secondaryIndexVariant() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantSecondaryAnnotationIndexParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "region",commandOptions.region, true); - putNestedIfNotNull(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotNull(beanParams, "overwrite",commandOptions.overwrite, true); + putNestedIfNotEmpty(beanParams, "region", commandOptions.region, true); + putNestedIfNotNull(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotNull(beanParams, "overwrite", commandOptions.overwrite, true); variantSecondaryAnnotationIndexParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1073,6 +1139,9 @@ private RestResponse deleteVariantSecondaryIndex() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("study", commandOptions.study); queryParams.putIfNotEmpty("samples", commandOptions.samples); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { @@ -1082,6 +1151,45 @@ private RestResponse deleteVariantSecondaryIndex() throws Exception { return openCGAClient.getVariantOperationClient().deleteVariantSecondaryIndex(queryParams); } + private RestResponse setupVariant() throws Exception { + logger.debug("Executing setupVariant in Operations - Variant Storage command line"); + + OperationsVariantStorageCommandOptions.SetupVariantCommandOptions commandOptions = operationsVariantStorageCommandOptions.setupVariantCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("study", commandOptions.study); + if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { + queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); + } + + + VariantSetupParams variantSetupParams = null; + if (commandOptions.jsonDataModel) { + RestResponse res = new RestResponse<>(); + res.setType(QueryType.VOID); + PrintUtils.println(getObjectAsJSON(categoryName,"/{apiVersion}/operation/variant/setup")); + return res; + } else if (commandOptions.jsonFile != null) { + variantSetupParams = JacksonUtils.getDefaultObjectMapper() + .readValue(new java.io.File(commandOptions.jsonFile), VariantSetupParams.class); + } else { + ObjectMap beanParams = new ObjectMap(); + putNestedIfNotNull(beanParams, "expectedSamples", commandOptions.expectedSamples, true); + putNestedIfNotNull(beanParams, "expectedFiles", commandOptions.expectedFiles, true); + putNestedIfNotNull(beanParams, "fileType", commandOptions.fileType, true); + putNestedIfNotEmpty(beanParams, "averageFileSize", commandOptions.averageFileSize, true); + putNestedIfNotNull(beanParams, "variantsPerSample", commandOptions.variantsPerSample, true); + putNestedIfNotNull(beanParams, "averageSamplesPerFile", commandOptions.averageSamplesPerFile, true); + putNestedIfNotNull(beanParams, "dataDistribution", commandOptions.dataDistribution, true); + putNestedIfNotNull(beanParams, "normalizeExtensions", commandOptions.normalizeExtensions, true); + + variantSetupParams = JacksonUtils.getDefaultObjectMapper().copy() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) + .readValue(beanParams.toJson(), VariantSetupParams.class); + } + return openCGAClient.getVariantOperationClient().setupVariant(variantSetupParams, queryParams); + } + private RestResponse deleteVariantStats() throws Exception { logger.debug("Executing deleteVariantStats in Operations - Variant Storage command line"); @@ -1093,6 +1201,9 @@ private RestResponse deleteVariantStats() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -1109,8 +1220,8 @@ private RestResponse deleteVariantStats() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantStatsDeleteParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "cohort",commandOptions.cohort, true); - putNestedIfNotNull(beanParams, "force",commandOptions.force, true); + putNestedIfNotNull(beanParams, "cohort", commandOptions.cohort, true); + putNestedIfNotNull(beanParams, "force", commandOptions.force, true); variantStatsDeleteParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1130,6 +1241,9 @@ private RestResponse indexVariantStats() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -1146,12 +1260,12 @@ private RestResponse indexVariantStats() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantStatsIndexParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "cohort",commandOptions.cohort, true); - putNestedIfNotEmpty(beanParams, "region",commandOptions.region, true); - putNestedIfNotNull(beanParams, "overwriteStats",commandOptions.overwriteStats, true); - putNestedIfNotNull(beanParams, "resume",commandOptions.resume, true); - putNestedIfNotNull(beanParams, "aggregated",commandOptions.aggregated, true); - putNestedIfNotEmpty(beanParams, "aggregationMappingFile",commandOptions.aggregationMappingFile, true); + putNestedIfNotNull(beanParams, "cohort", commandOptions.cohort, true); + putNestedIfNotEmpty(beanParams, "region", commandOptions.region, true); + putNestedIfNotNull(beanParams, "overwriteStats", commandOptions.overwriteStats, true); + putNestedIfNotNull(beanParams, "resume", commandOptions.resume, true); + putNestedIfNotNull(beanParams, "aggregated", commandOptions.aggregated, true); + putNestedIfNotEmpty(beanParams, "aggregationMappingFile", commandOptions.aggregationMappingFile, true); variantStatsIndexParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -1170,6 +1284,9 @@ private RestResponse deleteVariantStudy() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("study", commandOptions.study); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); @@ -1187,7 +1304,7 @@ private RestResponse deleteVariantStudy() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariantStudyDeleteParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "resume",commandOptions.resume, true); + putNestedIfNotNull(beanParams, "resume", commandOptions.resume, true); variantStudyDeleteParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OrganizationsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OrganizationsCommandExecutor.java new file mode 100644 index 00000000000..1294b1397e7 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OrganizationsCommandExecutor.java @@ -0,0 +1,397 @@ +package org.opencb.opencga.app.cli.main.executors; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.lang.Object; +import java.util.HashMap; +import java.util.List; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.utils.PrintUtils; +import org.opencb.opencga.app.cli.main.*; +import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; +import org.opencb.opencga.app.cli.main.options.OrganizationsCommandOptions; +import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; +import org.opencb.opencga.catalog.utils.ParamUtils.AddRemoveAction; +import org.opencb.opencga.catalog.utils.ParamUtils.BasicUpdateAction; +import org.opencb.opencga.catalog.utils.ParamUtils.UpdateAction; +import org.opencb.opencga.client.exceptions.ClientException; +import org.opencb.opencga.core.common.JacksonUtils; +import org.opencb.opencga.core.config.Optimizations; +import org.opencb.opencga.core.models.notes.Note; +import org.opencb.opencga.core.models.notes.NoteCreateParams; +import org.opencb.opencga.core.models.notes.NoteUpdateParams; +import org.opencb.opencga.core.models.organizations.Organization; +import org.opencb.opencga.core.models.organizations.OrganizationConfiguration; +import org.opencb.opencga.core.models.organizations.OrganizationCreateParams; +import org.opencb.opencga.core.models.organizations.OrganizationUpdateParams; +import org.opencb.opencga.core.models.organizations.TokenConfiguration; +import org.opencb.opencga.core.models.user.OrganizationUserUpdateParams; +import org.opencb.opencga.core.models.user.User; +import org.opencb.opencga.core.models.user.UserQuota; +import org.opencb.opencga.core.models.user.UserStatusUpdateParams; +import org.opencb.opencga.core.response.QueryType; +import org.opencb.opencga.core.response.RestResponse; + + +/* +* WARNING: AUTOGENERATED CODE +* +* This code was generated by a tool. +* +* Manual changes to this file may cause unexpected behavior in your application. +* Manual changes to this file will be overwritten if the code is regenerated. +* +*/ +/** + * This class contains methods for the Organizations command line. + * PATH: /{apiVersion}/organizations + */ +public class OrganizationsCommandExecutor extends OpencgaCommandExecutor { + + public String categoryName = "organizations"; + public OrganizationsCommandOptions organizationsCommandOptions; + + public OrganizationsCommandExecutor(OrganizationsCommandOptions organizationsCommandOptions) throws CatalogAuthenticationException { + super(organizationsCommandOptions.commonCommandOptions); + this.organizationsCommandOptions = organizationsCommandOptions; + } + + @Override + public void execute() throws Exception { + + logger.debug("Executing Organizations command line"); + + String subCommandString = getParsedSubCommand(organizationsCommandOptions.jCommander); + + RestResponse queryResponse = null; + + switch (subCommandString) { + case "create": + queryResponse = create(); + break; + case "notes-create": + queryResponse = createNotes(); + break; + case "notes-search": + queryResponse = searchNotes(); + break; + case "notes-delete": + queryResponse = deleteNotes(); + break; + case "notes-update": + queryResponse = updateNotes(); + break; + case "update-status-user": + queryResponse = userUpdateStatus(); + break; + case "user-update": + queryResponse = updateUser(); + break; + case "configuration-update": + queryResponse = updateConfiguration(); + break; + case "info": + queryResponse = info(); + break; + case "update": + queryResponse = update(); + break; + default: + logger.error("Subcommand not valid"); + break; + } + + createOutput(queryResponse); + + } + + private RestResponse create() throws Exception { + logger.debug("Executing create in Organizations command line"); + + OrganizationsCommandOptions.CreateCommandOptions commandOptions = organizationsCommandOptions.createCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("include", commandOptions.include); + queryParams.putIfNotEmpty("exclude", commandOptions.exclude); + queryParams.putIfNotNull("includeResult", commandOptions.includeResult); + + + OrganizationCreateParams organizationCreateParams = null; + if (commandOptions.jsonDataModel) { + RestResponse res = new RestResponse<>(); + res.setType(QueryType.VOID); + PrintUtils.println(getObjectAsJSON(categoryName,"/{apiVersion}/organizations/create")); + return res; + } else if (commandOptions.jsonFile != null) { + organizationCreateParams = JacksonUtils.getDefaultObjectMapper() + .readValue(new java.io.File(commandOptions.jsonFile), OrganizationCreateParams.class); + } else { + ObjectMap beanParams = new ObjectMap(); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "configuration.defaultUserExpirationDate", commandOptions.configurationDefaultUserExpirationDate, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); + + organizationCreateParams = JacksonUtils.getDefaultObjectMapper().copy() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) + .readValue(beanParams.toJson(), OrganizationCreateParams.class); + } + return openCGAClient.getOrganizationClient().create(organizationCreateParams, queryParams); + } + + private RestResponse createNotes() throws Exception { + logger.debug("Executing createNotes in Organizations command line"); + + OrganizationsCommandOptions.CreateNotesCommandOptions commandOptions = organizationsCommandOptions.createNotesCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("include", commandOptions.include); + queryParams.putIfNotEmpty("exclude", commandOptions.exclude); + queryParams.putIfNotNull("includeResult", commandOptions.includeResult); + + + NoteCreateParams noteCreateParams = null; + if (commandOptions.jsonDataModel) { + RestResponse res = new RestResponse<>(); + res.setType(QueryType.VOID); + PrintUtils.println(getObjectAsJSON(categoryName,"/{apiVersion}/organizations/notes/create")); + return res; + } else if (commandOptions.jsonFile != null) { + noteCreateParams = JacksonUtils.getDefaultObjectMapper() + .readValue(new java.io.File(commandOptions.jsonFile), NoteCreateParams.class); + } else { + ObjectMap beanParams = new ObjectMap(); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); + putNestedIfNotNull(beanParams, "visibility", commandOptions.visibility, true); + putNestedIfNotNull(beanParams, "valueType", commandOptions.valueType, true); + + noteCreateParams = JacksonUtils.getDefaultObjectMapper().copy() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) + .readValue(beanParams.toJson(), NoteCreateParams.class); + } + return openCGAClient.getOrganizationClient().createNotes(noteCreateParams, queryParams); + } + + private RestResponse searchNotes() throws Exception { + logger.debug("Executing searchNotes in Organizations command line"); + + OrganizationsCommandOptions.SearchNotesCommandOptions commandOptions = organizationsCommandOptions.searchNotesCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("include", commandOptions.include); + queryParams.putIfNotEmpty("exclude", commandOptions.exclude); + queryParams.putIfNotEmpty("creationDate", commandOptions.creationDate); + queryParams.putIfNotEmpty("modificationDate", commandOptions.modificationDate); + queryParams.putIfNotEmpty("id", commandOptions.id); + queryParams.putIfNotEmpty("scope", commandOptions.scope); + queryParams.putIfNotEmpty("visibility", commandOptions.visibility); + queryParams.putIfNotEmpty("uuid", commandOptions.uuid); + queryParams.putIfNotEmpty("userId", commandOptions.userId); + queryParams.putIfNotEmpty("tags", commandOptions.tags); + queryParams.putIfNotEmpty("version", commandOptions.version); + + return openCGAClient.getOrganizationClient().searchNotes(queryParams); + } + + private RestResponse deleteNotes() throws Exception { + logger.debug("Executing deleteNotes in Organizations command line"); + + OrganizationsCommandOptions.DeleteNotesCommandOptions commandOptions = organizationsCommandOptions.deleteNotesCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotNull("includeResult", commandOptions.includeResult); + + return openCGAClient.getOrganizationClient().deleteNotes(commandOptions.id, queryParams); + } + + private RestResponse updateNotes() throws Exception { + logger.debug("Executing updateNotes in Organizations command line"); + + OrganizationsCommandOptions.UpdateNotesCommandOptions commandOptions = organizationsCommandOptions.updateNotesCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("include", commandOptions.include); + queryParams.putIfNotEmpty("exclude", commandOptions.exclude); + queryParams.putIfNotNull("tagsAction", commandOptions.tagsAction); + queryParams.putIfNotNull("includeResult", commandOptions.includeResult); + + + NoteUpdateParams noteUpdateParams = null; + if (commandOptions.jsonDataModel) { + RestResponse res = new RestResponse<>(); + res.setType(QueryType.VOID); + PrintUtils.println(getObjectAsJSON(categoryName,"/{apiVersion}/organizations/notes/{id}/update")); + return res; + } else if (commandOptions.jsonFile != null) { + noteUpdateParams = JacksonUtils.getDefaultObjectMapper() + .readValue(new java.io.File(commandOptions.jsonFile), NoteUpdateParams.class); + } else { + ObjectMap beanParams = new ObjectMap(); + putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); + putNestedIfNotNull(beanParams, "visibility", commandOptions.visibility, true); + + noteUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) + .readValue(beanParams.toJson(), NoteUpdateParams.class); + } + return openCGAClient.getOrganizationClient().updateNotes(commandOptions.id, noteUpdateParams, queryParams); + } + + private RestResponse userUpdateStatus() throws Exception { + logger.debug("Executing userUpdateStatus in Organizations command line"); + + OrganizationsCommandOptions.UserUpdateStatusCommandOptions commandOptions = organizationsCommandOptions.userUpdateStatusCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("include", commandOptions.include); + queryParams.putIfNotEmpty("exclude", commandOptions.exclude); + queryParams.putIfNotEmpty("organization", commandOptions.organization); + queryParams.putIfNotNull("includeResult", commandOptions.includeResult); + + + UserStatusUpdateParams userStatusUpdateParams = null; + if (commandOptions.jsonDataModel) { + RestResponse res = new RestResponse<>(); + res.setType(QueryType.VOID); + PrintUtils.println(getObjectAsJSON(categoryName,"/{apiVersion}/organizations/user/{user}/status/update")); + return res; + } else if (commandOptions.jsonFile != null) { + userStatusUpdateParams = JacksonUtils.getDefaultObjectMapper() + .readValue(new java.io.File(commandOptions.jsonFile), UserStatusUpdateParams.class); + } else { + ObjectMap beanParams = new ObjectMap(); + putNestedIfNotEmpty(beanParams, "status", commandOptions.status, true); + + userStatusUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) + .readValue(beanParams.toJson(), UserStatusUpdateParams.class); + } + return openCGAClient.getOrganizationClient().userUpdateStatus(commandOptions.user, userStatusUpdateParams, queryParams); + } + + private RestResponse updateUser() throws Exception { + logger.debug("Executing updateUser in Organizations command line"); + + OrganizationsCommandOptions.UpdateUserCommandOptions commandOptions = organizationsCommandOptions.updateUserCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("include", commandOptions.include); + queryParams.putIfNotEmpty("exclude", commandOptions.exclude); + queryParams.putIfNotEmpty("organization", commandOptions.organization); + queryParams.putIfNotNull("includeResult", commandOptions.includeResult); + + + OrganizationUserUpdateParams organizationUserUpdateParams = null; + if (commandOptions.jsonDataModel) { + RestResponse res = new RestResponse<>(); + res.setType(QueryType.VOID); + PrintUtils.println(getObjectAsJSON(categoryName,"/{apiVersion}/organizations/user/{user}/update")); + return res; + } else if (commandOptions.jsonFile != null) { + organizationUserUpdateParams = JacksonUtils.getDefaultObjectMapper() + .readValue(new java.io.File(commandOptions.jsonFile), OrganizationUserUpdateParams.class); + } else { + ObjectMap beanParams = new ObjectMap(); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "email", commandOptions.email, true); + putNestedIfNotNull(beanParams, "quota.diskUsage", commandOptions.quotaDiskUsage, true); + putNestedIfNotNull(beanParams, "quota.cpuUsage", commandOptions.quotaCpuUsage, true); + putNestedIfNotNull(beanParams, "quota.maxDisk", commandOptions.quotaMaxDisk, true); + putNestedIfNotNull(beanParams, "quota.maxCpu", commandOptions.quotaMaxCpu, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); + + organizationUserUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) + .readValue(beanParams.toJson(), OrganizationUserUpdateParams.class); + } + return openCGAClient.getOrganizationClient().updateUser(commandOptions.user, organizationUserUpdateParams, queryParams); + } + + private RestResponse updateConfiguration() throws Exception { + logger.debug("Executing updateConfiguration in Organizations command line"); + + OrganizationsCommandOptions.UpdateConfigurationCommandOptions commandOptions = organizationsCommandOptions.updateConfigurationCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("include", commandOptions.include); + queryParams.putIfNotEmpty("exclude", commandOptions.exclude); + queryParams.putIfNotNull("includeResult", commandOptions.includeResult); + queryParams.putIfNotNull("authenticationOriginsAction", commandOptions.authenticationOriginsAction); + + + OrganizationConfiguration organizationConfiguration = null; + if (commandOptions.jsonDataModel) { + RestResponse res = new RestResponse<>(); + res.setType(QueryType.VOID); + PrintUtils.println(getObjectAsJSON(categoryName,"/{apiVersion}/organizations/{organization}/configuration/update")); + return res; + } else if (commandOptions.jsonFile != null) { + organizationConfiguration = JacksonUtils.getDefaultObjectMapper() + .readValue(new java.io.File(commandOptions.jsonFile), OrganizationConfiguration.class); + } else { + ObjectMap beanParams = new ObjectMap(); + putNestedIfNotEmpty(beanParams, "defaultUserExpirationDate", commandOptions.defaultUserExpirationDate, true); + putNestedIfNotNull(beanParams, "optimizations.simplifyPermissions", commandOptions.optimizationsSimplifyPermissions, true); + putNestedIfNotEmpty(beanParams, "token.algorithm", commandOptions.tokenAlgorithm, true); + putNestedIfNotEmpty(beanParams, "token.secretKey", commandOptions.tokenSecretKey, true); + putNestedIfNotNull(beanParams, "token.expiration", commandOptions.tokenExpiration, true); + + organizationConfiguration = JacksonUtils.getDefaultObjectMapper().copy() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) + .readValue(beanParams.toJson(), OrganizationConfiguration.class); + } + return openCGAClient.getOrganizationClient().updateConfiguration(commandOptions.organization, organizationConfiguration, queryParams); + } + + private RestResponse info() throws Exception { + logger.debug("Executing info in Organizations command line"); + + OrganizationsCommandOptions.InfoCommandOptions commandOptions = organizationsCommandOptions.infoCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("include", commandOptions.include); + queryParams.putIfNotEmpty("exclude", commandOptions.exclude); + + return openCGAClient.getOrganizationClient().info(commandOptions.organization, queryParams); + } + + private RestResponse update() throws Exception { + logger.debug("Executing update in Organizations command line"); + + OrganizationsCommandOptions.UpdateCommandOptions commandOptions = organizationsCommandOptions.updateCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("include", commandOptions.include); + queryParams.putIfNotEmpty("exclude", commandOptions.exclude); + queryParams.putIfNotNull("includeResult", commandOptions.includeResult); + queryParams.putIfNotNull("adminsAction", commandOptions.adminsAction); + + + OrganizationUpdateParams organizationUpdateParams = null; + if (commandOptions.jsonDataModel) { + RestResponse res = new RestResponse<>(); + res.setType(QueryType.VOID); + PrintUtils.println(getObjectAsJSON(categoryName,"/{apiVersion}/organizations/{organization}/update")); + return res; + } else if (commandOptions.jsonFile != null) { + organizationUpdateParams = JacksonUtils.getDefaultObjectMapper() + .readValue(new java.io.File(commandOptions.jsonFile), OrganizationUpdateParams.class); + } else { + ObjectMap beanParams = new ObjectMap(); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "owner", commandOptions.owner, true); + putNestedIfNotNull(beanParams, "admins", commandOptions.admins, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); + + organizationUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) + .readValue(beanParams.toJson(), OrganizationUpdateParams.class); + } + return openCGAClient.getOrganizationClient().update(commandOptions.organization, organizationUpdateParams, queryParams); + } +} \ No newline at end of file diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/ProjectsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/ProjectsCommandExecutor.java index b7c06afd6f6..929d89e1e4e 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/ProjectsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/ProjectsCommandExecutor.java @@ -4,7 +4,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.util.HashMap; import java.util.List; -import org.opencb.commons.datastore.core.FacetField; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.utils.PrintUtils; import org.opencb.opencga.app.cli.main.*; @@ -62,9 +61,6 @@ public void execute() throws Exception { case "search": queryResponse = search(); break; - case "aggregationstats": - queryResponse = aggregationStats(); - break; case "info": queryResponse = info(); break; @@ -108,19 +104,19 @@ private RestResponse create() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), ProjectCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "organism.scientificName",commandOptions.organismScientificName, true); - putNestedIfNotEmpty(beanParams, "organism.commonName",commandOptions.organismCommonName, true); - putNestedIfNotEmpty(beanParams, "organism.assembly",commandOptions.organismAssembly, true); - putNestedIfNotEmpty(beanParams, "cellbase.url",commandOptions.cellbaseUrl, true); - putNestedIfNotEmpty(beanParams, "cellbase.version",commandOptions.cellbaseVersion, true); - putNestedIfNotEmpty(beanParams, "cellbase.dataRelease",commandOptions.cellbaseDataRelease, true); - putNestedIfNotEmpty(beanParams, "cellbase.apiKey",commandOptions.cellbaseApiKey, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "organism.scientificName", commandOptions.organismScientificName, true); + putNestedIfNotEmpty(beanParams, "organism.commonName", commandOptions.organismCommonName, true); + putNestedIfNotEmpty(beanParams, "organism.assembly", commandOptions.organismAssembly, true); + putNestedIfNotEmpty(beanParams, "cellbase.url", commandOptions.cellbaseUrl, true); + putNestedIfNotEmpty(beanParams, "cellbase.version", commandOptions.cellbaseVersion, true); + putNestedIfNotEmpty(beanParams, "cellbase.dataRelease", commandOptions.cellbaseDataRelease, true); + putNestedIfNotEmpty(beanParams, "cellbase.apiKey", commandOptions.cellbaseApiKey, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); projectCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -139,7 +135,7 @@ private RestResponse search() throws Exception { queryParams.putIfNotEmpty("exclude", commandOptions.exclude); queryParams.putIfNotNull("limit", commandOptions.limit); queryParams.putIfNotNull("skip", commandOptions.skip); - queryParams.putIfNotEmpty("owner", commandOptions.owner); + queryParams.putIfNotEmpty("organization", commandOptions.organization); queryParams.putIfNotEmpty("id", commandOptions.id); queryParams.putIfNotEmpty("name", commandOptions.name); queryParams.putIfNotEmpty("fqn", commandOptions.fqn); @@ -157,23 +153,6 @@ private RestResponse search() throws Exception { return openCGAClient.getProjectClient().search(queryParams); } - private RestResponse aggregationStats() throws Exception { - logger.debug("Executing aggregationStats in Projects command line"); - - ProjectsCommandOptions.AggregationStatsCommandOptions commandOptions = projectsCommandOptions.aggregationStatsCommandOptions; - - ObjectMap queryParams = new ObjectMap(); - queryParams.putIfNotNull("default_values", commandOptions.default_values); - queryParams.putIfNotEmpty("fileFields", commandOptions.fileFields); - queryParams.putIfNotEmpty("individualFields", commandOptions.individualFields); - queryParams.putIfNotEmpty("familyFields", commandOptions.familyFields); - queryParams.putIfNotEmpty("sampleFields", commandOptions.sampleFields); - queryParams.putIfNotEmpty("cohortFields", commandOptions.cohortFields); - queryParams.putIfNotEmpty("jobFields", commandOptions.jobFields); - - return openCGAClient.getProjectClient().aggregationStats(commandOptions.projects, queryParams); - } - private RestResponse info() throws Exception { logger.debug("Executing info in Projects command line"); @@ -229,14 +208,14 @@ private RestResponse update() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), ProjectUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "organism.scientificName",commandOptions.organismScientificName, true); - putNestedIfNotEmpty(beanParams, "organism.commonName",commandOptions.organismCommonName, true); - putNestedIfNotEmpty(beanParams, "organism.assembly",commandOptions.organismAssembly, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "organism.scientificName", commandOptions.organismScientificName, true); + putNestedIfNotEmpty(beanParams, "organism.commonName", commandOptions.organismCommonName, true); + putNestedIfNotEmpty(beanParams, "organism.assembly", commandOptions.organismAssembly, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); projectUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/SamplesCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/SamplesCommandExecutor.java index 328cc9eb556..c8e02456a4a 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/SamplesCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/SamplesCommandExecutor.java @@ -6,7 +6,6 @@ import java.util.List; import java.util.Map; import org.opencb.biodata.models.core.OntologyTermAnnotation; -import org.opencb.commons.datastore.core.FacetField; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.utils.PrintUtils; import org.opencb.opencga.app.cli.main.*; @@ -72,9 +71,6 @@ public void execute() throws Exception { case "acl-update": queryResponse = updateAcl(); break; - case "aggregationstats": - queryResponse = aggregationStats(); - break; case "annotation-sets-load": queryResponse = loadAnnotationSets(); break; @@ -137,12 +133,12 @@ private RestResponse updateAcl() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), SampleAclUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "permissions",commandOptions.permissions, true); - putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); - putNestedIfNotEmpty(beanParams, "individual",commandOptions.individual, true); - putNestedIfNotEmpty(beanParams, "family",commandOptions.family, true); - putNestedIfNotEmpty(beanParams, "file",commandOptions.file, true); - putNestedIfNotEmpty(beanParams, "cohort",commandOptions.cohort, true); + putNestedIfNotEmpty(beanParams, "permissions", commandOptions.permissions, true); + putNestedIfNotEmpty(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotEmpty(beanParams, "individual", commandOptions.individual, true); + putNestedIfNotEmpty(beanParams, "family", commandOptions.family, true); + putNestedIfNotEmpty(beanParams, "file", commandOptions.file, true); + putNestedIfNotEmpty(beanParams, "cohort", commandOptions.cohort, true); sampleAclUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -151,34 +147,6 @@ private RestResponse updateAcl() throws Exception { return openCGAClient.getSampleClient().updateAcl(commandOptions.members, commandOptions.action, sampleAclUpdateParams, queryParams); } - private RestResponse aggregationStats() throws Exception { - logger.debug("Executing aggregationStats in Samples command line"); - - SamplesCommandOptions.AggregationStatsCommandOptions commandOptions = samplesCommandOptions.aggregationStatsCommandOptions; - - ObjectMap queryParams = new ObjectMap(); - queryParams.putIfNotEmpty("study", commandOptions.study); - queryParams.putIfNotEmpty("source", commandOptions.source); - queryParams.putIfNotEmpty("creationYear", commandOptions.creationYear); - queryParams.putIfNotEmpty("creationMonth", commandOptions.creationMonth); - queryParams.putIfNotEmpty("creationDay", commandOptions.creationDay); - queryParams.putIfNotEmpty("creationDayOfWeek", commandOptions.creationDayOfWeek); - queryParams.putIfNotEmpty("status", commandOptions.status); - queryParams.putIfNotEmpty("type", commandOptions.type); - queryParams.putIfNotEmpty("phenotypes", commandOptions.phenotypes); - queryParams.putIfNotEmpty("release", commandOptions.release); - queryParams.putIfNotEmpty("version", commandOptions.version); - queryParams.putIfNotNull("somatic", commandOptions.somatic); - queryParams.putIfNotEmpty("annotation", commandOptions.annotation); - queryParams.putIfNotNull("default_values", commandOptions.default_values); - queryParams.putIfNotEmpty("field", commandOptions.field); - if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { - queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); - } - - return openCGAClient.getSampleClient().aggregationStats(queryParams); - } - private RestResponse loadAnnotationSets() throws Exception { logger.debug("Executing loadAnnotationSets in Samples command line"); @@ -204,7 +172,7 @@ private RestResponse loadAnnotationSets() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), TsvAnnotationParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "content",commandOptions.content, true); + putNestedIfNotEmpty(beanParams, "content", commandOptions.content, true); tsvAnnotationParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -239,32 +207,32 @@ private RestResponse create() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), SampleCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "individualId",commandOptions.individualId, true); - putNestedIfNotEmpty(beanParams, "source.id",commandOptions.sourceId, true); - putNestedIfNotEmpty(beanParams, "source.name",commandOptions.sourceName, true); - putNestedIfNotEmpty(beanParams, "source.description",commandOptions.sourceDescription, true); - putNestedIfNotEmpty(beanParams, "source.source",commandOptions.sourceSource, true); - putNestedIfNotEmpty(beanParams, "source.url",commandOptions.sourceUrl, true); - putNestedIfNotEmpty(beanParams, "processing.preparationMethod",commandOptions.processingPreparationMethod, true); - putNestedIfNotEmpty(beanParams, "processing.extractionMethod",commandOptions.processingExtractionMethod, true); - putNestedIfNotEmpty(beanParams, "processing.labSampleId",commandOptions.processingLabSampleId, true); - putNestedIfNotEmpty(beanParams, "processing.quantity",commandOptions.processingQuantity, true); - putNestedIfNotEmpty(beanParams, "processing.date",commandOptions.processingDate, true); - putNestedIfNotNull(beanParams, "processing.attributes",commandOptions.processingAttributes, true); - putNestedIfNotEmpty(beanParams, "collection.type",commandOptions.collectionType, true); - putNestedIfNotEmpty(beanParams, "collection.quantity",commandOptions.collectionQuantity, true); - putNestedIfNotEmpty(beanParams, "collection.method",commandOptions.collectionMethod, true); - putNestedIfNotEmpty(beanParams, "collection.date",commandOptions.collectionDate, true); - putNestedIfNotNull(beanParams, "collection.attributes",commandOptions.collectionAttributes, true); - putNestedIfNotNull(beanParams, "somatic",commandOptions.somatic, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "individualId", commandOptions.individualId, true); + putNestedIfNotEmpty(beanParams, "source.id", commandOptions.sourceId, true); + putNestedIfNotEmpty(beanParams, "source.name", commandOptions.sourceName, true); + putNestedIfNotEmpty(beanParams, "source.description", commandOptions.sourceDescription, true); + putNestedIfNotEmpty(beanParams, "source.source", commandOptions.sourceSource, true); + putNestedIfNotEmpty(beanParams, "source.url", commandOptions.sourceUrl, true); + putNestedIfNotEmpty(beanParams, "processing.preparationMethod", commandOptions.processingPreparationMethod, true); + putNestedIfNotEmpty(beanParams, "processing.extractionMethod", commandOptions.processingExtractionMethod, true); + putNestedIfNotEmpty(beanParams, "processing.labSampleId", commandOptions.processingLabSampleId, true); + putNestedIfNotEmpty(beanParams, "processing.quantity", commandOptions.processingQuantity, true); + putNestedIfNotEmpty(beanParams, "processing.date", commandOptions.processingDate, true); + putNestedMapIfNotEmpty(beanParams, "processing.attributes", commandOptions.processingAttributes, true); + putNestedIfNotEmpty(beanParams, "collection.type", commandOptions.collectionType, true); + putNestedIfNotEmpty(beanParams, "collection.quantity", commandOptions.collectionQuantity, true); + putNestedIfNotEmpty(beanParams, "collection.method", commandOptions.collectionMethod, true); + putNestedIfNotEmpty(beanParams, "collection.date", commandOptions.collectionDate, true); + putNestedMapIfNotEmpty(beanParams, "collection.attributes", commandOptions.collectionAttributes, true); + putNestedIfNotNull(beanParams, "somatic", commandOptions.somatic, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); sampleCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -476,33 +444,33 @@ private RestResponse update() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), SampleUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "individualId",commandOptions.individualId, true); - putNestedIfNotEmpty(beanParams, "source.id",commandOptions.sourceId, true); - putNestedIfNotEmpty(beanParams, "source.name",commandOptions.sourceName, true); - putNestedIfNotEmpty(beanParams, "source.description",commandOptions.sourceDescription, true); - putNestedIfNotEmpty(beanParams, "source.source",commandOptions.sourceSource, true); - putNestedIfNotEmpty(beanParams, "source.url",commandOptions.sourceUrl, true); - putNestedIfNotEmpty(beanParams, "processing.preparationMethod",commandOptions.processingPreparationMethod, true); - putNestedIfNotEmpty(beanParams, "processing.extractionMethod",commandOptions.processingExtractionMethod, true); - putNestedIfNotEmpty(beanParams, "processing.labSampleId",commandOptions.processingLabSampleId, true); - putNestedIfNotEmpty(beanParams, "processing.quantity",commandOptions.processingQuantity, true); - putNestedIfNotEmpty(beanParams, "processing.date",commandOptions.processingDate, true); - putNestedIfNotNull(beanParams, "processing.attributes",commandOptions.processingAttributes, true); - putNestedIfNotEmpty(beanParams, "collection.type",commandOptions.collectionType, true); - putNestedIfNotEmpty(beanParams, "collection.quantity",commandOptions.collectionQuantity, true); - putNestedIfNotEmpty(beanParams, "collection.method",commandOptions.collectionMethod, true); - putNestedIfNotEmpty(beanParams, "collection.date",commandOptions.collectionDate, true); - putNestedIfNotNull(beanParams, "collection.attributes",commandOptions.collectionAttributes, true); - putNestedIfNotNull(beanParams, "qualityControl.files",commandOptions.qualityControlFiles, true); - putNestedIfNotNull(beanParams, "somatic",commandOptions.somatic, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "individualId", commandOptions.individualId, true); + putNestedIfNotEmpty(beanParams, "source.id", commandOptions.sourceId, true); + putNestedIfNotEmpty(beanParams, "source.name", commandOptions.sourceName, true); + putNestedIfNotEmpty(beanParams, "source.description", commandOptions.sourceDescription, true); + putNestedIfNotEmpty(beanParams, "source.source", commandOptions.sourceSource, true); + putNestedIfNotEmpty(beanParams, "source.url", commandOptions.sourceUrl, true); + putNestedIfNotEmpty(beanParams, "processing.preparationMethod", commandOptions.processingPreparationMethod, true); + putNestedIfNotEmpty(beanParams, "processing.extractionMethod", commandOptions.processingExtractionMethod, true); + putNestedIfNotEmpty(beanParams, "processing.labSampleId", commandOptions.processingLabSampleId, true); + putNestedIfNotEmpty(beanParams, "processing.quantity", commandOptions.processingQuantity, true); + putNestedIfNotEmpty(beanParams, "processing.date", commandOptions.processingDate, true); + putNestedMapIfNotEmpty(beanParams, "processing.attributes", commandOptions.processingAttributes, true); + putNestedIfNotEmpty(beanParams, "collection.type", commandOptions.collectionType, true); + putNestedIfNotEmpty(beanParams, "collection.quantity", commandOptions.collectionQuantity, true); + putNestedIfNotEmpty(beanParams, "collection.method", commandOptions.collectionMethod, true); + putNestedIfNotEmpty(beanParams, "collection.date", commandOptions.collectionDate, true); + putNestedMapIfNotEmpty(beanParams, "collection.attributes", commandOptions.collectionAttributes, true); + putNestedIfNotNull(beanParams, "qualityControl.files", commandOptions.qualityControlFiles, true); + putNestedIfNotNull(beanParams, "somatic", commandOptions.somatic, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); sampleUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/StudiesCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/StudiesCommandExecutor.java index 5289aedc3ef..0c18639ed52 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/StudiesCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/StudiesCommandExecutor.java @@ -8,7 +8,6 @@ import java.util.HashMap; import java.util.List; import java.util.Set; -import org.opencb.commons.datastore.core.FacetField; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.utils.PrintUtils; @@ -32,6 +31,9 @@ import org.opencb.opencga.core.models.common.Enums.Resource; import org.opencb.opencga.core.models.common.StatusParams; import org.opencb.opencga.core.models.job.Job; +import org.opencb.opencga.core.models.notes.Note; +import org.opencb.opencga.core.models.notes.NoteCreateParams; +import org.opencb.opencga.core.models.notes.NoteUpdateParams; import org.opencb.opencga.core.models.study.CustomGroup; import org.opencb.opencga.core.models.study.Group; import org.opencb.opencga.core.models.study.GroupCreateParams; @@ -97,9 +99,6 @@ public void execute() throws Exception { case "acl": queryResponse = acl(); break; - case "aggregationstats": - queryResponse = aggregationStats(); - break; case "info": queryResponse = info(); break; @@ -115,6 +114,18 @@ public void execute() throws Exception { case "groups-users-update": queryResponse = updateGroupsUsers(); break; + case "notes-create": + queryResponse = createNotes(); + break; + case "notes-search": + queryResponse = searchNotes(); + break; + case "notes-delete": + queryResponse = deleteNotes(); + break; + case "notes-update": + queryResponse = updateNotes(); + break; case "permissionrules": queryResponse = permissionRules(); break; @@ -167,9 +178,9 @@ private RestResponse updateAcl() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), StudyAclUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "study",commandOptions.study, true); - putNestedIfNotEmpty(beanParams, "template",commandOptions.template, true); - putNestedIfNotEmpty(beanParams, "permissions",commandOptions.permissions, true); + putNestedIfNotEmpty(beanParams, "study", commandOptions.study, true); + putNestedIfNotEmpty(beanParams, "template", commandOptions.template, true); + putNestedIfNotEmpty(beanParams, "permissions", commandOptions.permissions, true); studyAclUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -201,18 +212,18 @@ private RestResponse create() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), StudyCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "alias",commandOptions.alias, true); - putNestedIfNotEmpty(beanParams, "type.id",commandOptions.typeId, true); - putNestedIfNotEmpty(beanParams, "type.description",commandOptions.typeDescription, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "alias", commandOptions.alias, true); + putNestedIfNotEmpty(beanParams, "type.id", commandOptions.typeId, true); + putNestedIfNotEmpty(beanParams, "type.description", commandOptions.typeDescription, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); studyCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -258,23 +269,6 @@ private RestResponse acl() throws Exception { return openCGAClient.getStudyClient().acl(commandOptions.studies, queryParams); } - private RestResponse aggregationStats() throws Exception { - logger.debug("Executing aggregationStats in Studies command line"); - - StudiesCommandOptions.AggregationStatsCommandOptions commandOptions = studiesCommandOptions.aggregationStatsCommandOptions; - - ObjectMap queryParams = new ObjectMap(); - queryParams.putIfNotNull("default_values", commandOptions.default_values); - queryParams.putIfNotEmpty("fileFields", commandOptions.fileFields); - queryParams.putIfNotEmpty("individualFields", commandOptions.individualFields); - queryParams.putIfNotEmpty("familyFields", commandOptions.familyFields); - queryParams.putIfNotEmpty("sampleFields", commandOptions.sampleFields); - queryParams.putIfNotEmpty("cohortFields", commandOptions.cohortFields); - queryParams.putIfNotEmpty("jobFields", commandOptions.jobFields); - - return openCGAClient.getStudyClient().aggregationStats(commandOptions.studies, queryParams); - } - private RestResponse info() throws Exception { logger.debug("Executing info in Studies command line"); @@ -342,8 +336,8 @@ private RestResponse updateGroups() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), GroupCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotNull(beanParams, "users",commandOptions.users, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotNull(beanParams, "users", commandOptions.users, true); groupCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -372,7 +366,7 @@ private RestResponse updateGroupsUsers() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), GroupUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "users",commandOptions.users, true); + putNestedIfNotNull(beanParams, "users", commandOptions.users, true); groupUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -381,6 +375,104 @@ private RestResponse updateGroupsUsers() throws Exception { return openCGAClient.getStudyClient().updateGroupsUsers(commandOptions.study, commandOptions.group, groupUpdateParams, queryParams); } + private RestResponse createNotes() throws Exception { + logger.debug("Executing createNotes in Studies command line"); + + StudiesCommandOptions.CreateNotesCommandOptions commandOptions = studiesCommandOptions.createNotesCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("include", commandOptions.include); + queryParams.putIfNotEmpty("exclude", commandOptions.exclude); + queryParams.putIfNotNull("includeResult", commandOptions.includeResult); + + + NoteCreateParams noteCreateParams = null; + if (commandOptions.jsonDataModel) { + RestResponse res = new RestResponse<>(); + res.setType(QueryType.VOID); + PrintUtils.println(getObjectAsJSON(categoryName,"/{apiVersion}/studies/{study}/notes/create")); + return res; + } else if (commandOptions.jsonFile != null) { + noteCreateParams = JacksonUtils.getDefaultObjectMapper() + .readValue(new java.io.File(commandOptions.jsonFile), NoteCreateParams.class); + } else { + ObjectMap beanParams = new ObjectMap(); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); + putNestedIfNotNull(beanParams, "visibility", commandOptions.visibility, true); + putNestedIfNotNull(beanParams, "valueType", commandOptions.valueType, true); + + noteCreateParams = JacksonUtils.getDefaultObjectMapper().copy() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) + .readValue(beanParams.toJson(), NoteCreateParams.class); + } + return openCGAClient.getStudyClient().createNotes(commandOptions.study, noteCreateParams, queryParams); + } + + private RestResponse searchNotes() throws Exception { + logger.debug("Executing searchNotes in Studies command line"); + + StudiesCommandOptions.SearchNotesCommandOptions commandOptions = studiesCommandOptions.searchNotesCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("include", commandOptions.include); + queryParams.putIfNotEmpty("exclude", commandOptions.exclude); + queryParams.putIfNotEmpty("creationDate", commandOptions.creationDate); + queryParams.putIfNotEmpty("modificationDate", commandOptions.modificationDate); + queryParams.putIfNotEmpty("id", commandOptions.id); + queryParams.putIfNotEmpty("uuid", commandOptions.uuid); + queryParams.putIfNotEmpty("userId", commandOptions.userId); + queryParams.putIfNotEmpty("tags", commandOptions.tags); + queryParams.putIfNotEmpty("visibility", commandOptions.visibility); + queryParams.putIfNotEmpty("version", commandOptions.version); + + return openCGAClient.getStudyClient().searchNotes(commandOptions.study, queryParams); + } + + private RestResponse deleteNotes() throws Exception { + logger.debug("Executing deleteNotes in Studies command line"); + + StudiesCommandOptions.DeleteNotesCommandOptions commandOptions = studiesCommandOptions.deleteNotesCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotNull("includeResult", commandOptions.includeResult); + + return openCGAClient.getStudyClient().deleteNotes(commandOptions.study, commandOptions.id, queryParams); + } + + private RestResponse updateNotes() throws Exception { + logger.debug("Executing updateNotes in Studies command line"); + + StudiesCommandOptions.UpdateNotesCommandOptions commandOptions = studiesCommandOptions.updateNotesCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("include", commandOptions.include); + queryParams.putIfNotEmpty("exclude", commandOptions.exclude); + queryParams.putIfNotNull("tagsAction", commandOptions.tagsAction); + queryParams.putIfNotNull("includeResult", commandOptions.includeResult); + + + NoteUpdateParams noteUpdateParams = null; + if (commandOptions.jsonDataModel) { + RestResponse res = new RestResponse<>(); + res.setType(QueryType.VOID); + PrintUtils.println(getObjectAsJSON(categoryName,"/{apiVersion}/studies/{study}/notes/{id}/update")); + return res; + } else if (commandOptions.jsonFile != null) { + noteUpdateParams = JacksonUtils.getDefaultObjectMapper() + .readValue(new java.io.File(commandOptions.jsonFile), NoteUpdateParams.class); + } else { + ObjectMap beanParams = new ObjectMap(); + putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); + putNestedIfNotNull(beanParams, "visibility", commandOptions.visibility, true); + + noteUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) + .readValue(beanParams.toJson(), NoteUpdateParams.class); + } + return openCGAClient.getStudyClient().updateNotes(commandOptions.study, commandOptions.id, noteUpdateParams, queryParams); + } + private RestResponse permissionRules() throws Exception { logger.debug("Executing permissionRules in Studies command line"); @@ -408,10 +500,10 @@ private RestResponse updatePermissionRules() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), PermissionRule.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotNull(beanParams, "query",commandOptions.query, true); - putNestedIfNotNull(beanParams, "members",commandOptions.members, true); - putNestedIfNotNull(beanParams, "permissions",commandOptions.permissions, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedMapIfNotEmpty(beanParams, "query", commandOptions.query, true); + putNestedIfNotNull(beanParams, "members", commandOptions.members, true); + putNestedIfNotNull(beanParams, "permissions", commandOptions.permissions, true); permissionRule = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -430,6 +522,9 @@ private RestResponse runTemplates() throws Exception { queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("id", commandOptions.id); queryParams.putIfNotNull("overwrite", commandOptions.overwrite); queryParams.putIfNotNull("resume", commandOptions.resume); @@ -458,14 +553,7 @@ private RestResponse deleteTemplates() throws Exception { logger.debug("Executing deleteTemplates in Studies command line"); StudiesCommandOptions.DeleteTemplatesCommandOptions commandOptions = studiesCommandOptions.deleteTemplatesCommandOptions; - - ObjectMap queryParams = new ObjectMap(); - queryParams.putIfNotEmpty("study", commandOptions.study); - if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { - queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); - } - - return openCGAClient.getStudyClient().deleteTemplates(commandOptions.study, commandOptions.templateId, queryParams); + return openCGAClient.getStudyClient().deleteTemplates(commandOptions.study, commandOptions.templateId); } private RestResponse update() throws Exception { @@ -490,17 +578,17 @@ private RestResponse update() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), StudyUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "alias",commandOptions.alias, true); - putNestedIfNotEmpty(beanParams, "type.id",commandOptions.typeId, true); - putNestedIfNotEmpty(beanParams, "type.description",commandOptions.typeDescription, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); - putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); - putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); - putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); - putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "alias", commandOptions.alias, true); + putNestedIfNotEmpty(beanParams, "type.id", commandOptions.typeId, true); + putNestedIfNotEmpty(beanParams, "type.description", commandOptions.typeDescription, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "status.id", commandOptions.statusId, true); + putNestedIfNotEmpty(beanParams, "status.name", commandOptions.statusName, true); + putNestedIfNotEmpty(beanParams, "status.description", commandOptions.statusDescription, true); studyUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -540,11 +628,11 @@ private RestResponse updateVariableSets() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), VariableSetCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotNull(beanParams, "unique",commandOptions.unique, true); - putNestedIfNotNull(beanParams, "confidential",commandOptions.confidential, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotNull(beanParams, "unique", commandOptions.unique, true); + putNestedIfNotNull(beanParams, "confidential", commandOptions.confidential, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); variableSetCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -573,18 +661,18 @@ private RestResponse updateVariableSetsVariables() throws Exception .readValue(new java.io.File(commandOptions.jsonFile), Variable.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "category",commandOptions.category, true); - putNestedIfNotNull(beanParams, "type",commandOptions.type, true); - putNestedIfNotNull(beanParams, "required",commandOptions.required, true); - putNestedIfNotNull(beanParams, "multiValue",commandOptions.multiValue, true); - putNestedIfNotNull(beanParams, "allowedValues",commandOptions.allowedValues, true); - putNestedIfNotNull(beanParams, "allowedKeys",commandOptions.allowedKeys, true); - putNestedIfNotNull(beanParams, "rank",commandOptions.rank, true); - putNestedIfNotEmpty(beanParams, "dependsOn",commandOptions.dependsOn, true); - putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "category", commandOptions.category, true); + putNestedIfNotNull(beanParams, "type", commandOptions.type, true); + putNestedIfNotNull(beanParams, "required", commandOptions.required, true); + putNestedIfNotNull(beanParams, "multiValue", commandOptions.multiValue, true); + putNestedIfNotNull(beanParams, "allowedValues", commandOptions.allowedValues, true); + putNestedIfNotNull(beanParams, "allowedKeys", commandOptions.allowedKeys, true); + putNestedIfNotNull(beanParams, "rank", commandOptions.rank, true); + putNestedIfNotEmpty(beanParams, "dependsOn", commandOptions.dependsOn, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); variable = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/UsersCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/UsersCommandExecutor.java index e25fb1e1f1c..5f8dfce148c 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/UsersCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/UsersCommandExecutor.java @@ -19,13 +19,13 @@ import org.opencb.opencga.client.exceptions.ClientException; import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.models.common.Enums; -import org.opencb.opencga.core.models.project.Project; import org.opencb.opencga.core.models.user.AuthenticationResponse; import org.opencb.opencga.core.models.user.ConfigUpdateParams; import org.opencb.opencga.core.models.user.FilterUpdateParams; import org.opencb.opencga.core.models.user.LoginParams; import org.opencb.opencga.core.models.user.PasswordChangeParams; import org.opencb.opencga.core.models.user.User; +import org.opencb.opencga.core.models.user.UserCreateParams; import org.opencb.opencga.core.models.user.UserFilter; import org.opencb.opencga.core.models.user.UserUpdateParams; import org.opencb.opencga.core.response.QueryType; @@ -65,12 +65,21 @@ public void execute() throws Exception { RestResponse queryResponse = null; switch (subCommandString) { + case "anonymous": + queryResponse = anonymous(); + break; + case "create": + queryResponse = create(); + break; case "login": queryResponse = login(); break; case "password": queryResponse = password(); break; + case "search": + queryResponse = search(); + break; case "info": queryResponse = info(); break; @@ -86,9 +95,6 @@ public void execute() throws Exception { case "password-reset": queryResponse = resetPassword(); break; - case "projects": - queryResponse = projects(); - break; case "update": queryResponse = update(); break; @@ -107,11 +113,48 @@ public void execute() throws Exception { } + private RestResponse anonymous() throws Exception { + logger.debug("Executing anonymous in Users command line"); + + UsersCommandOptions.AnonymousCommandOptions commandOptions = usersCommandOptions.anonymousCommandOptions; + return openCGAClient.getUserClient().anonymous(commandOptions.organization); + } + + private RestResponse create() throws Exception { + logger.debug("Executing create in Users command line"); + + UsersCommandOptions.CreateCommandOptions commandOptions = usersCommandOptions.createCommandOptions; + + UserCreateParams userCreateParams = null; + if (commandOptions.jsonDataModel) { + RestResponse res = new RestResponse<>(); + res.setType(QueryType.VOID); + PrintUtils.println(getObjectAsJSON(categoryName,"/{apiVersion}/users/create")); + return res; + } else if (commandOptions.jsonFile != null) { + userCreateParams = JacksonUtils.getDefaultObjectMapper() + .readValue(new java.io.File(commandOptions.jsonFile), UserCreateParams.class); + } else { + ObjectMap beanParams = new ObjectMap(); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "email", commandOptions.email, true); + putNestedIfNotEmpty(beanParams, "password", commandOptions.password, true); + putNestedIfNotEmpty(beanParams, "organization", commandOptions.organization, true); + + userCreateParams = JacksonUtils.getDefaultObjectMapper().copy() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) + .readValue(beanParams.toJson(), UserCreateParams.class); + } + return openCGAClient.getUserClient().create(userCreateParams); + } + private RestResponse login() throws Exception { logger.debug("Executing login in Users command line"); CustomUsersCommandOptions.LoginCommandOptions commandOptions = usersCommandOptions.loginCommandOptions; ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("organization", commandOptions.organization); queryParams.putIfNotEmpty("user", commandOptions.user); queryParams.putIfNotEmpty("password", commandOptions.password); queryParams.putIfNotEmpty("refreshToken", commandOptions.refreshToken); @@ -135,10 +178,11 @@ private RestResponse password() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), PasswordChangeParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "user",commandOptions.user, true); - putNestedIfNotEmpty(beanParams, "password",commandOptions.password, true); - putNestedIfNotEmpty(beanParams, "newPassword",commandOptions.newPassword, true); - putNestedIfNotEmpty(beanParams, "reset",commandOptions.reset, true); + putNestedIfNotEmpty(beanParams, "organizationId", commandOptions.organizationId, true); + putNestedIfNotEmpty(beanParams, "user", commandOptions.user, true); + putNestedIfNotEmpty(beanParams, "password", commandOptions.password, true); + putNestedIfNotEmpty(beanParams, "newPassword", commandOptions.newPassword, true); + putNestedIfNotEmpty(beanParams, "reset", commandOptions.reset, true); passwordChangeParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -147,6 +191,24 @@ private RestResponse password() throws Exception { return openCGAClient.getUserClient().password(passwordChangeParams); } + private RestResponse search() throws Exception { + logger.debug("Executing search in Users command line"); + + UsersCommandOptions.SearchCommandOptions commandOptions = usersCommandOptions.searchCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("include", commandOptions.include); + queryParams.putIfNotEmpty("exclude", commandOptions.exclude); + queryParams.putIfNotNull("limit", commandOptions.limit); + queryParams.putIfNotNull("skip", commandOptions.skip); + queryParams.putIfNotNull("count", commandOptions.count); + queryParams.putIfNotEmpty("organization", commandOptions.organization); + queryParams.putIfNotEmpty("id", commandOptions.id); + queryParams.putIfNotEmpty("authenticationId", commandOptions.authenticationId); + + return openCGAClient.getUserClient().search(queryParams); + } + private RestResponse info() throws Exception { logger.debug("Executing info in Users command line"); @@ -155,6 +217,7 @@ private RestResponse info() throws Exception { ObjectMap queryParams = new ObjectMap(); queryParams.putIfNotEmpty("include", commandOptions.include); queryParams.putIfNotEmpty("exclude", commandOptions.exclude); + queryParams.putIfNotEmpty("organization", commandOptions.organization); return openCGAClient.getUserClient().info(commandOptions.users, queryParams); } @@ -190,8 +253,8 @@ private RestResponse updateConfigs() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), ConfigUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); - putNestedIfNotNull(beanParams, "configuration",commandOptions.configuration, true); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedMapIfNotEmpty(beanParams, "configuration", commandOptions.configuration, true); configUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -218,20 +281,6 @@ private RestResponse resetPassword() throws Exception { return openCGAClient.getUserClient().resetPassword(commandOptions.user); } - private RestResponse projects() throws Exception { - logger.debug("Executing projects in Users command line"); - - UsersCommandOptions.ProjectsCommandOptions commandOptions = usersCommandOptions.projectsCommandOptions; - - ObjectMap queryParams = new ObjectMap(); - queryParams.putIfNotEmpty("include", commandOptions.include); - queryParams.putIfNotEmpty("exclude", commandOptions.exclude); - queryParams.putIfNotNull("limit", commandOptions.limit); - queryParams.putIfNotNull("skip", commandOptions.skip); - - return openCGAClient.getUserClient().projects(commandOptions.user, queryParams); - } - private RestResponse update() throws Exception { logger.debug("Executing update in Users command line"); @@ -254,10 +303,8 @@ private RestResponse update() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), UserUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); - putNestedIfNotEmpty(beanParams, "email",commandOptions.email, true); - putNestedIfNotEmpty(beanParams, "organization",commandOptions.organization, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "email", commandOptions.email, true); userUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/io/TextOutputWriter.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/io/TextOutputWriter.java index a4e75a72794..500ca529aa7 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/io/TextOutputWriter.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/io/TextOutputWriter.java @@ -313,16 +313,15 @@ private void printUser(List> queryResultList) { for (DataResult queryResult : queryResultList) { // Write header if (writerConfiguration.isHeader()) { - sb.append("#(U)ID\tNAME\tE-MAIL\tORGANIZATION\tACCOUNT_TYPE\tSIZE\tQUOTA\n"); + sb.append("#(U)ID\tNAME\tE-MAIL\tORGANIZATION\tQUOTA\n"); sb.append("#(P)\tID\tNAME\tDESCRIPTION\n"); sb.append("#(S)\t\tID\tNAME\tDESCRIPTION\t#GROUPS\tSIZE\n"); } for (User user : queryResult.getResults()) { - sb.append(String.format("%s%s\t%s\t%s\t%s\t%s\t%d\n", "", + sb.append(String.format("%s%s\t%s\t%s\t%s\t%d\n", "", StringUtils.defaultIfEmpty(user.getId(), "-"), StringUtils.defaultIfEmpty(user.getName(), "-"), StringUtils.defaultIfEmpty(user.getEmail(), "-"), StringUtils.defaultIfEmpty(user.getOrganization(), "-"), - StringUtils.defaultIfEmpty(user.getAccount() != null ? user.getAccount().getType().name() : "-", "-"), user.getQuota().getMaxDisk())); if (user.getProjects().size() > 0) { @@ -376,11 +375,6 @@ private void printStudy(List> queryResultList) { .addColumn("DESCRIPTION", Study::getDescription) .addColumnNumber("#GROUPS", s -> s.getGroups().size()) .addColumnNumber("SIZE", Study::getSize) - .addColumnNumber("#FILES", s -> s.getFiles().size()) - .addColumnNumber("#SAMPLES", s -> s.getSamples().size()) - .addColumnNumber("#COHORTS", s -> s.getCohorts().size()) - .addColumnNumber("#INDIVIDUALS", s -> s.getIndividuals().size()) - .addColumnNumber("#JOBS", s -> s.getJobs().size()) .addColumnNumber("#VARIABLE_SETS", s -> s.getVariableSets().size()) .addColumn("STATUS", s -> s.getInternal().getStatus().getId()); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AdminCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AdminCommandOptions.java index f246d8a36d8..60684639ca0 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AdminCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AdminCommandOptions.java @@ -34,11 +34,11 @@ public class AdminCommandOptions { public CommonCommandOptions commonCommandOptions; public GroupByAuditCommandOptions groupByAuditCommandOptions; - public IndexStatsCatalogCommandOptions indexStatsCatalogCommandOptions; public InstallCatalogCommandOptions installCatalogCommandOptions; public JwtCatalogCommandOptions jwtCatalogCommandOptions; public CreateUsersCommandOptions createUsersCommandOptions; public ImportUsersCommandOptions importUsersCommandOptions; + public PermissionsUsersCommandOptions permissionsUsersCommandOptions; public SearchUsersCommandOptions searchUsersCommandOptions; public SyncUsersCommandOptions syncUsersCommandOptions; public UsersUpdateGroupsCommandOptions usersUpdateGroupsCommandOptions; @@ -49,11 +49,11 @@ public AdminCommandOptions(CommonCommandOptions commonCommandOptions, JCommander this.jCommander = jCommander; this.commonCommandOptions = commonCommandOptions; this.groupByAuditCommandOptions = new GroupByAuditCommandOptions(); - this.indexStatsCatalogCommandOptions = new IndexStatsCatalogCommandOptions(); this.installCatalogCommandOptions = new InstallCatalogCommandOptions(); this.jwtCatalogCommandOptions = new JwtCatalogCommandOptions(); this.createUsersCommandOptions = new CreateUsersCommandOptions(); this.importUsersCommandOptions = new ImportUsersCommandOptions(); + this.permissionsUsersCommandOptions = new PermissionsUsersCommandOptions(); this.searchUsersCommandOptions = new SearchUsersCommandOptions(); this.syncUsersCommandOptions = new SyncUsersCommandOptions(); this.usersUpdateGroupsCommandOptions = new UsersUpdateGroupsCommandOptions(); @@ -92,23 +92,6 @@ public class GroupByAuditCommandOptions { } - @Parameters(commandNames = {"catalog-index-stats"}, commandDescription ="Sync Catalog into the Solr") - public class IndexStatsCatalogCommandOptions { - - @ParametersDelegate - public CommonCommandOptions commonOptions = commonCommandOptions; - - @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) - public String jsonFile; - - @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) - public Boolean jsonDataModel = false; - - @Parameter(names = {"--collection"}, description = "Collection to be indexed (file, sample, individual, family, cohort and/or job). If not provided, all of them will be indexed.", required = false, arity = 1) - public String collection; - - } - @Parameters(commandNames = {"catalog-install"}, commandDescription ="Install OpenCGA database") public class InstallCatalogCommandOptions { @@ -130,9 +113,6 @@ public class InstallCatalogCommandOptions { @Parameter(names = {"--email"}, description = "The body web service email parameter", required = false, arity = 1) public String email; - @Parameter(names = {"--organization"}, description = "The body web service organization parameter", required = false, arity = 1) - public String organization; - } @Parameters(commandNames = {"catalog-jwt"}, commandDescription ="Change JWT secret key") @@ -147,6 +127,9 @@ public class JwtCatalogCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; + @Parameter(names = {"--organization"}, description = "Organization id", required = false, arity = 1) + public String organization; + @Parameter(names = {"--secret-key"}, description = "The body web service secretKey parameter", required = false, arity = 1) public String secretKey; @@ -179,9 +162,6 @@ public class CreateUsersCommandOptions { @Parameter(names = {"--organization"}, description = "The body web service organization parameter", required = false, arity = 1) public String organization; - @Parameter(names = {"--type"}, description = "Enum param allowed values: GUEST, FULL, ADMINISTRATOR", required = false, arity = 1) - public String type; - } @Parameters(commandNames = {"users-import"}, commandDescription ="Import users or a group of users from LDAP or AAD") @@ -196,6 +176,9 @@ public class ImportUsersCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; + @Parameter(names = {"--organization"}, description = "Organization id", required = false, arity = 1) + public String organization; + @Parameter(names = {"--authentication-origin-id"}, description = "The body web service authenticationOriginId parameter", required = false, arity = 1) public String authenticationOriginId; @@ -213,6 +196,26 @@ public class ImportUsersCommandOptions { } + @Parameters(commandNames = {"users-permissions"}, commandDescription ="User permissions") + public class PermissionsUsersCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + public String study; + + @Parameter(names = {"--entry-ids"}, description = "Comma separated list of entry ids.", required = false, arity = 1) + public String entryIds; + + @Parameter(names = {"--permissions"}, description = "Comma separated list of permissions to be retrieved.", required = false, arity = 1) + public String permissions; + + @Parameter(names = {"--category"}, description = "Category corresponding to the id's provided.", required = false, arity = 1) + public String category; + + } + @Parameters(commandNames = {"users-search"}, commandDescription ="User search method") public class SearchUsersCommandOptions { @@ -234,12 +237,12 @@ public class SearchUsersCommandOptions { @Parameter(names = {"--count"}, description = "Get the total number of results matching the query. Deactivated by default.", required = false, help = true, arity = 0) public boolean count = false; + @Parameter(names = {"--organization"}, description = "Organization id", required = false, arity = 1) + public String organization; + @Parameter(names = {"--user", "-u"}, description = "User ID", required = false, arity = 1) public String user; - @Parameter(names = {"--account"}, description = "Account type [GUEST, FULL, ADMINISTRATOR]", required = false, arity = 1) - public String account; - @Parameter(names = {"--authentication-id"}, description = "Authentication origin ID", required = false, arity = 1) public String authenticationId; @@ -257,6 +260,9 @@ public class SyncUsersCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; + @Parameter(names = {"--organization"}, description = "Organization id", required = false, arity = 1) + public String organization; + @Parameter(names = {"--authentication-origin-id"}, description = "The body web service authenticationOriginId parameter", required = false, arity = 1) public String authenticationOriginId; @@ -272,9 +278,6 @@ public class SyncUsersCommandOptions { @Parameter(names = {"--sync-all"}, description = "The body web service syncAll parameter", required = false, help = true, arity = 0) public boolean syncAll = false; - @Parameter(names = {"--type"}, description = "Enum param allowed values: GUEST, FULL, ADMINISTRATOR", required = false, arity = 1) - public String type; - @Parameter(names = {"--force"}, description = "The body web service force parameter", required = false, help = true, arity = 0) public boolean force = false; @@ -292,6 +295,9 @@ public class UsersUpdateGroupsCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; + @Parameter(names = {"--organization"}, description = "Organization id", required = false, arity = 1) + public String organization; + @Parameter(names = {"--user", "-u"}, description = "User ID", required = true, arity = 1) public String user; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisAlignmentCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisAlignmentCommandOptions.java index 004e650e9c1..61cc3634fb1 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisAlignmentCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisAlignmentCommandOptions.java @@ -95,6 +95,15 @@ public class RunBwaCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--command"}, description = "The body web service command parameter", required = false, arity = 1) public String command; @@ -142,6 +151,15 @@ public class RunCoverageIndexCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--bam-file-id"}, description = "BAM file ID.", required = false, arity = 1) public String bamFileId; @@ -180,6 +198,15 @@ public class CoverageQcGeneCoverageStatsRunCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--bam-file"}, description = "The body web service bamFile parameter", required = false, arity = 1) public String bamFile; @@ -200,7 +227,7 @@ public class QueryCoverageCommandOptions { @Parameter(names = {"--file"}, description = "File ID", required = true, arity = 1) public String file; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--region"}, description = "Comma separated list of regions 'chr:start-end, e.g.: 2,3:63500-65000", required = false, arity = 1) @@ -238,7 +265,7 @@ public class RatioCoverageCommandOptions { @Parameter(names = {"--file2"}, description = "Input file #2 (e.g. germline file)", required = true, arity = 1) public String file2; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--skip-log2"}, description = "Do not apply Log2 to normalise the coverage ratio", required = false, help = true, arity = 0) @@ -276,7 +303,7 @@ public class StatsCoverageCommandOptions { @Parameter(names = {"--gene"}, description = "Comma separated list of genes, e.g.: BCRA2,TP53", required = true, arity = 1) public String gene; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--threshold"}, description = "Only regions whose coverage depth is under this threshold will be reported.", required = false, arity = 1) @@ -311,6 +338,15 @@ public class RunDeeptoolsCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--command"}, description = "The body web service command parameter", required = false, arity = 1) public String command; @@ -349,6 +385,15 @@ public class RunFastqcCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--input-file"}, description = "The body web service inputFile parameter", required = false, arity = 1) public String inputFile; @@ -387,6 +432,15 @@ public class RunIndexCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--file-id"}, description = "File ID, (i.e., BAM/CRAM file ID).", required = false, arity = 1) public String fileId; @@ -422,6 +476,15 @@ public class RunPicardCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--command"}, description = "The body web service command parameter", required = false, arity = 1) public String command; @@ -460,6 +523,15 @@ public class RunQcCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--bam-file"}, description = "ID for the BAM file to process.", required = false, arity = 1) public String bamFile; @@ -492,7 +564,7 @@ public class QueryCommandOptions { @Parameter(names = {"--file"}, description = "File ID", required = true, arity = 1) public String file; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--region"}, description = "Comma separated list of regions 'chr:start-end, e.g.: 2,3:63500-65000", required = false, arity = 1) @@ -569,6 +641,15 @@ public class RunSamtoolsCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--command"}, description = "The body web service command parameter", required = false, arity = 1) public String command; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisClinicalCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisClinicalCommandOptions.java index 592dbfc3efc..b9dc5e1956a 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisClinicalCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisClinicalCommandOptions.java @@ -46,6 +46,7 @@ public class AnalysisClinicalCommandOptions { public RunInterpreterTeamCommandOptions runInterpreterTeamCommandOptions; public RunInterpreterTieringCommandOptions runInterpreterTieringCommandOptions; public RunInterpreterZettaCommandOptions runInterpreterZettaCommandOptions; + public LoadCommandOptions loadCommandOptions; public AggregationStatsRgaCommandOptions aggregationStatsRgaCommandOptions; public QueryRgaGeneCommandOptions queryRgaGeneCommandOptions; public SummaryRgaGeneCommandOptions summaryRgaGeneCommandOptions; @@ -86,6 +87,7 @@ public AnalysisClinicalCommandOptions(CommonCommandOptions commonCommandOptions, this.runInterpreterTeamCommandOptions = new RunInterpreterTeamCommandOptions(); this.runInterpreterTieringCommandOptions = new RunInterpreterTieringCommandOptions(); this.runInterpreterZettaCommandOptions = new RunInterpreterZettaCommandOptions(); + this.loadCommandOptions = new LoadCommandOptions(); this.aggregationStatsRgaCommandOptions = new AggregationStatsRgaCommandOptions(); this.queryRgaGeneCommandOptions = new QueryRgaGeneCommandOptions(); this.summaryRgaGeneCommandOptions = new SummaryRgaGeneCommandOptions(); @@ -122,7 +124,7 @@ public class UpdateAclCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--members"}, description = "Comma separated list of user or group IDs", required = true, arity = 1) @@ -154,7 +156,7 @@ public class LoadAnnotationSetsCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--variable-set-id"}, description = "Variable set ID or name", required = true, arity = 1) @@ -186,7 +188,7 @@ public class UpdateClinicalConfigurationCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @DynamicParameter(names = {"--interpretation-default-filter"}, description = "The body web service defaultFilter parameter. Use: --interpretation-default-filter key=value", required = false) @@ -212,7 +214,7 @@ public class CreateCommandOptions { @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) public String exclude; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--skip-create-default-interpretation"}, description = "Flag to skip creating and initialise an empty default primary interpretation (Id will be '{clinicalAnalysisId}.1'). This flag is only considered if no Interpretation object is passed.", required = false, help = true, arity = 0) @@ -239,8 +241,8 @@ public class CreateCommandOptions { @Parameter(names = {"--family-id"}, description = "The body web service id parameter", required = false, arity = 1) public String familyId; - @Parameter(names = {"--panel-lock"}, description = "The body web service panelLock parameter", required = false, arity = 1) - public Boolean panelLock; + @Parameter(names = {"--panel-locked"}, description = "The body web service panelLocked parameter", required = false, arity = 1) + public Boolean panelLocked; @Parameter(names = {"--analyst-id"}, description = "The body web service id parameter", required = false, arity = 1) public String analystId; @@ -299,6 +301,9 @@ public class CreateCommandOptions { @Parameter(names = {"--responsible-postcode"}, description = "The body web service postcode parameter", required = false, arity = 1) public String responsiblePostcode; + @Parameter(names = {"--interpretation-name"}, description = "The body web service name parameter", required = false, arity = 1) + public String interpretationName; + @Parameter(names = {"--interpretation-description"}, description = "The body web service description parameter", required = false, arity = 1) public String interpretationDescription; @@ -352,7 +357,7 @@ public class DistinctCommandOptions { @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--id"}, description = "Comma separated list of Clinical Analysis IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = false, arity = 1) @@ -421,6 +426,9 @@ public class DistinctCommandOptions { @Parameter(names = {"--release"}, description = "Release when it was created", required = false, arity = 1) public String release; + @Parameter(names = {"--snapshot"}, description = "Snapshot value (Latest version of the entry in the specified release)", required = false, arity = 1) + public Integer snapshot; + @Parameter(names = {"--status"}, description = "Filter by status", required = false, arity = 1) public String status; @@ -444,7 +452,7 @@ public class DistinctInterpretationCommandOptions { @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--id"}, description = "Comma separated list of Interpretation IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = false, arity = 1) @@ -453,6 +461,9 @@ public class DistinctInterpretationCommandOptions { @Parameter(names = {"--uuid"}, description = "Comma separated list of Interpretation UUIDs up to a maximum of 100", required = false, arity = 1) public String uuid; + @Parameter(names = {"--name", "-n"}, description = "Comma separated list of Interpretation names up to a maximum of 100", required = false, arity = 1) + public String name; + @Parameter(names = {"--clinical-analysis-id"}, description = "Clinical Analysis id", required = false, arity = 1) public String clinicalAnalysisId; @@ -512,7 +523,7 @@ public class SearchInterpretationCommandOptions { @Parameter(names = {"--sort"}, description = "Sort the results", required = false, help = true, arity = 0) public boolean sort = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--id"}, description = "Comma separated list of Interpretation IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = false, arity = 1) @@ -521,6 +532,9 @@ public class SearchInterpretationCommandOptions { @Parameter(names = {"--uuid"}, description = "Comma separated list of Interpretation UUIDs up to a maximum of 100", required = false, arity = 1) public String uuid; + @Parameter(names = {"--name", "-n"}, description = "Comma separated list of Interpretation names up to a maximum of 100", required = false, arity = 1) + public String name; + @Parameter(names = {"--clinical-analysis-id"}, description = "Clinical Analysis id", required = false, arity = 1) public String clinicalAnalysisId; @@ -571,7 +585,7 @@ public class InfoInterpretationCommandOptions { @Parameter(names = {"--interpretations"}, description = "Comma separated list of clinical interpretation IDs up to a maximum of 100", required = true, arity = 1) public String interpretations; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--version"}, description = "Comma separated list of interpretation versions. 'all' to get all the interpretation versions. Not supported if multiple interpretation ids are provided.", required = false, arity = 1) @@ -594,7 +608,7 @@ public class RunInterpreterCancerTieringCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) @@ -609,6 +623,15 @@ public class RunInterpreterCancerTieringCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--clinical-analysis"}, description = "The body web service clinicalAnalysis parameter", required = false, arity = 1) public String clinicalAnalysis; @@ -632,7 +655,7 @@ public class RunInterpreterExomiserCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) @@ -647,9 +670,21 @@ public class RunInterpreterExomiserCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; - @Parameter(names = {"--clinical-analysis"}, description = "The body web service clinicalAnalysis parameter", required = false, arity = 1) + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + + @Parameter(names = {"--clinical-analysis"}, description = "Clinical analysis ID.", required = false, arity = 1) public String clinicalAnalysis; + @Parameter(names = {"--exomiser-version"}, description = "Exomiser version in the format X.Y where X is the major version and Y the minor version, e.g.: 14.0. If the version is not specified, the default version will be used. Refer to the configuration file to view all installed Exomiser versions and identify the default version.", required = false, arity = 1) + public String exomiserVersion; + } @Parameters(commandNames = {"interpreter-team-run"}, commandDescription ="Run TEAM interpretation analysis") @@ -664,7 +699,7 @@ public class RunInterpreterTeamCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) @@ -679,6 +714,15 @@ public class RunInterpreterTeamCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--clinical-analysis"}, description = "The body web service clinicalAnalysis parameter", required = false, arity = 1) public String clinicalAnalysis; @@ -705,7 +749,7 @@ public class RunInterpreterTieringCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) @@ -720,6 +764,15 @@ public class RunInterpreterTieringCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--clinical-analysis"}, description = "The body web service clinicalAnalysis parameter", required = false, arity = 1) public String clinicalAnalysis; @@ -746,7 +799,7 @@ public class RunInterpreterZettaCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) @@ -761,6 +814,15 @@ public class RunInterpreterZettaCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--clinical-analysis"}, description = "The body web service clinicalAnalysis parameter", required = false, arity = 1) public String clinicalAnalysis; @@ -916,6 +978,47 @@ public class RunInterpreterZettaCommandOptions { } + @Parameters(commandNames = {"load"}, commandDescription ="Load clinical analyses from a file") + public class LoadCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) + public String jsonFile; + + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) + public Boolean jsonDataModel = false; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + public String study; + + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) + public String jobId; + + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) + public String jobDescription; + + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) + public String jobDependsOn; + + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) + public String jobTags; + + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + + @Parameter(names = {"--file"}, description = "The body web service file parameter", required = false, arity = 1) + public String file; + + } + @Parameters(commandNames = {"rga-aggregation-stats"}, commandDescription ="RGA aggregation stats") public class AggregationStatsRgaCommandOptions { @@ -988,7 +1091,7 @@ public class AggregationStatsRgaCommandOptions { @Parameter(names = {"--consequence-type"}, description = "Filter by consequence type.", required = false, arity = 1) public String consequenceType; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--field"}, description = "List of fields separated by semicolons, e.g.: clinicalSignificances;type. For nested fields use >>, e.g.: type>>clinicalSignificances;knockoutType", required = true, arity = 1) @@ -1086,7 +1189,7 @@ public class QueryRgaGeneCommandOptions { @Parameter(names = {"--consequence-type"}, description = "Filter by consequence type.", required = false, arity = 1) public String consequenceType; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; } @@ -1166,7 +1269,7 @@ public class SummaryRgaGeneCommandOptions { @Parameter(names = {"--consequence-type"}, description = "Filter by consequence type.", required = false, arity = 1) public String consequenceType; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; } @@ -1183,7 +1286,7 @@ public class RunRgaIndexCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) @@ -1198,6 +1301,15 @@ public class RunRgaIndexCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--auxiliar-index"}, description = "Index auxiliar collection to improve performance assuming RGA is completely indexed.", required = false, help = true, arity = 0) public boolean auxiliarIndex = false; @@ -1287,7 +1399,7 @@ public class QueryRgaIndividualCommandOptions { @Parameter(names = {"--consequence-type"}, description = "Filter by consequence type.", required = false, arity = 1) public String consequenceType; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; } @@ -1367,7 +1479,7 @@ public class SummaryRgaIndividualCommandOptions { @Parameter(names = {"--consequence-type"}, description = "Filter by consequence type.", required = false, arity = 1) public String consequenceType; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; } @@ -1462,7 +1574,7 @@ public class QueryRgaVariantCommandOptions { @Parameter(names = {"--consequence-type"}, description = "Filter by consequence type.", required = false, arity = 1) public String consequenceType; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; } @@ -1542,7 +1654,7 @@ public class SummaryRgaVariantCommandOptions { @Parameter(names = {"--consequence-type"}, description = "Filter by consequence type.", required = false, arity = 1) public String consequenceType; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; } @@ -1571,7 +1683,7 @@ public class SearchCommandOptions { @Parameter(names = {"--flatten-annotations"}, description = "Flatten the annotations?", required = false, help = true, arity = 0) public boolean flattenAnnotations = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--id"}, description = "Comma separated list of Clinical Analysis IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = false, arity = 1) @@ -1640,6 +1752,9 @@ public class SearchCommandOptions { @Parameter(names = {"--release"}, description = "Release when it was created", required = false, arity = 1) public String release; + @Parameter(names = {"--snapshot"}, description = "Snapshot value (Latest version of the entry in the specified release)", required = false, arity = 1) + public Integer snapshot; + @Parameter(names = {"--status"}, description = "Filter by status", required = false, arity = 1) public String status; @@ -1696,7 +1811,7 @@ public class QueryVariantCommandOptions { @Parameter(names = {"--type"}, description = "List of types, accepted values are SNV, MNV, INDEL, SV, COPY_NUMBER, COPY_NUMBER_LOSS, COPY_NUMBER_GAIN, INSERTION, DELETION, DUPLICATION, TANDEM_DUPLICATION, BREAKEND, e.g. SNV,INDEL", required = false, arity = 1) public String type; - @Parameter(names = {"--study", "-s"}, description = "Filter variants from the given studies, these can be either the numeric ID or the alias with the format user@project:study", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Filter variants from the given studies, these can be either the numeric ID or the alias with the format organization@project:study", required = false, arity = 1) public String study; @Parameter(names = {"--file"}, description = "Filter variants from the files specified. This will set includeFile parameter when not provided", required = false, arity = 1) @@ -1854,7 +1969,7 @@ public class AclCommandOptions { @Parameter(names = {"--clinical-analyses"}, description = "Comma separated list of clinical analysis IDs or names up to a maximum of 100", required = true, arity = 1) public String clinicalAnalyses; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--member"}, description = "User or group ID", required = false, arity = 1) @@ -1871,7 +1986,7 @@ public class DeleteCommandOptions { @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--force"}, description = "Force deletion if the ClinicalAnalysis contains interpretations or is locked", required = false, help = true, arity = 0) @@ -1903,7 +2018,7 @@ public class UpdateCommandOptions { @Parameter(names = {"--clinical-analyses"}, description = "Comma separated list of clinical analysis IDs", required = true, arity = 1) public String clinicalAnalyses; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--analysts-action"}, description = "Action to be performed if the array of analysts is being updated.", required = false, arity = 1) @@ -1927,8 +2042,8 @@ public class UpdateCommandOptions { @Parameter(names = {"--disorder-id"}, description = "The body web service id parameter", required = false, arity = 1) public String disorderId; - @Parameter(names = {"--panel-lock"}, description = "The body web service panelLock parameter", required = false, arity = 1) - public Boolean panelLock; + @Parameter(names = {"--panel-locked"}, description = "The body web service panelLocked parameter", required = false, arity = 1) + public Boolean panelLocked; @Parameter(names = {"--proband-id"}, description = "The body web service id parameter", required = false, arity = 1) public String probandId; @@ -2023,6 +2138,9 @@ public class UpdateCommandOptions { @Parameter(names = {"--status-id"}, description = "The body web service id parameter", required = false, arity = 1) public String statusId; + @Parameter(names = {"--panel-lock"}, description = "The body web service panelLock parameter", required = false, arity = 1) + public Boolean panelLock; + } @Parameters(commandNames = {"annotation-sets-annotations-update"}, commandDescription ="Update annotations from an annotationSet") @@ -2040,7 +2158,7 @@ public class UpdateAnnotationSetsAnnotationsCommandOptions { @Parameter(names = {"--clinical-analysis"}, description = "Clinical analysis ID", required = true, arity = 1) public String clinicalAnalysis; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--annotation-set"}, description = "AnnotationSet ID to be updated.", required = true, arity = 1) @@ -2069,9 +2187,12 @@ public class InfoCommandOptions { @Parameter(names = {"--clinical-analysis"}, description = "Comma separated list of clinical analysis IDs or names up to a maximum of 100", required = true, arity = 1) public String clinicalAnalysis; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; + @Parameter(names = {"--version"}, description = "Comma separated list of clinical versions. 'all' to get all the clinical versions. Not supported if multiple clinical ids are provided", required = false, arity = 1) + public String version; + @Parameter(names = {"--deleted"}, description = "Boolean to retrieve deleted entries", required = false, help = true, arity = 0) public boolean deleted = false; @@ -2098,7 +2219,7 @@ public class CreateInterpretationCommandOptions { @Parameter(names = {"--clinical-analysis"}, description = "Clinical analysis ID", required = true, arity = 1) public String clinicalAnalysis; - @Parameter(names = {"--study", "-s"}, description = "[[user@]project:]study id", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "[[organization@]project:]study id", required = false, arity = 1) public String study; @Parameter(names = {"--set-as"}, description = "Set interpretation as", required = false, arity = 1) @@ -2107,6 +2228,9 @@ public class CreateInterpretationCommandOptions { @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) public boolean includeResult = false; + @Parameter(names = {"--name", "-n"}, description = "The body web service name parameter", required = false, arity = 1) + public String name; + @Parameter(names = {"--description"}, description = "The body web service description parameter", required = false, arity = 1) public String description; @@ -2154,7 +2278,7 @@ public class ClearInterpretationCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "[[user@]project:]study ID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "[[organization@]project:]study ID", required = false, arity = 1) public String study; @Parameter(names = {"--interpretations"}, description = "Interpretation IDs of the Clinical Analysis", required = true, arity = 1) @@ -2171,7 +2295,7 @@ public class DeleteInterpretationCommandOptions { @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - @Parameter(names = {"--study", "-s"}, description = "[[user@]project:]study ID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "[[organization@]project:]study ID", required = false, arity = 1) public String study; @Parameter(names = {"--clinical-analysis"}, description = "Clinical analysis ID", required = true, arity = 1) @@ -2197,7 +2321,7 @@ public class RevertInterpretationCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "[[user@]project:]study ID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "[[organization@]project:]study ID", required = false, arity = 1) public String study; @Parameter(names = {"--clinical-analysis"}, description = "Clinical analysis ID", required = true, arity = 1) @@ -2229,7 +2353,7 @@ public class UpdateInterpretationCommandOptions { @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) public String exclude; - @Parameter(names = {"--study", "-s"}, description = "[[user@]project:]study ID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "[[organization@]project:]study ID", required = false, arity = 1) public String study; @Parameter(names = {"--set-as"}, description = "Set interpretation as", required = false, arity = 1) @@ -2244,6 +2368,9 @@ public class UpdateInterpretationCommandOptions { @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) public boolean includeResult = false; + @Parameter(names = {"--name", "-n"}, description = "The body web service name parameter", required = false, arity = 1) + public String name; + @Parameter(names = {"--description"}, description = "The body web service description parameter", required = false, arity = 1) public String description; @@ -2297,7 +2424,7 @@ public class UpdateReportCommandOptions { @Parameter(names = {"--clinical-analysis"}, description = "Clinical analysis ID", required = true, arity = 1) public String clinicalAnalysis; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--supporting-evidences-action"}, description = "Action to be performed if the array of supporting evidences is being updated.", required = false, arity = 1) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java index 069e1ef9322..44a0ad64dd7 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java @@ -135,10 +135,10 @@ public class AggregationStatsCommandOptions { @Parameter(names = {"--type"}, description = "List of types, accepted values are SNV, MNV, INDEL, SV, COPY_NUMBER, COPY_NUMBER_LOSS, COPY_NUMBER_GAIN, INSERTION, DELETION, DUPLICATION, TANDEM_DUPLICATION, BREAKEND, e.g. SNV,INDEL", required = false, arity = 1) public String type; - @Parameter(names = {"--project", "-p"}, description = "Project [user@]project where project can be either the ID or the alias", required = false, arity = 1) + @Parameter(names = {"--project", "-p"}, description = "Project [organization@]project where project can be either the ID or the alias", required = false, arity = 1) public String project; - @Parameter(names = {"--study", "-s"}, description = "Filter variants from the given studies, these can be either the numeric ID or the alias with the format user@project:study", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Filter variants from the given studies, these can be either the numeric ID or the alias with the format organization@project:study", required = false, arity = 1) public String study; @Parameter(names = {"--cohort"}, description = "Select variants with calculated stats for the selected cohorts", required = false, arity = 1) @@ -248,7 +248,7 @@ public class MetadataAnnotationCommandOptions { @Parameter(names = {"--annotation-id"}, description = "Annotation identifier", required = false, arity = 1) public String annotationId; - @Parameter(names = {"--project", "-p"}, description = "Project [user@]project where project can be either the ID or the alias", required = false, arity = 1) + @Parameter(names = {"--project", "-p"}, description = "Project [organization@]project where project can be either the ID or the alias", required = false, arity = 1) public String project; } @@ -351,7 +351,7 @@ public class RunCohortStatsCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) @@ -366,6 +366,15 @@ public class RunCohortStatsCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--cohort"}, description = "The body web service cohort parameter", required = false, arity = 1) public String cohort; @@ -410,9 +419,21 @@ public class RunExomiserCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--sample"}, description = "Sample ID.", required = false, arity = 1) public String sample; + @Parameter(names = {"--exomiser-version"}, description = "Exomiser version in the format X.Y where X is the major version and Y the minor version, e.g.: 14.0. If the version is not specified, the default version will be used. Refer to the configuration file to view all installed Exomiser versions and identify the default version.", required = false, arity = 1) + public String exomiserVersion; + @Parameter(names = {"--clinical-analysis-type"}, description = "Clinical analysis type: SINGLE or FAMILY.", required = false, arity = 1) public String clinicalAnalysisType = "SINGLE"; @@ -439,10 +460,10 @@ public class RunExportCommandOptions { @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) public String exclude; - @Parameter(names = {"--project", "-p"}, description = "Project [user@]project where project can be either the ID or the alias", required = false, arity = 1) + @Parameter(names = {"--project", "-p"}, description = "Project [organization@]project where project can be either the ID or the alias", required = false, arity = 1) public String project; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) @@ -457,6 +478,15 @@ public class RunExportCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--id"}, description = "The body web service id parameter", required = false, arity = 1) public String id; @@ -726,7 +756,7 @@ public class GenotypesFamilyCommandOptions { @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--family"}, description = "Family id", required = false, arity = 1) @@ -758,7 +788,7 @@ public class RunFamilyQcCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) @@ -773,6 +803,15 @@ public class RunFamilyQcCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--family"}, description = "The body web service family parameter", required = false, arity = 1) public String family; @@ -805,7 +844,16 @@ public class DeleteFileCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--file"}, description = "Files to remove", required = false, arity = 1) @@ -843,6 +891,15 @@ public class RunGatkCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--command"}, description = "The body web service command parameter", required = false, arity = 1) public String command; @@ -866,7 +923,7 @@ public class RunGenomePlotCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) @@ -881,6 +938,15 @@ public class RunGenomePlotCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--sample"}, description = "The body web service sample parameter", required = false, arity = 1) public String sample; @@ -910,7 +976,7 @@ public class RunGwasCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) @@ -925,6 +991,15 @@ public class RunGwasCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--phenotype"}, description = "The body web service phenotype parameter", required = false, arity = 1) public String phenotype; @@ -975,7 +1050,7 @@ public class RunHrDetectCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) @@ -990,6 +1065,15 @@ public class RunHrDetectCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--id"}, description = "ID to identify the HRDetect results.", required = false, arity = 1) public String id; @@ -1043,7 +1127,7 @@ public class RunIndexCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) @@ -1058,6 +1142,15 @@ public class RunIndexCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--file"}, description = "List of files to be indexed.", required = false, arity = 1) public String file; @@ -1162,7 +1255,7 @@ public class RunIndividualQcCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) @@ -1177,6 +1270,15 @@ public class RunIndividualQcCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--individual"}, description = "Individual ID", required = false, arity = 1) public String individual; @@ -1203,7 +1305,7 @@ public class RunInferredSexCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) @@ -1218,6 +1320,15 @@ public class RunInferredSexCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--individual"}, description = "Individual ID", required = false, arity = 1) public String individual; @@ -1296,6 +1407,15 @@ public class RunKnockoutCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--sample"}, description = "The body web service sample parameter", required = false, arity = 1) public String sample; @@ -1340,7 +1460,7 @@ public class RunMendelianErrorCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) @@ -1355,6 +1475,15 @@ public class RunMendelianErrorCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--family"}, description = "The body web service family parameter", required = false, arity = 1) public String family; @@ -1375,10 +1504,10 @@ public class MetadataCommandOptions { @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - @Parameter(names = {"--project", "-p"}, description = "Project [user@]project where project can be either the ID or the alias", required = false, arity = 1) + @Parameter(names = {"--project", "-p"}, description = "Project [organization@]project where project can be either the ID or the alias", required = false, arity = 1) public String project; - @Parameter(names = {"--study", "-s"}, description = "Filter variants from the given studies, these can be either the numeric ID or the alias with the format user@project:study", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Filter variants from the given studies, these can be either the numeric ID or the alias with the format organization@project:study", required = false, arity = 1) public String study; @Parameter(names = {"--file"}, description = "Filter variants from the files specified. This will set includeFile parameter when not provided", required = false, arity = 1) @@ -1410,7 +1539,7 @@ public class QueryMutationalSignatureCommandOptions { @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - @Parameter(names = {"--study", "-s"}, description = "Filter variants from the given studies, these can be either the numeric ID or the alias with the format user@project:study", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Filter variants from the given studies, these can be either the numeric ID or the alias with the format organization@project:study", required = false, arity = 1) public String study; @Parameter(names = {"--sample"}, description = "Sample name", required = false, arity = 1) @@ -1478,7 +1607,7 @@ public class RunMutationalSignatureCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) @@ -1493,6 +1622,15 @@ public class RunMutationalSignatureCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--id"}, description = "Signature ID.", required = false, arity = 1) public String id; @@ -1570,6 +1708,15 @@ public class RunPlinkCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--outdir"}, description = "The body web service outdir parameter", required = false, arity = 1) public String outdir; @@ -1629,10 +1776,10 @@ public class QueryCommandOptions { @Parameter(names = {"--alternate"}, description = "Main alternate allele", required = false, arity = 1) public String alternate; - @Parameter(names = {"--project", "-p"}, description = "Project [user@]project where project can be either the ID or the alias", required = false, arity = 1) + @Parameter(names = {"--project", "-p"}, description = "Project [organization@]project where project can be either the ID or the alias", required = false, arity = 1) public String project; - @Parameter(names = {"--study", "-s"}, description = "Filter variants from the given studies, these can be either the numeric ID or the alias with the format user@project:study", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Filter variants from the given studies, these can be either the numeric ID or the alias with the format organization@project:study", required = false, arity = 1) public String study; @Parameter(names = {"--file"}, description = "Filter variants from the files specified. This will set includeFile parameter when not provided", required = false, arity = 1) @@ -1829,7 +1976,7 @@ public class RunRelatednessCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) @@ -1844,6 +1991,15 @@ public class RunRelatednessCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--individuals"}, description = "The body web service individuals parameter", required = false, arity = 1) public String individuals; @@ -1888,6 +2044,15 @@ public class RunRvtestsCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--command"}, description = "The body web service command parameter", required = false, arity = 1) public String command; @@ -1914,10 +2079,10 @@ public class AggregationStatsSampleCommandOptions { @Parameter(names = {"--type"}, description = "List of types, accepted values are SNV, MNV, INDEL, SV, COPY_NUMBER, COPY_NUMBER_LOSS, COPY_NUMBER_GAIN, INSERTION, DELETION, DUPLICATION, TANDEM_DUPLICATION, BREAKEND, e.g. SNV,INDEL", required = false, arity = 1) public String type; - @Parameter(names = {"--project", "-p"}, description = "Project [user@]project where project can be either the ID or the alias", required = false, arity = 1) + @Parameter(names = {"--project", "-p"}, description = "Project [organization@]project where project can be either the ID or the alias", required = false, arity = 1) public String project; - @Parameter(names = {"--study", "-s"}, description = "Filter variants from the given studies, these can be either the numeric ID or the alias with the format user@project:study", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Filter variants from the given studies, these can be either the numeric ID or the alias with the format organization@project:study", required = false, arity = 1) public String study; @Parameter(names = {"--file"}, description = "Filter variants from the files specified. This will set includeFile parameter when not provided", required = false, arity = 1) @@ -1985,7 +2150,7 @@ public class RunSampleEligibilityCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) @@ -2000,6 +2165,15 @@ public class RunSampleEligibilityCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--query"}, description = "The body web service query parameter", required = false, arity = 1) public String query; @@ -2023,7 +2197,7 @@ public class RunSampleQcCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) @@ -2038,6 +2212,15 @@ public class RunSampleQcCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--sample"}, description = "Sample ID.", required = false, arity = 1) public String sample; @@ -2216,7 +2399,7 @@ public class RunSampleCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) @@ -2231,6 +2414,15 @@ public class RunSampleCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--id"}, description = "The body web service id parameter", required = false, arity = 1) public String id; @@ -2335,7 +2527,7 @@ public class QuerySampleStatsCommandOptions { @Parameter(names = {"--type"}, description = "List of types, accepted values are SNV, MNV, INDEL, SV, COPY_NUMBER, COPY_NUMBER_LOSS, COPY_NUMBER_GAIN, INSERTION, DELETION, DUPLICATION, TANDEM_DUPLICATION, BREAKEND, e.g. SNV,INDEL", required = false, arity = 1) public String type; - @Parameter(names = {"--study", "-s"}, description = "Filter variants from the given studies, these can be either the numeric ID or the alias with the format user@project:study", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Filter variants from the given studies, these can be either the numeric ID or the alias with the format organization@project:study", required = false, arity = 1) public String study; @Parameter(names = {"--file"}, description = "Filter variants from the files specified. This will set includeFile parameter when not provided", required = false, arity = 1) @@ -2388,7 +2580,7 @@ public class RunSampleStatsCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) @@ -2403,6 +2595,15 @@ public class RunSampleStatsCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--sample"}, description = "The body web service sample parameter", required = false, arity = 1) public String sample; @@ -2525,10 +2726,10 @@ public class RunStatsExportCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--project", "-p"}, description = "Project [user@]project where project can be either the ID or the alias", required = false, arity = 1) + @Parameter(names = {"--project", "-p"}, description = "Project [organization@]project where project can be either the ID or the alias", required = false, arity = 1) public String project; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) @@ -2543,6 +2744,15 @@ public class RunStatsExportCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--cohorts"}, description = "The body web service cohorts parameter", required = false, arity = 1) public String cohorts; @@ -2572,7 +2782,7 @@ public class RunStatsCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) @@ -2587,6 +2797,15 @@ public class RunStatsCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--cohort"}, description = "The body web service cohort parameter", required = false, arity = 1) public String cohort; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/CohortsCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/CohortsCommandOptions.java index 03f08e58c9e..632b7eaa1b3 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/CohortsCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/CohortsCommandOptions.java @@ -34,7 +34,6 @@ public class CohortsCommandOptions { public CommonCommandOptions commonCommandOptions; public UpdateAclCommandOptions updateAclCommandOptions; - public AggregationStatsCommandOptions aggregationStatsCommandOptions; public LoadAnnotationSetsCommandOptions loadAnnotationSetsCommandOptions; public CreateCommandOptions createCommandOptions; public DistinctCommandOptions distinctCommandOptions; @@ -52,7 +51,6 @@ public CohortsCommandOptions(CommonCommandOptions commonCommandOptions, JCommand this.jCommander = jCommander; this.commonCommandOptions = commonCommandOptions; this.updateAclCommandOptions = new UpdateAclCommandOptions(); - this.aggregationStatsCommandOptions = new AggregationStatsCommandOptions(); this.loadAnnotationSetsCommandOptions = new LoadAnnotationSetsCommandOptions(); this.createCommandOptions = new CreateCommandOptions(); this.distinctCommandOptions = new DistinctCommandOptions(); @@ -78,7 +76,7 @@ public class UpdateAclCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--members"}, description = "Comma separated list of user or group ids", required = true, arity = 1) @@ -95,50 +93,6 @@ public class UpdateAclCommandOptions { } - @Parameters(commandNames = {"aggregationstats"}, commandDescription ="Fetch catalog cohort stats") - public class AggregationStatsCommandOptions { - - @ParametersDelegate - public CommonCommandOptions commonOptions = commonCommandOptions; - - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - - @Parameter(names = {"--type"}, description = "Type", required = false, arity = 1) - public String type; - - @Parameter(names = {"--creation-year"}, description = "Creation year", required = false, arity = 1) - public String creationYear; - - @Parameter(names = {"--creation-month"}, description = "Creation month (JANUARY, FEBRUARY...)", required = false, arity = 1) - public String creationMonth; - - @Parameter(names = {"--creation-day"}, description = "Creation day", required = false, arity = 1) - public String creationDay; - - @Parameter(names = {"--creation-day-of-week"}, description = "Creation day of week (MONDAY, TUESDAY...)", required = false, arity = 1) - public String creationDayOfWeek; - - @Parameter(names = {"--num-samples"}, description = "Number of samples", required = false, arity = 1) - public String numSamples; - - @Parameter(names = {"--status"}, description = "Status", required = false, arity = 1) - public String status; - - @Parameter(names = {"--release"}, description = "Release", required = false, arity = 1) - public String release; - - @Parameter(names = {"--annotation"}, description = "Annotation filters. Example: age>30;gender=FEMALE. For more information, please visit http://docs.opencb.org/display/opencga/AnnotationSets+1.4.0", required = false, arity = 1) - public String annotation; - - @Parameter(names = {"--default"}, description = "Calculate default stats", required = false, help = true, arity = 0) - public boolean default_values = false; - - @Parameter(names = {"--field"}, description = "List of fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type;numSamples[0..10]:1", required = false, arity = 1) - public String field; - - } - @Parameters(commandNames = {"annotation-sets-load"}, commandDescription ="Load annotation sets from a TSV file") public class LoadAnnotationSetsCommandOptions { @@ -151,7 +105,7 @@ public class LoadAnnotationSetsCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--variable-set-id"}, description = "Variable set ID or name", required = true, arity = 1) @@ -189,7 +143,7 @@ public class CreateCommandOptions { @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) public String exclude; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--variable-set"}, description = "Deprecated: Use /generate web service and filter by annotation", required = false, arity = 1) @@ -239,7 +193,7 @@ public class DistinctCommandOptions { @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--id"}, description = "Comma separated list of cohort IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = false, arity = 1) @@ -307,7 +261,7 @@ public class GenerateCommandOptions { @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) public String exclude; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--id"}, description = "Comma separated list sample IDs or UUIDs up to a maximum of 100", required = false, arity = 1) @@ -408,7 +362,7 @@ public class SearchCommandOptions { @Parameter(names = {"--flatten-annotations"}, description = "Flatten the annotations?", required = false, help = true, arity = 0) public boolean flattenAnnotations = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--id"}, description = "Comma separated list of cohort IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = false, arity = 1) @@ -464,7 +418,7 @@ public class AclCommandOptions { @Parameter(names = {"--cohorts"}, description = "Comma separated list of cohort IDs or UUIDs up to a maximum of 100", required = true, arity = 1) public String cohorts; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--member"}, description = "User or group id", required = false, arity = 1) @@ -481,7 +435,7 @@ public class DeleteCommandOptions { @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--cohorts"}, description = "Comma separated list of cohort ids", required = true, arity = 1) @@ -507,7 +461,7 @@ public class InfoCommandOptions { @Parameter(names = {"--cohorts"}, description = "Comma separated list of cohort IDs or UUIDs up to a maximum of 100", required = true, arity = 1) public String cohorts; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--deleted"}, description = "Boolean to retrieve deleted cohorts", required = false, help = true, arity = 0) @@ -536,7 +490,7 @@ public class UpdateCommandOptions { @Parameter(names = {"--cohorts"}, description = "Comma separated list of cohort ids", required = true, arity = 1) public String cohorts; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) @@ -589,7 +543,7 @@ public class UpdateAnnotationSetsAnnotationsCommandOptions { @Parameter(names = {"--cohort"}, description = "Cohort ID", required = true, arity = 1) public String cohort; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--annotation-set"}, description = "AnnotationSet ID to be updated.", required = true, arity = 1) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/DiseasePanelsCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/DiseasePanelsCommandOptions.java index 1f0f1cbb9bc..d69121d7c21 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/DiseasePanelsCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/DiseasePanelsCommandOptions.java @@ -72,7 +72,7 @@ public class UpdateAclCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--members"}, description = "Comma separated list of user or group ids", required = true, arity = 1) @@ -107,7 +107,7 @@ public class CreateCommandOptions { @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) public String exclude; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) @@ -154,7 +154,7 @@ public class DistinctCommandOptions { @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--id"}, description = "Comma separated list of panel IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = false, arity = 1) @@ -228,7 +228,7 @@ public class ImportCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) @@ -243,6 +243,15 @@ public class ImportCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--source"}, description = "Comma separated list of sources to import panels from. Current supported sources are 'panelapp' and 'cancer-gene-census'", required = false, arity = 1) public String source; @@ -272,7 +281,7 @@ public class SearchCommandOptions { @Parameter(names = {"--count"}, description = "Get the total number of results matching the query. Deactivated by default.", required = false, help = true, arity = 0) public boolean count = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--id"}, description = "Comma separated list of panel IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = false, arity = 1) @@ -340,7 +349,7 @@ public class AclCommandOptions { @Parameter(names = {"--panels"}, description = "Comma separated list of panel IDs up to a maximum of 100", required = true, arity = 1) public String panels; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--member"}, description = "User or group id", required = false, arity = 1) @@ -357,7 +366,7 @@ public class DeleteCommandOptions { @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--panels"}, description = "Comma separated list of panel ids", required = true, arity = 1) @@ -380,7 +389,7 @@ public class InfoCommandOptions { @Parameter(names = {"--panels"}, description = "Comma separated list of panel IDs up to a maximum of 100", required = true, arity = 1) public String panels; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--version"}, description = "Comma separated list of panel versions. 'all' to get all the panel versions. Not supported if multiple panel ids are provided", required = false, arity = 1) @@ -409,7 +418,7 @@ public class UpdateCommandOptions { @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) public String exclude; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--panels"}, description = "Comma separated list of panel ids", required = true, arity = 1) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/FamiliesCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/FamiliesCommandOptions.java index 3c57ef335ba..0fc4c466ae5 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/FamiliesCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/FamiliesCommandOptions.java @@ -34,7 +34,6 @@ public class FamiliesCommandOptions { public CommonCommandOptions commonCommandOptions; public UpdateAclCommandOptions updateAclCommandOptions; - public AggregationStatsCommandOptions aggregationStatsCommandOptions; public LoadAnnotationSetsCommandOptions loadAnnotationSetsCommandOptions; public CreateCommandOptions createCommandOptions; public DistinctCommandOptions distinctCommandOptions; @@ -51,7 +50,6 @@ public FamiliesCommandOptions(CommonCommandOptions commonCommandOptions, JComman this.jCommander = jCommander; this.commonCommandOptions = commonCommandOptions; this.updateAclCommandOptions = new UpdateAclCommandOptions(); - this.aggregationStatsCommandOptions = new AggregationStatsCommandOptions(); this.loadAnnotationSetsCommandOptions = new LoadAnnotationSetsCommandOptions(); this.createCommandOptions = new CreateCommandOptions(); this.distinctCommandOptions = new DistinctCommandOptions(); @@ -76,7 +74,7 @@ public class UpdateAclCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--members"}, description = "Comma separated list of user or group ids", required = true, arity = 1) @@ -102,56 +100,6 @@ public class UpdateAclCommandOptions { } - @Parameters(commandNames = {"aggregationstats"}, commandDescription ="Fetch catalog family stats") - public class AggregationStatsCommandOptions { - - @ParametersDelegate - public CommonCommandOptions commonOptions = commonCommandOptions; - - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - - @Parameter(names = {"--creation-year"}, description = "Creation year", required = false, arity = 1) - public String creationYear; - - @Parameter(names = {"--creation-month"}, description = "Creation month (JANUARY, FEBRUARY...)", required = false, arity = 1) - public String creationMonth; - - @Parameter(names = {"--creation-day"}, description = "Creation day", required = false, arity = 1) - public String creationDay; - - @Parameter(names = {"--creation-day-of-week"}, description = "Creation day of week (MONDAY, TUESDAY...)", required = false, arity = 1) - public String creationDayOfWeek; - - @Parameter(names = {"--status"}, description = "Status", required = false, arity = 1) - public String status; - - @Parameter(names = {"--phenotypes"}, description = "Phenotypes", required = false, arity = 1) - public String phenotypes; - - @Parameter(names = {"--release"}, description = "Release", required = false, arity = 1) - public String release; - - @Parameter(names = {"--version"}, description = "Version", required = false, arity = 1) - public String version; - - @Parameter(names = {"--num-members"}, description = "Number of members", required = false, arity = 1) - public String numMembers; - - @Parameter(names = {"--expected-size"}, description = "Expected size", required = false, arity = 1) - public String expectedSize; - - @Parameter(names = {"--annotation"}, description = "Annotation filters. Example: age>30;gender=FEMALE. For more information, please visit http://docs.opencb.org/display/opencga/AnnotationSets+1.4.0", required = false, arity = 1) - public String annotation; - - @Parameter(names = {"--default"}, description = "Calculate default stats", required = false, help = true, arity = 0) - public boolean default_values = false; - - @Parameter(names = {"--field"}, description = "List of fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type;numSamples[0..10]:1", required = false, arity = 1) - public String field; - - } - @Parameters(commandNames = {"annotation-sets-load"}, commandDescription ="Load annotation sets from a TSV file") public class LoadAnnotationSetsCommandOptions { @@ -164,7 +112,7 @@ public class LoadAnnotationSetsCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--variable-set-id"}, description = "Variable set ID or name", required = true, arity = 1) @@ -202,7 +150,7 @@ public class CreateCommandOptions { @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) public String exclude; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--members"}, description = "Comma separated list of member ids to be associated to the created family", required = false, arity = 1) @@ -249,7 +197,7 @@ public class DistinctCommandOptions { @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--id"}, description = "Comma separated list family IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = false, arity = 1) @@ -332,7 +280,7 @@ public class SearchCommandOptions { @Parameter(names = {"--flatten-annotations"}, description = "Flatten the annotations?", required = false, help = true, arity = 0) public boolean flattenAnnotations = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--id"}, description = "Comma separated list family IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = false, arity = 1) @@ -397,7 +345,7 @@ public class AclCommandOptions { @Parameter(names = {"--families"}, description = "Comma separated list of family IDs or names up to a maximum of 100", required = true, arity = 1) public String families; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--member"}, description = "User or group id", required = false, arity = 1) @@ -414,7 +362,7 @@ public class DeleteCommandOptions { @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--families"}, description = "Comma separated list of family ids", required = true, arity = 1) @@ -440,7 +388,7 @@ public class InfoCommandOptions { @Parameter(names = {"--families"}, description = "Comma separated list of family IDs or names up to a maximum of 100", required = true, arity = 1) public String families; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--version"}, description = "Comma separated list of family versions. 'all' to get all the family versions. Not supported if multiple family ids are provided", required = false, arity = 1) @@ -472,7 +420,7 @@ public class UpdateCommandOptions { @Parameter(names = {"--families"}, description = "Comma separated list of family ids", required = true, arity = 1) public String families; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--update-roles"}, description = "Update the member roles within the family", required = false, help = true, arity = 0) @@ -534,7 +482,7 @@ public class UpdateAnnotationSetsAnnotationsCommandOptions { @Parameter(names = {"--family"}, description = "Family id", required = true, arity = 1) public String family; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--annotation-set"}, description = "AnnotationSet ID to be updated.", required = true, arity = 1) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/FilesCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/FilesCommandOptions.java index a3c6cd4bf9c..dfbd2cb0e5d 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/FilesCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/FilesCommandOptions.java @@ -34,7 +34,6 @@ public class FilesCommandOptions extends CustomFilesCommandOptions { public UpdateAclCommandOptions updateAclCommandOptions; - public AggregationStatsCommandOptions aggregationStatsCommandOptions; public LoadAnnotationSetsCommandOptions loadAnnotationSetsCommandOptions; public BioformatsCommandOptions bioformatsCommandOptions; public CreateCommandOptions createCommandOptions; @@ -67,7 +66,6 @@ public FilesCommandOptions(CommonCommandOptions commonCommandOptions, JCommander super(commonCommandOptions,jCommander); this.updateAclCommandOptions = new UpdateAclCommandOptions(); - this.aggregationStatsCommandOptions = new AggregationStatsCommandOptions(); this.loadAnnotationSetsCommandOptions = new LoadAnnotationSetsCommandOptions(); this.bioformatsCommandOptions = new BioformatsCommandOptions(); this.createCommandOptions = new CreateCommandOptions(); @@ -109,7 +107,7 @@ public class UpdateAclCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--members"}, description = "Comma separated list of user or group ids", required = true, arity = 1) @@ -129,74 +127,6 @@ public class UpdateAclCommandOptions { } - @Parameters(commandNames = {"aggregationstats"}, commandDescription ="Fetch catalog file stats") - public class AggregationStatsCommandOptions { - - @ParametersDelegate - public CommonCommandOptions commonOptions = commonCommandOptions; - - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - - @Parameter(names = {"--name", "-n"}, description = "Name", required = false, arity = 1) - public String name; - - @Parameter(names = {"--type"}, description = "Type", required = false, arity = 1) - public String type; - - @Parameter(names = {"--format"}, description = "Format", required = false, arity = 1) - public String format; - - @Parameter(names = {"--bioformat"}, description = "Bioformat", required = false, arity = 1) - public String bioformat; - - @Parameter(names = {"--creation-year"}, description = "Creation year", required = false, arity = 1) - public String creationYear; - - @Parameter(names = {"--creation-month"}, description = "Creation month (JANUARY, FEBRUARY...)", required = false, arity = 1) - public String creationMonth; - - @Parameter(names = {"--creation-day"}, description = "Creation day", required = false, arity = 1) - public String creationDay; - - @Parameter(names = {"--creation-day-of-week"}, description = "Creation day of week (MONDAY, TUESDAY...)", required = false, arity = 1) - public String creationDayOfWeek; - - @Parameter(names = {"--status"}, description = "Status", required = false, arity = 1) - public String status; - - @Parameter(names = {"--release"}, description = "Release", required = false, arity = 1) - public String release; - - @Parameter(names = {"--external"}, description = "External", required = false, arity = 1) - public Boolean external; - - @Parameter(names = {"--size"}, description = "Size", required = false, arity = 1) - public String size; - - @Parameter(names = {"--software"}, description = "Software", required = false, arity = 1) - public String software; - - @Parameter(names = {"--experiment"}, description = "Experiment", required = false, arity = 1) - public String experiment; - - @Parameter(names = {"--num-samples"}, description = "Number of samples", required = false, arity = 1) - public String numSamples; - - @Parameter(names = {"--num-related-files"}, description = "Number of related files", required = false, arity = 1) - public String numRelatedFiles; - - @Parameter(names = {"--annotation"}, description = "Annotation filters. Example: age>30;gender=FEMALE. For more information, please visit http://docs.opencb.org/display/opencga/AnnotationSets+1.4.0", required = false, arity = 1) - public String annotation; - - @Parameter(names = {"--default"}, description = "Calculate default stats", required = false, help = true, arity = 0) - public boolean default_values = false; - - @Parameter(names = {"--field"}, description = "List of fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type;numSamples[0..10]:1", required = false, arity = 1) - public String field; - - } - @Parameters(commandNames = {"annotation-sets-load"}, commandDescription ="Load annotation sets from a TSV file") public class LoadAnnotationSetsCommandOptions { @@ -209,7 +139,7 @@ public class LoadAnnotationSetsCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--variable-set-id"}, description = "Variable set ID or name", required = true, arity = 1) @@ -249,7 +179,7 @@ public class CreateCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--parents"}, description = "Create the parent directories if they do not exist", required = false, help = true, arity = 0) @@ -326,7 +256,7 @@ public class DistinctCommandOptions { @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--id"}, description = "Comma separated list of file IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = false, arity = 1) @@ -433,7 +363,16 @@ public class FetchCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--url"}, description = "The body web service url parameter", required = true, arity = 1) @@ -464,7 +403,7 @@ public class LinkCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--parents"}, description = "Create the parent directories if they do not exist", required = false, help = true, arity = 0) @@ -514,7 +453,7 @@ public class RunLinkCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) @@ -529,6 +468,15 @@ public class RunLinkCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--uri", "--input", "-i"}, description = "The body web service uri parameter", required = false, arity = 1) public String uri; @@ -561,7 +509,7 @@ public class RunPostlinkCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) @@ -576,6 +524,15 @@ public class RunPostlinkCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--files"}, description = "The body web service files parameter", required = false, arity = 1) public String files; @@ -608,7 +565,7 @@ public class SearchCommandOptions { @Parameter(names = {"--flatten-annotations"}, description = "Boolean indicating to flatten the annotations.", required = false, help = true, arity = 0) public boolean flattenAnnotations = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--id"}, description = "Comma separated list of file IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = false, arity = 1) @@ -697,7 +654,7 @@ public class AclCommandOptions { @Parameter(names = {"--files"}, description = "Comma separated list of file IDs or names up to a maximum of 100", required = true, arity = 1) public String files; - @Parameter(names = {"--study", "-s"}, description = "Comma separated list of Studies [[user@]project:]study where study and project can be either the ID or UUID up to a maximum of 100", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Comma separated list of Studies [[organization@]project:]study where study and project can be either the ID or UUID up to a maximum of 100", required = false, arity = 1) public String study; @Parameter(names = {"--member"}, description = "User or group id", required = false, arity = 1) @@ -714,7 +671,7 @@ public class DeleteCommandOptions { @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--files"}, description = "Comma separated list of file ids, names or paths.", required = true, arity = 1) @@ -743,7 +700,7 @@ public class InfoCommandOptions { @Parameter(names = {"--files"}, description = "Comma separated list of file IDs or names up to a maximum of 100", required = true, arity = 1) public String files; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--deleted"}, description = "Boolean to retrieve deleted files", required = false, help = true, arity = 0) @@ -757,7 +714,7 @@ public class UnlinkCommandOptions { @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--files"}, description = "Comma separated list of file ids, names or paths.", required = true, arity = 1) @@ -786,7 +743,7 @@ public class UpdateCommandOptions { @Parameter(names = {"--files"}, description = "Comma separated list of file ids, names or paths. Paths must be separated by : instead of /", required = true, arity = 1) public String files; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--sample-ids-action"}, description = "Action to be performed if the array of samples is being updated.", required = false, arity = 1) @@ -902,7 +859,7 @@ public class UpdateAnnotationSetsAnnotationsCommandOptions { @Parameter(names = {"--file"}, description = "File id, name or path. Paths must be separated by : instead of /", required = true, arity = 1) public String file; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--annotation-set"}, description = "AnnotationSet ID to be updated.", required = true, arity = 1) @@ -922,7 +879,7 @@ public class DownloadCommandOptions { @Parameter(names = {"--file"}, description = "File id, name or path. Paths must be separated by : instead of /", required = true, arity = 1) public String file; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; } @@ -936,7 +893,7 @@ public class GrepCommandOptions { @Parameter(names = {"--file"}, description = "File uuid, id, or name.", required = true, arity = 1) public String file; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--pattern"}, description = "String pattern", required = false, arity = 1) @@ -959,7 +916,7 @@ public class HeadCommandOptions { @Parameter(names = {"--file"}, description = "File uuid, id, or name.", required = true, arity = 1) public String file; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--offset"}, description = "Starting byte from which the file will be read", required = false, arity = 1) @@ -979,7 +936,7 @@ public class ImageCommandOptions { @Parameter(names = {"--file"}, description = "File ID", required = true, arity = 1) public String file; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; } @@ -1005,7 +962,7 @@ public class MoveCommandOptions { @Parameter(names = {"--file"}, description = "File id, UUID or name.", required = true, arity = 1) public String file; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--path"}, description = "The body web service path parameter", required = false, arity = 1) @@ -1022,7 +979,7 @@ public class RefreshCommandOptions { @Parameter(names = {"--file"}, description = "File id, name or path. Paths must be separated by : instead of /", required = true, arity = 1) public String file; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; } @@ -1036,7 +993,7 @@ public class TailCommandOptions { @Parameter(names = {"--file"}, description = "File uuid, id, or name.", required = true, arity = 1) public String file; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--lines"}, description = "Maximum number of lines to be returned up to a maximum of 1000", required = false, arity = 1) @@ -1068,7 +1025,7 @@ public class ListCommandOptions { @Parameter(names = {"--folder"}, description = "Folder ID, name or path", required = true, arity = 1) public String folder; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; } @@ -1088,7 +1045,7 @@ public class TreeCommandOptions { @Parameter(names = {"--folder"}, description = "Folder id or name. Paths must be separated by : instead of /", required = true, arity = 1) public String folder; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--max-depth"}, description = "Maximum depth to get files from", required = false, arity = 1) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/IndividualsCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/IndividualsCommandOptions.java index d0238597f52..139e2d4ab65 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/IndividualsCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/IndividualsCommandOptions.java @@ -34,7 +34,6 @@ public class IndividualsCommandOptions { public CommonCommandOptions commonCommandOptions; public UpdateAclCommandOptions updateAclCommandOptions; - public AggregationStatsCommandOptions aggregationStatsCommandOptions; public LoadAnnotationSetsCommandOptions loadAnnotationSetsCommandOptions; public CreateCommandOptions createCommandOptions; public DistinctCommandOptions distinctCommandOptions; @@ -52,7 +51,6 @@ public IndividualsCommandOptions(CommonCommandOptions commonCommandOptions, JCom this.jCommander = jCommander; this.commonCommandOptions = commonCommandOptions; this.updateAclCommandOptions = new UpdateAclCommandOptions(); - this.aggregationStatsCommandOptions = new AggregationStatsCommandOptions(); this.loadAnnotationSetsCommandOptions = new LoadAnnotationSetsCommandOptions(); this.createCommandOptions = new CreateCommandOptions(); this.distinctCommandOptions = new DistinctCommandOptions(); @@ -78,7 +76,7 @@ public class UpdateAclCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--members"}, description = "Comma separated list of user or group ids", required = true, arity = 1) @@ -101,77 +99,6 @@ public class UpdateAclCommandOptions { } - @Parameters(commandNames = {"aggregationstats"}, commandDescription ="Fetch catalog individual stats") - public class AggregationStatsCommandOptions { - - @ParametersDelegate - public CommonCommandOptions commonOptions = commonCommandOptions; - - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - - @Parameter(names = {"--has-father"}, description = "Has father", required = false, arity = 1) - public Boolean hasFather; - - @Parameter(names = {"--has-mother"}, description = "Has mother", required = false, arity = 1) - public Boolean hasMother; - - @Parameter(names = {"--sex"}, description = "Sex", required = false, arity = 1) - public String sex; - - @Parameter(names = {"--karyotypic-sex"}, description = "Karyotypic sex", required = false, arity = 1) - public String karyotypicSex; - - @Parameter(names = {"--ethnicity"}, description = "Ethnicity", required = false, arity = 1) - public String ethnicity; - - @Parameter(names = {"--population"}, description = "Population", required = false, arity = 1) - public String population; - - @Parameter(names = {"--creation-year"}, description = "Creation year", required = false, arity = 1) - public String creationYear; - - @Parameter(names = {"--creation-month"}, description = "Creation month (JANUARY, FEBRUARY...)", required = false, arity = 1) - public String creationMonth; - - @Parameter(names = {"--creation-day"}, description = "Creation day", required = false, arity = 1) - public String creationDay; - - @Parameter(names = {"--creation-day-of-week"}, description = "Creation day of week (MONDAY, TUESDAY...)", required = false, arity = 1) - public String creationDayOfWeek; - - @Parameter(names = {"--status"}, description = "Status", required = false, arity = 1) - public String status; - - @Parameter(names = {"--life-status"}, description = "Life status", required = false, arity = 1) - public String lifeStatus; - - @Parameter(names = {"--phenotypes"}, description = "Phenotypes", required = false, arity = 1) - public String phenotypes; - - @Parameter(names = {"--num-samples"}, description = "Number of samples", required = false, arity = 1) - public String numSamples; - - @Parameter(names = {"--parental-consanguinity"}, description = "Parental consanguinity", required = false, arity = 1) - public Boolean parentalConsanguinity; - - @Parameter(names = {"--release"}, description = "Release", required = false, arity = 1) - public String release; - - @Parameter(names = {"--version"}, description = "Version", required = false, arity = 1) - public String version; - - @Parameter(names = {"--annotation"}, description = "Annotation filters. Example: age>30;gender=FEMALE. For more information, please visit http://docs.opencb.org/display/opencga/AnnotationSets+1.4.0", required = false, arity = 1) - public String annotation; - - @Parameter(names = {"--default"}, description = "Calculate default stats", required = false, help = true, arity = 0) - public boolean default_values = false; - - @Parameter(names = {"--field"}, description = "List of fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type;numSamples[0..10]:1", required = false, arity = 1) - public String field; - - } - @Parameters(commandNames = {"annotation-sets-load"}, commandDescription ="Load annotation sets from a TSV file") public class LoadAnnotationSetsCommandOptions { @@ -184,7 +111,7 @@ public class LoadAnnotationSetsCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--variable-set-id"}, description = "Variable set ID or name", required = true, arity = 1) @@ -222,7 +149,7 @@ public class CreateCommandOptions { @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) public String exclude; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--samples"}, description = "Comma separated list of sample ids to be associated to the created individual", required = false, arity = 1) @@ -347,7 +274,7 @@ public class DistinctCommandOptions { @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--id"}, description = "Comma separated list individual IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = false, arity = 1) @@ -454,7 +381,7 @@ public class SearchCommandOptions { @Parameter(names = {"--flatten-annotations"}, description = "Flatten the annotations?", required = false, help = true, arity = 0) public boolean flattenAnnotations = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--id"}, description = "Comma separated list individual IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = false, arity = 1) @@ -543,7 +470,7 @@ public class AclCommandOptions { @Parameter(names = {"--individuals"}, description = "Comma separated list of individual IDs, names or UUIDs up to a maximum of 100", required = true, arity = 1) public String individuals; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--member"}, description = "User or group id", required = false, arity = 1) @@ -563,7 +490,7 @@ public class DeleteCommandOptions { @Parameter(names = {"--force"}, description = "Force the deletion of individuals that already belong to families", required = false, help = true, arity = 0) public boolean force = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--individuals"}, description = "Comma separated list of individual ids", required = true, arity = 1) @@ -589,7 +516,7 @@ public class InfoCommandOptions { @Parameter(names = {"--individuals"}, description = "Comma separated list of individual IDs, names or UUIDs up to a maximum of 100", required = true, arity = 1) public String individuals; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--version"}, description = "Comma separated list of individual versions. 'all' to get all the individual versions. Not supported if multiple individual ids are provided", required = false, arity = 1) @@ -621,7 +548,7 @@ public class UpdateCommandOptions { @Parameter(names = {"--individuals"}, description = "Comma separated list of individual ids", required = true, arity = 1) public String individuals; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) @@ -755,7 +682,7 @@ public class UpdateAnnotationSetsAnnotationsCommandOptions { @Parameter(names = {"--individual"}, description = "Individual ID, name or UUID", required = true, arity = 1) public String individual; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--annotation-set"}, description = "AnnotationSet ID to be updated.", required = true, arity = 1) @@ -784,7 +711,7 @@ public class RelativesCommandOptions { @Parameter(names = {"--individual"}, description = "Individual ID, name or UUID", required = true, arity = 1) public String individual; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--degree"}, description = "Pedigree degree", required = false, arity = 1) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/JobsCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/JobsCommandOptions.java index 04b6caae03c..327d9487dfb 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/JobsCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/JobsCommandOptions.java @@ -34,7 +34,6 @@ public class JobsCommandOptions extends CustomJobsCommandOptions { public UpdateAclCommandOptions updateAclCommandOptions; - public AggregationStatsCommandOptions aggregationStatsCommandOptions; public CreateCommandOptions createCommandOptions; public DistinctCommandOptions distinctCommandOptions; public RetryCommandOptions retryCommandOptions; @@ -44,6 +43,7 @@ public class JobsCommandOptions extends CustomJobsCommandOptions { public DeleteCommandOptions deleteCommandOptions; public InfoCommandOptions infoCommandOptions; public UpdateCommandOptions updateCommandOptions; + public KillCommandOptions killCommandOptions; public HeadLogCommandOptions headLogCommandOptions; public TailLogCommandOptions tailLogCommandOptions; @@ -52,7 +52,6 @@ public JobsCommandOptions(CommonCommandOptions commonCommandOptions, JCommander super(commonCommandOptions,jCommander); this.updateAclCommandOptions = new UpdateAclCommandOptions(); - this.aggregationStatsCommandOptions = new AggregationStatsCommandOptions(); this.createCommandOptions = new CreateCommandOptions(); this.distinctCommandOptions = new DistinctCommandOptions(); this.retryCommandOptions = new RetryCommandOptions(); @@ -62,6 +61,7 @@ public JobsCommandOptions(CommonCommandOptions commonCommandOptions, JCommander this.deleteCommandOptions = new DeleteCommandOptions(); this.infoCommandOptions = new InfoCommandOptions(); this.updateCommandOptions = new UpdateCommandOptions(); + this.killCommandOptions = new KillCommandOptions(); this.headLogCommandOptions = new HeadLogCommandOptions(); this.tailLogCommandOptions = new TailLogCommandOptions(); @@ -93,68 +93,6 @@ public class UpdateAclCommandOptions { } - @Parameters(commandNames = {"aggregationstats"}, commandDescription ="Fetch catalog job stats") - public class AggregationStatsCommandOptions { - - @ParametersDelegate - public CommonCommandOptions commonOptions = commonCommandOptions; - - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - - @Parameter(names = {"--tool-id"}, description = "Tool id", required = false, arity = 1) - public String toolId; - - @Parameter(names = {"--tool-scope"}, description = "Tool scope", required = false, arity = 1) - public String toolScope; - - @Parameter(names = {"--tool-type"}, description = "Tool type", required = false, arity = 1) - public String toolType; - - @Parameter(names = {"--tool-resource"}, description = "Tool resource", required = false, arity = 1) - public String toolResource; - - @Parameter(names = {"--user-id"}, description = "User id", required = false, arity = 1) - public String userId; - - @Parameter(names = {"--priority"}, description = "Priority", required = false, arity = 1) - public String priority; - - @Parameter(names = {"--tags"}, description = "Tags", required = false, arity = 1) - public String tags; - - @Parameter(names = {"--executor-id"}, description = "Executor id", required = false, arity = 1) - public String executorId; - - @Parameter(names = {"--executor-framework"}, description = "Executor framework", required = false, arity = 1) - public String executorFramework; - - @Parameter(names = {"--creation-year"}, description = "Creation year", required = false, arity = 1) - public String creationYear; - - @Parameter(names = {"--creation-month"}, description = "Creation month (JANUARY, FEBRUARY...)", required = false, arity = 1) - public String creationMonth; - - @Parameter(names = {"--creation-day"}, description = "Creation day", required = false, arity = 1) - public String creationDay; - - @Parameter(names = {"--creation-day-of-week"}, description = "Creation day of week (MONDAY, TUESDAY...)", required = false, arity = 1) - public String creationDayOfWeek; - - @Parameter(names = {"--status"}, description = "Status", required = false, arity = 1) - public String status; - - @Parameter(names = {"--release"}, description = "Release", required = false, arity = 1) - public String release; - - @Parameter(names = {"--default"}, description = "Calculate default stats", required = false, help = true, arity = 0) - public boolean default_values = false; - - @Parameter(names = {"--field"}, description = "List of fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type;numSamples[0..10]:1", required = false, arity = 1) - public String field; - - } - @Parameters(commandNames = {"create"}, commandDescription ="Register an executed job with POST method") public class CreateCommandOptions { @@ -167,7 +105,7 @@ public class CreateCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--id"}, description = "The body web service id parameter", required = true, arity = 1) @@ -235,7 +173,7 @@ public class DistinctCommandOptions { @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--other-studies"}, description = "Flag indicating the entries being queried can belong to any related study, not just the primary one.", required = false, help = true, arity = 0) @@ -321,7 +259,10 @@ public class RetryCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--job"}, description = "The body web service job parameter", required = false, arity = 1) @@ -356,7 +297,7 @@ public class SearchCommandOptions { @Parameter(names = {"--count"}, description = "Get the total number of results matching the query. Deactivated by default.", required = false, help = true, arity = 0) public boolean count = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--other-studies"}, description = "Flag indicating the entries being queried can belong to any related study, not just the primary one.", required = false, help = true, arity = 0) @@ -438,7 +379,7 @@ public class DeleteCommandOptions { @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--jobs"}, description = "Comma separated list of job ids", required = true, arity = 1) @@ -461,7 +402,7 @@ public class InfoCommandOptions { @Parameter(names = {"--jobs"}, description = "Comma separated list of job IDs or UUIDs up to a maximum of 100", required = true, arity = 1) public String jobs; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--deleted"}, description = "Boolean to retrieve deleted jobs", required = false, help = true, arity = 0) @@ -490,7 +431,7 @@ public class UpdateCommandOptions { @Parameter(names = {"--jobs"}, description = "Comma separated list of job IDs or UUIDs up to a maximum of 100", required = true, arity = 1) public String jobs; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) @@ -510,6 +451,26 @@ public class UpdateCommandOptions { } + @Parameters(commandNames = {"kill"}, commandDescription ="Send a signal to kill a pending or running job") + public class KillCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) + public String jsonFile; + + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) + public Boolean jsonDataModel = false; + + @Parameter(names = {"--job"}, description = "Job ID or UUID", required = true, arity = 1) + public String job; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + public String study; + + } + @Parameters(commandNames = {"log-head"}, commandDescription ="Show the first lines of a log file (up to a limit)") public class HeadLogCommandOptions { @@ -519,7 +480,7 @@ public class HeadLogCommandOptions { @Parameter(names = {"--job"}, description = "Job ID or UUID", required = true, arity = 1) public String job; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--offset"}, description = "Starting byte from which the file will be read", required = false, arity = 1) @@ -542,7 +503,7 @@ public class TailLogCommandOptions { @Parameter(names = {"--job"}, description = "Job ID or UUID", required = true, arity = 1) public String job; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--lines"}, description = "Maximum number of lines to be returned up to a maximum of 1000", required = false, arity = 1) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OperationsVariantStorageCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OperationsVariantStorageCommandOptions.java index 4f81b547328..6f932901d33 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OperationsVariantStorageCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OperationsVariantStorageCommandOptions.java @@ -58,6 +58,7 @@ public class OperationsVariantStorageCommandOptions { public ConfigureVariantSecondarySampleIndexCommandOptions configureVariantSecondarySampleIndexCommandOptions; public SecondaryIndexVariantCommandOptions secondaryIndexVariantCommandOptions; public DeleteVariantSecondaryIndexCommandOptions deleteVariantSecondaryIndexCommandOptions; + public SetupVariantCommandOptions setupVariantCommandOptions; public DeleteVariantStatsCommandOptions deleteVariantStatsCommandOptions; public IndexVariantStatsCommandOptions indexVariantStatsCommandOptions; public DeleteVariantStudyCommandOptions deleteVariantStudyCommandOptions; @@ -92,6 +93,7 @@ public OperationsVariantStorageCommandOptions(CommonCommandOptions commonCommand this.configureVariantSecondarySampleIndexCommandOptions = new ConfigureVariantSecondarySampleIndexCommandOptions(); this.secondaryIndexVariantCommandOptions = new SecondaryIndexVariantCommandOptions(); this.deleteVariantSecondaryIndexCommandOptions = new DeleteVariantSecondaryIndexCommandOptions(); + this.setupVariantCommandOptions = new SetupVariantCommandOptions(); this.deleteVariantStatsCommandOptions = new DeleteVariantStatsCommandOptions(); this.indexVariantStatsCommandOptions = new IndexVariantStatsCommandOptions(); this.deleteVariantStudyCommandOptions = new DeleteVariantStudyCommandOptions(); @@ -110,7 +112,7 @@ public class ConfigureCellbaseCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--project", "-p"}, description = "Project [user@]project where project can be either the ID or the alias", required = false, arity = 1) + @Parameter(names = {"--project", "-p"}, description = "Project [organization@]project where project can be either the ID or the alias", required = false, arity = 1) public String project; @Parameter(names = {"--annotation-update"}, description = "Create and load variant annotations into the database", required = false, help = true, arity = 0) @@ -157,7 +159,16 @@ public class AggregateVariantCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--overwrite"}, description = "Overwrite aggregation for all files and variants. Repeat operation for already processed variants.", required = false, help = true, arity = 0) @@ -186,7 +197,16 @@ public class DeleteVariantAnnotationCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; - @Parameter(names = {"--project", "-p"}, description = "Project [user@]project where project can be either the ID or the alias", required = false, arity = 1) + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + + @Parameter(names = {"--project", "-p"}, description = "Project [organization@]project where project can be either the ID or the alias", required = false, arity = 1) public String project; @Parameter(names = {"--annotation-id"}, description = "Annotation identifier", required = false, arity = 1) @@ -218,10 +238,19 @@ public class IndexVariantAnnotationCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; - @Parameter(names = {"--project", "-p"}, description = "Project [user@]project where project can be either the ID or the alias", required = false, arity = 1) + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + + @Parameter(names = {"--project", "-p"}, description = "Project [organization@]project where project can be either the ID or the alias", required = false, arity = 1) public String project; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--outdir"}, description = "The body web service outdir parameter", required = false, arity = 1) @@ -277,7 +306,16 @@ public class SaveVariantAnnotationCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; - @Parameter(names = {"--project", "-p"}, description = "Project [user@]project where project can be either the ID or the alias", required = false, arity = 1) + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + + @Parameter(names = {"--project", "-p"}, description = "Project [organization@]project where project can be either the ID or the alias", required = false, arity = 1) public String project; @Parameter(names = {"--annotation-id"}, description = "New Variant Annotation identifier", required = false, arity = 1) @@ -297,10 +335,10 @@ public class ConfigureVariantCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--project", "-p"}, description = "Project [user@]project where project can be either the ID or the alias", required = false, arity = 1) + @Parameter(names = {"--project", "-p"}, description = "Project [organization@]project where project can be either the ID or the alias", required = false, arity = 1) public String project; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @DynamicParameter(names = {"--configuration"}, description = "The body web service configuration parameter. Use: --configuration key=value", required = false) @@ -332,15 +370,27 @@ public class DeleteVariantCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; - @Parameter(names = {"--file"}, description = "The body web service file parameter", required = false, arity = 1) + @Parameter(names = {"--file"}, description = "List of file ids to delete. Use 'all' to remove the whole study", required = false, arity = 1) public String file; - @Parameter(names = {"--resume"}, description = "The body web service resume parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--resume"}, description = "Resume failed delete operation.", required = false, help = true, arity = 0) public boolean resume = false; + @Parameter(names = {"--force"}, description = "Force delete operation. This would allow deleting partially loaded files.", required = false, help = true, arity = 0) + public boolean force = false; + } @Parameters(commandNames = {"variant-family-aggregate"}, commandDescription ="Find variants where not all the samples are present, and fill the empty values.") @@ -367,7 +417,16 @@ public class AggregateVariantFamilyCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--samples"}, description = "Samples within the same study to aggregate", required = false, arity = 1) @@ -405,7 +464,16 @@ public class IndexVariantFamilyCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--family"}, description = "The body web service family parameter", required = false, arity = 1) @@ -446,7 +514,16 @@ public class IndexVariantCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--file"}, description = "List of files to be indexed.", required = false, arity = 1) @@ -565,7 +642,16 @@ public class LauncherVariantIndexCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--name", "-n"}, description = "The body web service name parameter", required = false, arity = 1) @@ -699,6 +785,15 @@ public class RunVariantJulieCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--project", "-p"}, description = "project", required = false, arity = 1) public String project; @@ -737,6 +832,15 @@ public class RepairVariantMetadataCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--studies"}, description = "The body web service studies parameter", required = false, arity = 1) public String studies; @@ -769,7 +873,16 @@ public class SynchronizeVariantMetadataCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--body_study"}, description = "The body web service study parameter", required = false, arity = 1) @@ -804,6 +917,15 @@ public class PruneVariantCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--project", "-p"}, description = "The body web service project parameter", required = false, arity = 1) public String project; @@ -839,7 +961,16 @@ public class DeleteVariantSampleCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--sample"}, description = "The body web service sample parameter", required = false, arity = 1) @@ -877,7 +1008,16 @@ public class IndexVariantSampleCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--sample"}, description = "The body web service sample parameter", required = false, arity = 1) @@ -909,7 +1049,7 @@ public class VariantSampleIndexConfigureCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--skip-rebuild"}, description = "Skip sample index re-build", required = false, help = true, arity = 0) @@ -935,7 +1075,16 @@ public class DeleteVariantScoreCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--name", "-n"}, description = "Unique name of the score within the study", required = false, arity = 1) @@ -973,7 +1122,16 @@ public class IndexVariantScoreCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--score-name"}, description = "The body web service scoreName parameter", required = false, arity = 1) @@ -1020,10 +1178,19 @@ public class VariantSecondaryAnnotationIndexCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; - @Parameter(names = {"--project", "-p"}, description = "Project [user@]project where project can be either the ID or the alias", required = false, arity = 1) + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + + @Parameter(names = {"--project", "-p"}, description = "Project [organization@]project where project can be either the ID or the alias", required = false, arity = 1) public String project; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--region"}, description = "The body web service region parameter", required = false, arity = 1) @@ -1061,7 +1228,16 @@ public class VariantSecondarySampleIndexCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--sample"}, description = "The body web service sample parameter", required = false, arity = 1) @@ -1093,7 +1269,7 @@ public class ConfigureVariantSecondarySampleIndexCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--skip-rebuild"}, description = "Skip sample index re-build", required = false, help = true, arity = 0) @@ -1125,10 +1301,19 @@ public class SecondaryIndexVariantCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; - @Parameter(names = {"--project", "-p"}, description = "Project [user@]project where project can be either the ID or the alias", required = false, arity = 1) + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + + @Parameter(names = {"--project", "-p"}, description = "Project [organization@]project where project can be either the ID or the alias", required = false, arity = 1) public String project; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--region"}, description = "The body web service region parameter", required = false, arity = 1) @@ -1160,7 +1345,16 @@ public class DeleteVariantSecondaryIndexCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--samples"}, description = "Samples to remove. Needs to provide all the samples in the secondary index.", required = false, arity = 1) @@ -1168,6 +1362,47 @@ public class DeleteVariantSecondaryIndexCommandOptions { } + @Parameters(commandNames = {"variant-setup"}, commandDescription ="Execute Variant Setup to allow using the variant engine. This setup is necessary before starting any variant operation.") + public class SetupVariantCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) + public String jsonFile; + + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) + public Boolean jsonDataModel = false; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + public String study; + + @Parameter(names = {"--expected-samples"}, description = "Expected number of samples that will be loaded. Used to infer some parameters. This number is only used as a hint. If the real number of samples is different, if it grows beyond expectation, or if , the loader should be able to handle it.", required = false, arity = 1) + public Integer expectedSamples; + + @Parameter(names = {"--expected-files"}, description = "Expected number of files that will be loaded. Used to infer some parameters. This number is only used as a hint. If the real number of files is different, the loader should be able to handle it.", required = false, arity = 1) + public Integer expectedFiles; + + @Parameter(names = {"--file-type"}, description = "Main type of the files that will be loaded. If the dataset contains multiple types of files, provide the one that matches most of the files.", required = false, arity = 1) + public String fileType; + + @Parameter(names = {"--average-file-size"}, description = "Average size of the files that will be loaded. This number is only used as a hint. If the real size of the files is different, the loader should be able to handle it. Accepts units. e.g. 435MB, 2GB, 86KB. If not provided, the value will be inferred from the file type.", required = false, arity = 1) + public String averageFileSize; + + @Parameter(names = {"--variants-per-sample"}, description = "Number of variants per sample (non hom_ref variants). This number is only used as a hint. If the real number of variants per sample is different, the loader should be able to handle it. If not provided, the value will be inferred from the file type.", required = false, arity = 1) + public Integer variantsPerSample; + + @Parameter(names = {"--average-samples-per-file"}, description = "Average number of samples per file. This number is only used as a hint. If the real number of samples per file is different, the loader should be able to handle it. If not provided, the value will be inferred from the expectedSamples and expectedFiles and dataDistribution.", required = false, arity = 1) + public Float averageSamplesPerFile; + + @Parameter(names = {"--data-distribution"}, description = "Data distribution of the files. This parameter is used to infer the number of samples per file.", required = false, arity = 1) + public String dataDistribution; + + @Parameter(names = {"--normalize-extensions"}, description = "List of normalization extensions that will be used to normalize the files.", required = false, arity = 1) + public String normalizeExtensions; + + } + @Parameters(commandNames = {"variant-stats-delete"}, commandDescription ="Deletes the VariantStats of a cohort/s from the database") public class DeleteVariantStatsCommandOptions { @@ -1180,7 +1415,7 @@ public class DeleteVariantStatsCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) @@ -1195,6 +1430,15 @@ public class DeleteVariantStatsCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--cohort"}, description = "The body web service cohort parameter", required = false, arity = 1) public String cohort; @@ -1215,7 +1459,7 @@ public class IndexVariantStatsCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) @@ -1230,6 +1474,15 @@ public class IndexVariantStatsCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--cohort"}, description = "The body web service cohort parameter", required = false, arity = 1) public String cohort; @@ -1274,7 +1527,16 @@ public class DeleteVariantStudyCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--resume"}, description = "The body web service resume parameter", required = false, help = true, arity = 0) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OrganizationsCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OrganizationsCommandOptions.java new file mode 100644 index 00000000000..0f9f2a18c49 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OrganizationsCommandOptions.java @@ -0,0 +1,421 @@ +package org.opencb.opencga.app.cli.main.options; + +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import com.beust.jcommander.DynamicParameter; +import com.beust.jcommander.ParametersDelegate; + +import java.util.HashMap; +import java.util.Map; +import java.util.List; + +import static org.opencb.opencga.app.cli.GeneralCliOptions.*; + + +/* +* WARNING: AUTOGENERATED CODE +* +* This code was generated by a tool. +* +* Manual changes to this file may cause unexpected behavior in your application. +* Manual changes to this file will be overwritten if the code is regenerated. +* +*/ + +/** + * This class contains methods for the Organizations command line. + * PATH: /{apiVersion}/organizations + */ +@Parameters(commandNames = {"organizations"}, commandDescription = "Organizations commands") +public class OrganizationsCommandOptions { + + public JCommander jCommander; + public CommonCommandOptions commonCommandOptions; + + public CreateCommandOptions createCommandOptions; + public CreateNotesCommandOptions createNotesCommandOptions; + public SearchNotesCommandOptions searchNotesCommandOptions; + public DeleteNotesCommandOptions deleteNotesCommandOptions; + public UpdateNotesCommandOptions updateNotesCommandOptions; + public UserUpdateStatusCommandOptions userUpdateStatusCommandOptions; + public UpdateUserCommandOptions updateUserCommandOptions; + public UpdateConfigurationCommandOptions updateConfigurationCommandOptions; + public InfoCommandOptions infoCommandOptions; + public UpdateCommandOptions updateCommandOptions; + + + public OrganizationsCommandOptions(CommonCommandOptions commonCommandOptions, JCommander jCommander) { + + this.jCommander = jCommander; + this.commonCommandOptions = commonCommandOptions; + this.createCommandOptions = new CreateCommandOptions(); + this.createNotesCommandOptions = new CreateNotesCommandOptions(); + this.searchNotesCommandOptions = new SearchNotesCommandOptions(); + this.deleteNotesCommandOptions = new DeleteNotesCommandOptions(); + this.updateNotesCommandOptions = new UpdateNotesCommandOptions(); + this.userUpdateStatusCommandOptions = new UserUpdateStatusCommandOptions(); + this.updateUserCommandOptions = new UpdateUserCommandOptions(); + this.updateConfigurationCommandOptions = new UpdateConfigurationCommandOptions(); + this.infoCommandOptions = new InfoCommandOptions(); + this.updateCommandOptions = new UpdateCommandOptions(); + + } + + @Parameters(commandNames = {"create"}, commandDescription ="Create a new organization") + public class CreateCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) + public String jsonFile; + + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) + public Boolean jsonDataModel = false; + + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) + public String include; + + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) + public String exclude; + + @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) + public boolean includeResult = false; + + @Parameter(names = {"--id"}, description = "Organization unique identifier.", required = true, arity = 1) + public String id; + + @Parameter(names = {"--name", "-n"}, description = "Organization name.", required = false, arity = 1) + public String name; + + @Parameter(names = {"--creation-date", "--cd"}, description = "Autogenerated date following the format YYYYMMDDhhmmss containing the date when the entry was first registered.", required = false, arity = 1) + public String creationDate; + + @Parameter(names = {"--modification-date", "--md"}, description = "Autogenerated date following the format YYYYMMDDhhmmss containing the date when the entry was last modified.", required = false, arity = 1) + public String modificationDate; + + @Parameter(names = {"--configuration-default-user-expiration-date"}, description = "The body web service defaultUserExpirationDate parameter", required = false, arity = 1) + public String configurationDefaultUserExpirationDate; + + @DynamicParameter(names = {"--attributes"}, description = "You can use this field to store any other information, keep in mind this is not indexed so you cannot search by attributes.. Use: --attributes key=value", required = false) + public java.util.Map attributes = new HashMap<>(); //Dynamic parameters must be initialized; + + } + + @Parameters(commandNames = {"notes-create"}, commandDescription ="Create a new note") + public class CreateNotesCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) + public String jsonFile; + + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) + public Boolean jsonDataModel = false; + + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) + public String include; + + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) + public String exclude; + + @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) + public boolean includeResult = false; + + @Parameter(names = {"--id"}, description = "The body web service id parameter", required = false, arity = 1) + public String id; + + @Parameter(names = {"--tags"}, description = "The body web service tags parameter", required = false, arity = 1) + public String tags; + + @Parameter(names = {"--visibility"}, description = "Enum param allowed values: PUBLIC, PRIVATE", required = false, arity = 1) + public String visibility; + + @Parameter(names = {"--value-type"}, description = "Enum param allowed values: OBJECT, ARRAY, STRING, INTEGER, DOUBLE", required = false, arity = 1) + public String valueType; + + } + + @Parameters(commandNames = {"notes-search"}, commandDescription ="Search for notes of scope ORGANIZATION") + public class SearchNotesCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) + public String include; + + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) + public String exclude; + + @Parameter(names = {"--creation-date", "--cd"}, description = "Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805", required = false, arity = 1) + public String creationDate; + + @Parameter(names = {"--modification-date", "--md"}, description = "Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805", required = false, arity = 1) + public String modificationDate; + + @Parameter(names = {"--id"}, description = "Note unique identifier.", required = false, arity = 1) + public String id; + + @Parameter(names = {"--scope"}, description = "Scope of the Note.", required = false, arity = 1) + public String scope; + + @Parameter(names = {"--visibility"}, description = "Visibility of the Note.", required = false, arity = 1) + public String visibility; + + @Parameter(names = {"--uuid"}, description = "Unique 32-character identifier assigned automatically by OpenCGA.", required = false, arity = 1) + public String uuid; + + @Parameter(names = {"--user-id"}, description = "User that wrote that Note.", required = false, arity = 1) + public String userId; + + @Parameter(names = {"--tags"}, description = "Note tags.", required = false, arity = 1) + public String tags; + + @Parameter(names = {"--version"}, description = "Autoincremental version assigned to the registered entry. By default, updates does not create new versions. To enable versioning, users must set the `incVersion` flag from the /update web service when updating the document.", required = false, arity = 1) + public String version; + + } + + @Parameters(commandNames = {"notes-delete"}, commandDescription ="Delete note") + public class DeleteNotesCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--id"}, description = "Note unique identifier.", required = true, arity = 1) + public String id; + + @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) + public boolean includeResult = false; + + } + + @Parameters(commandNames = {"notes-update"}, commandDescription ="Update a note") + public class UpdateNotesCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) + public String jsonFile; + + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) + public Boolean jsonDataModel = false; + + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) + public String include; + + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) + public String exclude; + + @Parameter(names = {"--id"}, description = "Note unique identifier.", required = true, arity = 1) + public String id; + + @Parameter(names = {"--tags-action"}, description = "Action to be performed if the array of tags is being updated.", required = false, arity = 1) + public String tagsAction = "ADD"; + + @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) + public boolean includeResult = false; + + @Parameter(names = {"--tags"}, description = "The body web service tags parameter", required = false, arity = 1) + public String tags; + + @Parameter(names = {"--visibility"}, description = "Enum param allowed values: PUBLIC, PRIVATE", required = false, arity = 1) + public String visibility; + + } + + @Parameters(commandNames = {"update-status-user"}, commandDescription ="Update the user status") + public class UserUpdateStatusCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) + public String jsonFile; + + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) + public Boolean jsonDataModel = false; + + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) + public String include; + + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) + public String exclude; + + @Parameter(names = {"--user", "-u"}, description = "User ID", required = true, arity = 1) + public String user; + + @Parameter(names = {"--organization"}, description = "Organization id", required = false, arity = 1) + public String organization; + + @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) + public boolean includeResult = false; + + @Parameter(names = {"--status"}, description = "The body web service status parameter", required = false, arity = 1) + public String status; + + } + + @Parameters(commandNames = {"user-update"}, commandDescription ="Update the user information") + public class UpdateUserCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) + public String jsonFile; + + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) + public Boolean jsonDataModel = false; + + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) + public String include; + + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) + public String exclude; + + @Parameter(names = {"--user", "-u"}, description = "User ID", required = true, arity = 1) + public String user; + + @Parameter(names = {"--organization"}, description = "Organization id", required = false, arity = 1) + public String organization; + + @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) + public boolean includeResult = false; + + @Parameter(names = {"--name", "-n"}, description = "The body web service name parameter", required = false, arity = 1) + public String name; + + @Parameter(names = {"--email"}, description = "The body web service email parameter", required = false, arity = 1) + public String email; + + @Parameter(names = {"--quota-disk-usage"}, description = "The body web service diskUsage parameter", required = false, arity = 1) + public Long quotaDiskUsage; + + @Parameter(names = {"--quota-cpu-usage"}, description = "The body web service cpuUsage parameter", required = false, arity = 1) + public Integer quotaCpuUsage; + + @Parameter(names = {"--quota-max-disk"}, description = "The body web service maxDisk parameter", required = false, arity = 1) + public Long quotaMaxDisk; + + @Parameter(names = {"--quota-max-cpu"}, description = "The body web service maxCpu parameter", required = false, arity = 1) + public Integer quotaMaxCpu; + + @DynamicParameter(names = {"--attributes"}, description = "The body web service attributes parameter. Use: --attributes key=value", required = false) + public java.util.Map attributes = new HashMap<>(); //Dynamic parameters must be initialized; + + } + + @Parameters(commandNames = {"configuration-update"}, commandDescription ="Update the Organization configuration attributes") + public class UpdateConfigurationCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) + public String jsonFile; + + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) + public Boolean jsonDataModel = false; + + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) + public String include; + + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) + public String exclude; + + @Parameter(names = {"--organization"}, description = "Organization id", required = true, arity = 1) + public String organization; + + @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) + public boolean includeResult = false; + + @Parameter(names = {"--authentication-origins-action"}, description = "Action to be performed if the array of authenticationOrigins is being updated.", required = false, arity = 1) + public String authenticationOriginsAction = "ADD"; + + @Parameter(names = {"--default-user-expiration-date"}, description = "The body web service defaultUserExpirationDate parameter", required = false, arity = 1) + public String defaultUserExpirationDate; + + @Parameter(names = {"--optimizations-simplify-permissions"}, description = "The body web service simplifyPermissions parameter", required = false, help = true, arity = 0) + public boolean optimizationsSimplifyPermissions = false; + + @Parameter(names = {"--token-algorithm"}, description = "The body web service algorithm parameter", required = false, arity = 1) + public String tokenAlgorithm; + + @Parameter(names = {"--token-secret-key"}, description = "The body web service secretKey parameter", required = false, arity = 1) + public String tokenSecretKey; + + @Parameter(names = {"--token-expiration"}, description = "The body web service expiration parameter", required = false, arity = 1) + public Long tokenExpiration; + + } + + @Parameters(commandNames = {"info"}, commandDescription ="Return the organization information") + public class InfoCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) + public String include; + + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) + public String exclude; + + @Parameter(names = {"--organization"}, description = "Organization id", required = true, arity = 1) + public String organization; + + } + + @Parameters(commandNames = {"update"}, commandDescription ="Update some organization attributes") + public class UpdateCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) + public String jsonFile; + + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) + public Boolean jsonDataModel = false; + + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) + public String include; + + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) + public String exclude; + + @Parameter(names = {"--organization"}, description = "Organization id", required = true, arity = 1) + public String organization; + + @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) + public boolean includeResult = false; + + @Parameter(names = {"--admins-action"}, description = "Action to be performed if the array of admins is being updated.", required = false, arity = 1) + public String adminsAction = "ADD"; + + @Parameter(names = {"--name", "-n"}, description = "Organization name.", required = false, arity = 1) + public String name; + + @Parameter(names = {"--owner"}, description = "Owner of the organization.", required = false, arity = 1) + public String owner; + + @Parameter(names = {"--admins"}, description = "Administrative users of the organization.", required = false, arity = 1) + public String admins; + + @Parameter(names = {"--creation-date", "--cd"}, description = "Autogenerated date following the format YYYYMMDDhhmmss containing the date when the entry was first registered.", required = false, arity = 1) + public String creationDate; + + @Parameter(names = {"--modification-date", "--md"}, description = "Autogenerated date following the format YYYYMMDDhhmmss containing the date when the entry was last modified.", required = false, arity = 1) + public String modificationDate; + + @DynamicParameter(names = {"--attributes"}, description = "You can use this field to store any other information, keep in mind this is not indexed so you cannot search by attributes.. Use: --attributes key=value", required = false) + public java.util.Map attributes = new HashMap<>(); //Dynamic parameters must be initialized; + + } + +} \ No newline at end of file diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/ProjectsCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/ProjectsCommandOptions.java index a2eca8cb958..d49c1e59004 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/ProjectsCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/ProjectsCommandOptions.java @@ -35,7 +35,6 @@ public class ProjectsCommandOptions { public CreateCommandOptions createCommandOptions; public SearchCommandOptions searchCommandOptions; - public AggregationStatsCommandOptions aggregationStatsCommandOptions; public InfoCommandOptions infoCommandOptions; public IncReleaseCommandOptions incReleaseCommandOptions; public StudiesCommandOptions studiesCommandOptions; @@ -48,7 +47,6 @@ public ProjectsCommandOptions(CommonCommandOptions commonCommandOptions, JComman this.commonCommandOptions = commonCommandOptions; this.createCommandOptions = new CreateCommandOptions(); this.searchCommandOptions = new SearchCommandOptions(); - this.aggregationStatsCommandOptions = new AggregationStatsCommandOptions(); this.infoCommandOptions = new InfoCommandOptions(); this.incReleaseCommandOptions = new IncReleaseCommandOptions(); this.studiesCommandOptions = new StudiesCommandOptions(); @@ -136,10 +134,10 @@ public class SearchCommandOptions { @Parameter(names = {"--skip"}, description = "Number of results to skip", required = false, arity = 1) public Integer skip; - @Parameter(names = {"--owner"}, description = "Owner of the project", required = false, arity = 1) - public String owner; + @Parameter(names = {"--organization"}, description = "Organization id", required = false, arity = 1) + public String organization; - @Parameter(names = {"--id"}, description = "Project [user@]project where project can be either the ID or the alias", required = false, arity = 1) + @Parameter(names = {"--id"}, description = "Project [organization@]project where project can be either the ID or the alias", required = false, arity = 1) public String id; @Parameter(names = {"--name", "-n"}, description = "Project name", required = false, arity = 1) @@ -148,9 +146,6 @@ public class SearchCommandOptions { @Parameter(names = {"--fqn"}, description = "Project fqn", required = false, arity = 1) public String fqn; - @Parameter(names = {"--organization"}, description = "Project organization", required = false, arity = 1) - public String organization; - @Parameter(names = {"--description"}, description = "Project description", required = false, arity = 1) public String description; @@ -171,38 +166,6 @@ public class SearchCommandOptions { } - @Parameters(commandNames = {"aggregationstats"}, commandDescription ="Fetch catalog project stats") - public class AggregationStatsCommandOptions { - - @ParametersDelegate - public CommonCommandOptions commonOptions = commonCommandOptions; - - @Parameter(names = {"--projects"}, description = "Comma separated list of projects [user@]project up to a maximum of 100", required = true, arity = 1) - public String projects; - - @Parameter(names = {"--default"}, description = "Calculate default stats", required = false, arity = 1) - public Boolean default_values = true; - - @Parameter(names = {"--file-fields"}, description = "List of file fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type", required = false, arity = 1) - public String fileFields; - - @Parameter(names = {"--individual-fields"}, description = "List of individual fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type", required = false, arity = 1) - public String individualFields; - - @Parameter(names = {"--family-fields"}, description = "List of family fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type", required = false, arity = 1) - public String familyFields; - - @Parameter(names = {"--sample-fields"}, description = "List of sample fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type", required = false, arity = 1) - public String sampleFields; - - @Parameter(names = {"--cohort-fields"}, description = "List of cohort fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type", required = false, arity = 1) - public String cohortFields; - - @Parameter(names = {"--job-fields"}, description = "List of job fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type", required = false, arity = 1) - public String jobFields; - - } - @Parameters(commandNames = {"info"}, commandDescription ="Fetch project information") public class InfoCommandOptions { @@ -215,7 +178,7 @@ public class InfoCommandOptions { @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) public String exclude; - @Parameter(names = {"--projects"}, description = "Comma separated list of projects [user@]project up to a maximum of 100", required = true, arity = 1) + @Parameter(names = {"--projects"}, description = "Comma separated list of projects [organization@]project up to a maximum of 100", required = true, arity = 1) public String projects; } @@ -232,7 +195,7 @@ public class IncReleaseCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--project", "-p"}, description = "Project [user@]project where project can be either the ID or the alias", required = true, arity = 1) + @Parameter(names = {"--project", "-p"}, description = "Project [organization@]project where project can be either the ID or the alias", required = true, arity = 1) public String project; } @@ -255,7 +218,7 @@ public class StudiesCommandOptions { @Parameter(names = {"--skip"}, description = "Number of results to skip", required = false, arity = 1) public Integer skip; - @Parameter(names = {"--project", "-p"}, description = "Project [user@]project where project can be either the ID or the alias", required = true, arity = 1) + @Parameter(names = {"--project", "-p"}, description = "Project [organization@]project where project can be either the ID or the alias", required = true, arity = 1) public String project; } @@ -278,7 +241,7 @@ public class UpdateCommandOptions { @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) public String exclude; - @Parameter(names = {"--project", "-p"}, description = "Project [user@]project where project can be either the ID or the alias", required = true, arity = 1) + @Parameter(names = {"--project", "-p"}, description = "Project [organization@]project where project can be either the ID or the alias", required = true, arity = 1) public String project; @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/SamplesCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/SamplesCommandOptions.java index 8cfc8fed346..885e3cabd52 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/SamplesCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/SamplesCommandOptions.java @@ -34,7 +34,6 @@ public class SamplesCommandOptions { public CommonCommandOptions commonCommandOptions; public UpdateAclCommandOptions updateAclCommandOptions; - public AggregationStatsCommandOptions aggregationStatsCommandOptions; public LoadAnnotationSetsCommandOptions loadAnnotationSetsCommandOptions; public CreateCommandOptions createCommandOptions; public DistinctCommandOptions distinctCommandOptions; @@ -52,7 +51,6 @@ public SamplesCommandOptions(CommonCommandOptions commonCommandOptions, JCommand this.jCommander = jCommander; this.commonCommandOptions = commonCommandOptions; this.updateAclCommandOptions = new UpdateAclCommandOptions(); - this.aggregationStatsCommandOptions = new AggregationStatsCommandOptions(); this.loadAnnotationSetsCommandOptions = new LoadAnnotationSetsCommandOptions(); this.createCommandOptions = new CreateCommandOptions(); this.distinctCommandOptions = new DistinctCommandOptions(); @@ -78,7 +76,7 @@ public class UpdateAclCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--members"}, description = "Comma separated list of user or group ids", required = true, arity = 1) @@ -107,59 +105,6 @@ public class UpdateAclCommandOptions { } - @Parameters(commandNames = {"aggregationstats"}, commandDescription ="Fetch catalog sample stats") - public class AggregationStatsCommandOptions { - - @ParametersDelegate - public CommonCommandOptions commonOptions = commonCommandOptions; - - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) - public String study; - - @Parameter(names = {"--source"}, description = "Source", required = false, arity = 1) - public String source; - - @Parameter(names = {"--creation-year"}, description = "Creation year", required = false, arity = 1) - public String creationYear; - - @Parameter(names = {"--creation-month"}, description = "Creation month (JANUARY, FEBRUARY...)", required = false, arity = 1) - public String creationMonth; - - @Parameter(names = {"--creation-day"}, description = "Creation day", required = false, arity = 1) - public String creationDay; - - @Parameter(names = {"--creation-day-of-week"}, description = "Creation day of week (MONDAY, TUESDAY...)", required = false, arity = 1) - public String creationDayOfWeek; - - @Parameter(names = {"--status"}, description = "Status", required = false, arity = 1) - public String status; - - @Parameter(names = {"--type"}, description = "Type", required = false, arity = 1) - public String type; - - @Parameter(names = {"--phenotypes"}, description = "Phenotypes", required = false, arity = 1) - public String phenotypes; - - @Parameter(names = {"--release"}, description = "Release", required = false, arity = 1) - public String release; - - @Parameter(names = {"--version"}, description = "Version", required = false, arity = 1) - public String version; - - @Parameter(names = {"--somatic"}, description = "Somatic", required = false, arity = 1) - public Boolean somatic; - - @Parameter(names = {"--annotation"}, description = "Annotation filters. Example: age>30;gender=FEMALE. For more information, please visit http://docs.opencb.org/display/opencga/AnnotationSets+1.4.0", required = false, arity = 1) - public String annotation; - - @Parameter(names = {"--default"}, description = "Calculate default stats", required = false, help = true, arity = 0) - public boolean default_values = false; - - @Parameter(names = {"--field"}, description = "List of fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type;numSamples[0..10]:1", required = false, arity = 1) - public String field; - - } - @Parameters(commandNames = {"annotation-sets-load"}, commandDescription ="Load annotation sets from a TSV file") public class LoadAnnotationSetsCommandOptions { @@ -172,7 +117,7 @@ public class LoadAnnotationSetsCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--variable-set-id"}, description = "Variable set ID or name", required = true, arity = 1) @@ -210,7 +155,7 @@ public class CreateCommandOptions { @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) public String exclude; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) @@ -302,7 +247,7 @@ public class DistinctCommandOptions { @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--id"}, description = "Comma separated list sample IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = false, arity = 1) @@ -427,7 +372,7 @@ public class LoadCommandOptions { @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--file"}, description = "file", required = true, arity = 1) @@ -465,7 +410,7 @@ public class SearchCommandOptions { @Parameter(names = {"--flatten-annotations"}, description = "Flatten the annotations?", required = false, help = true, arity = 0) public boolean flattenAnnotations = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--id"}, description = "Comma separated list sample IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = false, arity = 1) @@ -590,7 +535,7 @@ public class AclCommandOptions { @Parameter(names = {"--samples"}, description = "Comma separated list sample IDs or UUIDs up to a maximum of 100", required = true, arity = 1) public String samples; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--member"}, description = "User or group id", required = false, arity = 1) @@ -616,7 +561,7 @@ public class DeleteCommandOptions { @Parameter(names = {"--delete-empty-cohorts"}, description = "Boolean indicating if the cohorts associated only to the sample to be deleted should be also deleted.", required = false, help = true, arity = 0) public boolean deleteEmptyCohorts = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--samples"}, description = "Comma separated list sample IDs or UUIDs up to a maximum of 100", required = true, arity = 1) @@ -645,7 +590,7 @@ public class InfoCommandOptions { @Parameter(names = {"--samples"}, description = "Comma separated list sample IDs or UUIDs up to a maximum of 100", required = true, arity = 1) public String samples; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--version"}, description = "Comma separated list of sample versions. 'all' to get all the sample versions. Not supported if multiple sample ids are provided", required = false, arity = 1) @@ -677,7 +622,7 @@ public class UpdateCommandOptions { @Parameter(names = {"--samples"}, description = "Comma separated list sample IDs or UUIDs up to a maximum of 100", required = true, arity = 1) public String samples; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) @@ -781,7 +726,7 @@ public class UpdateAnnotationSetsAnnotationsCommandOptions { @Parameter(names = {"--sample"}, description = "Sample ID", required = true, arity = 1) public String sample; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @Parameter(names = {"--annotation-set"}, description = "AnnotationSet ID to be updated.", required = true, arity = 1) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/StudiesCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/StudiesCommandOptions.java index b9b4695cb12..d75022de0b9 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/StudiesCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/StudiesCommandOptions.java @@ -37,12 +37,15 @@ public class StudiesCommandOptions extends CustomStudiesCommandOptions { public CreateCommandOptions createCommandOptions; public SearchCommandOptions searchCommandOptions; public AclCommandOptions aclCommandOptions; - public AggregationStatsCommandOptions aggregationStatsCommandOptions; public InfoCommandOptions infoCommandOptions; public SearchAuditCommandOptions searchAuditCommandOptions; public GroupsCommandOptions groupsCommandOptions; public UpdateGroupsCommandOptions updateGroupsCommandOptions; public UpdateGroupsUsersCommandOptions updateGroupsUsersCommandOptions; + public CreateNotesCommandOptions createNotesCommandOptions; + public SearchNotesCommandOptions searchNotesCommandOptions; + public DeleteNotesCommandOptions deleteNotesCommandOptions; + public UpdateNotesCommandOptions updateNotesCommandOptions; public PermissionRulesCommandOptions permissionRulesCommandOptions; public UpdatePermissionRulesCommandOptions updatePermissionRulesCommandOptions; public RunTemplatesCommandOptions runTemplatesCommandOptions; @@ -61,12 +64,15 @@ public StudiesCommandOptions(CommonCommandOptions commonCommandOptions, JCommand this.createCommandOptions = new CreateCommandOptions(); this.searchCommandOptions = new SearchCommandOptions(); this.aclCommandOptions = new AclCommandOptions(); - this.aggregationStatsCommandOptions = new AggregationStatsCommandOptions(); this.infoCommandOptions = new InfoCommandOptions(); this.searchAuditCommandOptions = new SearchAuditCommandOptions(); this.groupsCommandOptions = new GroupsCommandOptions(); this.updateGroupsCommandOptions = new UpdateGroupsCommandOptions(); this.updateGroupsUsersCommandOptions = new UpdateGroupsUsersCommandOptions(); + this.createNotesCommandOptions = new CreateNotesCommandOptions(); + this.searchNotesCommandOptions = new SearchNotesCommandOptions(); + this.deleteNotesCommandOptions = new DeleteNotesCommandOptions(); + this.updateNotesCommandOptions = new UpdateNotesCommandOptions(); this.permissionRulesCommandOptions = new PermissionRulesCommandOptions(); this.updatePermissionRulesCommandOptions = new UpdatePermissionRulesCommandOptions(); this.runTemplatesCommandOptions = new RunTemplatesCommandOptions(); @@ -126,7 +132,7 @@ public class CreateCommandOptions { @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) public String exclude; - @Parameter(names = {"--project", "-p"}, description = "Project [user@]project where project can be either the ID or the alias", required = false, arity = 1) + @Parameter(names = {"--project", "-p"}, description = "Project [organization@]project where project can be either the ID or the alias", required = false, arity = 1) public String project; @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) @@ -191,7 +197,7 @@ public class SearchCommandOptions { @Parameter(names = {"--count"}, description = "Get the total number of results matching the query. Deactivated by default.", required = false, help = true, arity = 0) public boolean count = false; - @Parameter(names = {"--project", "-p"}, description = "Project [user@]project where project can be either the ID or the alias", required = true, arity = 1) + @Parameter(names = {"--project", "-p"}, description = "Project [organization@]project where project can be either the ID or the alias", required = true, arity = 1) public String project; @Parameter(names = {"--name", "-n"}, description = "Study name", required = false, arity = 1) @@ -232,7 +238,7 @@ public class AclCommandOptions { @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - @Parameter(names = {"--studies"}, description = "Comma separated list of Studies [[user@]project:]study where study and project can be either the ID or UUID up to a maximum of 100", required = true, arity = 1) + @Parameter(names = {"--studies"}, description = "Comma separated list of Studies [[organization@]project:]study where study and project can be either the ID or UUID up to a maximum of 100", required = true, arity = 1) public String studies; @Parameter(names = {"--member"}, description = "User or group id", required = false, arity = 1) @@ -243,38 +249,6 @@ public class AclCommandOptions { } - @Parameters(commandNames = {"aggregationstats"}, commandDescription ="Fetch catalog study stats") - public class AggregationStatsCommandOptions { - - @ParametersDelegate - public CommonCommandOptions commonOptions = commonCommandOptions; - - @Parameter(names = {"--studies"}, description = "Comma separated list of studies [[user@]project:]study up to a maximum of 100", required = true, arity = 1) - public String studies; - - @Parameter(names = {"--default"}, description = "Calculate default stats", required = false, arity = 1) - public Boolean default_values = true; - - @Parameter(names = {"--file-fields"}, description = "List of file fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type", required = false, arity = 1) - public String fileFields; - - @Parameter(names = {"--individual-fields"}, description = "List of individual fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type", required = false, arity = 1) - public String individualFields; - - @Parameter(names = {"--family-fields"}, description = "List of family fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type", required = false, arity = 1) - public String familyFields; - - @Parameter(names = {"--sample-fields"}, description = "List of sample fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type", required = false, arity = 1) - public String sampleFields; - - @Parameter(names = {"--cohort-fields"}, description = "List of cohort fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type", required = false, arity = 1) - public String cohortFields; - - @Parameter(names = {"--job-fields"}, description = "List of job fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type", required = false, arity = 1) - public String jobFields; - - } - @Parameters(commandNames = {"info"}, commandDescription ="Fetch study information") public class InfoCommandOptions { @@ -287,7 +261,7 @@ public class InfoCommandOptions { @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) public String exclude; - @Parameter(names = {"--studies"}, description = "Comma separated list of Studies [[user@]project:]study where study and project can be either the ID or UUID up to a maximum of 100", required = true, arity = 1) + @Parameter(names = {"--studies"}, description = "Comma separated list of Studies [[organization@]project:]study where study and project can be either the ID or UUID up to a maximum of 100", required = true, arity = 1) public String studies; } @@ -348,7 +322,7 @@ public class GroupsCommandOptions { @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = true, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = true, arity = 1) public String study; @Parameter(names = {"--id"}, description = "Group id. If provided, it will only fetch information for the provided group.", required = false, arity = 1) @@ -371,7 +345,7 @@ public class UpdateGroupsCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = true, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = true, arity = 1) public String study; @Parameter(names = {"--action"}, description = "Action to be performed: ADD or REMOVE a group", required = false, arity = 1) @@ -397,7 +371,7 @@ public class UpdateGroupsUsersCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = true, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = true, arity = 1) public String study; @Parameter(names = {"--group"}, description = "Group name", required = true, arity = 1) @@ -411,13 +385,147 @@ public class UpdateGroupsUsersCommandOptions { } + @Parameters(commandNames = {"notes-create"}, commandDescription ="Create a new note") + public class CreateNotesCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) + public String jsonFile; + + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) + public Boolean jsonDataModel = false; + + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) + public String include; + + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) + public String exclude; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = true, arity = 1) + public String study; + + @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) + public boolean includeResult = false; + + @Parameter(names = {"--id"}, description = "The body web service id parameter", required = false, arity = 1) + public String id; + + @Parameter(names = {"--tags"}, description = "The body web service tags parameter", required = false, arity = 1) + public String tags; + + @Parameter(names = {"--visibility"}, description = "Enum param allowed values: PUBLIC, PRIVATE", required = false, arity = 1) + public String visibility; + + @Parameter(names = {"--value-type"}, description = "Enum param allowed values: OBJECT, ARRAY, STRING, INTEGER, DOUBLE", required = false, arity = 1) + public String valueType; + + } + + @Parameters(commandNames = {"notes-search"}, commandDescription ="Search for notes of scope STUDY") + public class SearchNotesCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) + public String include; + + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) + public String exclude; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = true, arity = 1) + public String study; + + @Parameter(names = {"--creation-date", "--cd"}, description = "Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805", required = false, arity = 1) + public String creationDate; + + @Parameter(names = {"--modification-date", "--md"}, description = "Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805", required = false, arity = 1) + public String modificationDate; + + @Parameter(names = {"--id"}, description = "Note unique identifier.", required = false, arity = 1) + public String id; + + @Parameter(names = {"--uuid"}, description = "Unique 32-character identifier assigned automatically by OpenCGA.", required = false, arity = 1) + public String uuid; + + @Parameter(names = {"--user-id"}, description = "User that wrote that Note.", required = false, arity = 1) + public String userId; + + @Parameter(names = {"--tags"}, description = "Note tags.", required = false, arity = 1) + public String tags; + + @Parameter(names = {"--visibility"}, description = "Visibility of the Note.", required = false, arity = 1) + public String visibility; + + @Parameter(names = {"--version"}, description = "Autoincremental version assigned to the registered entry. By default, updates does not create new versions. To enable versioning, users must set the `incVersion` flag from the /update web service when updating the document.", required = false, arity = 1) + public String version; + + } + + @Parameters(commandNames = {"notes-delete"}, commandDescription ="Delete note") + public class DeleteNotesCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = true, arity = 1) + public String study; + + @Parameter(names = {"--id"}, description = "Note unique identifier.", required = true, arity = 1) + public String id; + + @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) + public boolean includeResult = false; + + } + + @Parameters(commandNames = {"notes-update"}, commandDescription ="Update a note") + public class UpdateNotesCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) + public String jsonFile; + + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) + public Boolean jsonDataModel = false; + + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) + public String include; + + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) + public String exclude; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = true, arity = 1) + public String study; + + @Parameter(names = {"--id"}, description = "Note unique identifier.", required = true, arity = 1) + public String id; + + @Parameter(names = {"--tags-action"}, description = "Action to be performed if the array of tags is being updated.", required = false, arity = 1) + public String tagsAction = "ADD"; + + @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) + public boolean includeResult = false; + + @Parameter(names = {"--tags"}, description = "The body web service tags parameter", required = false, arity = 1) + public String tags; + + @Parameter(names = {"--visibility"}, description = "Enum param allowed values: PUBLIC, PRIVATE", required = false, arity = 1) + public String visibility; + + } + @Parameters(commandNames = {"permissionrules"}, commandDescription ="Fetch permission rules") public class PermissionRulesCommandOptions { @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = true, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = true, arity = 1) public String study; @Parameter(names = {"--entity"}, description = "Entity where the permission rules should be applied to", required = true, arity = 1) @@ -437,7 +545,7 @@ public class UpdatePermissionRulesCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = true, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = true, arity = 1) public String study; @Parameter(names = {"--entity"}, description = "Entity where the permission rules should be applied to", required = true, arity = 1) @@ -466,7 +574,7 @@ public class DeleteTemplatesCommandOptions { @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = true, arity = 1) public String study; @Parameter(names = {"--template-id"}, description = "Template id", required = true, arity = 1) @@ -492,7 +600,7 @@ public class UpdateCommandOptions { @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) public String exclude; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = true, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = true, arity = 1) public String study; @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) @@ -539,7 +647,7 @@ public class VariableSetsCommandOptions { @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = true, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = true, arity = 1) public String study; @Parameter(names = {"--id"}, description = "Id of the variableSet to be retrieved. If no id is passed, it will show all the variableSets of the study", required = false, arity = 1) @@ -559,7 +667,7 @@ public class UpdateVariableSetsCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = true, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = true, arity = 1) public String study; @Parameter(names = {"--action"}, description = "Action to be performed: ADD, REMOVE or FORCE_REMOVE a variableSet", required = false, arity = 1) @@ -594,7 +702,7 @@ public class UpdateVariableSetsVariablesCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; - @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = true, arity = 1) + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = true, arity = 1) public String study; @Parameter(names = {"--variable-set"}, description = "VariableSet id of the VariableSet to be updated", required = true, arity = 1) @@ -612,7 +720,7 @@ public class UpdateVariableSetsVariablesCommandOptions { @Parameter(names = {"--category"}, description = "The body web service category parameter", required = false, arity = 1) public String category; - @Parameter(names = {"--type"}, description = "Enum param allowed values: BOOLEAN, CATEGORICAL, INTEGER, DOUBLE, TEXT, STRING, OBJECT, MAP_BOOLEAN, MAP_INTEGER, MAP_DOUBLE, MAP_STRING", required = false, arity = 1) + @Parameter(names = {"--type"}, description = "Enum param allowed values: BOOLEAN, CATEGORICAL, INTEGER, DOUBLE, STRING, OBJECT, MAP_BOOLEAN, MAP_INTEGER, MAP_DOUBLE, MAP_STRING", required = false, arity = 1) public String type; @Parameter(names = {"--required"}, description = "The body web service required parameter", required = false, help = true, arity = 0) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/UsersCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/UsersCommandOptions.java index 8fa672db469..66d7cee55a8 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/UsersCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/UsersCommandOptions.java @@ -33,32 +33,82 @@ public class UsersCommandOptions extends CustomUsersCommandOptions { + public AnonymousCommandOptions anonymousCommandOptions; + public CreateCommandOptions createCommandOptions; public LoginCommandOptions loginCommandOptions; public PasswordCommandOptions passwordCommandOptions; + public SearchCommandOptions searchCommandOptions; public InfoCommandOptions infoCommandOptions; public ConfigsCommandOptions configsCommandOptions; public UpdateConfigsCommandOptions updateConfigsCommandOptions; public FiltersCommandOptions filtersCommandOptions; public ResetPasswordCommandOptions resetPasswordCommandOptions; - public ProjectsCommandOptions projectsCommandOptions; public UpdateCommandOptions updateCommandOptions; public UsersCommandOptions(CommonCommandOptions commonCommandOptions, JCommander jCommander) { super(commonCommandOptions,jCommander); + this.anonymousCommandOptions = new AnonymousCommandOptions(); + this.createCommandOptions = new CreateCommandOptions(); this.loginCommandOptions = new LoginCommandOptions(); this.passwordCommandOptions = new PasswordCommandOptions(); + this.searchCommandOptions = new SearchCommandOptions(); this.infoCommandOptions = new InfoCommandOptions(); this.configsCommandOptions = new ConfigsCommandOptions(); this.updateConfigsCommandOptions = new UpdateConfigsCommandOptions(); this.filtersCommandOptions = new FiltersCommandOptions(); this.resetPasswordCommandOptions = new ResetPasswordCommandOptions(); - this.projectsCommandOptions = new ProjectsCommandOptions(); this.updateCommandOptions = new UpdateCommandOptions(); } + @Parameters(commandNames = {"anonymous"}, commandDescription ="Get an anonymous token to gain access to the system") + public class AnonymousCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) + public String jsonFile; + + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) + public Boolean jsonDataModel = false; + + @Parameter(names = {"--organization"}, description = "Organization id", required = true, arity = 1) + public String organization; + + } + + @Parameters(commandNames = {"create"}, commandDescription ="Create a new user") + public class CreateCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) + public String jsonFile; + + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) + public Boolean jsonDataModel = false; + + @Parameter(names = {"--id"}, description = "The body web service id parameter", required = true, arity = 1) + public String id; + + @Parameter(names = {"--name", "-n"}, description = "The body web service name parameter", required = false, arity = 1) + public String name; + + @Parameter(names = {"--email"}, description = "The body web service email parameter", required = false, arity = 1) + public String email; + + @Parameter(names = {"--password"}, description = "The body web service password parameter", required = false, arity = 1) + public String password; + + @Parameter(names = {"--organization"}, description = "The body web service organization parameter", required = false, arity = 1) + public String organization; + + } + @Parameters(commandNames = {"password"}, commandDescription ="Change the password of a user") public class PasswordCommandOptions { @@ -71,6 +121,9 @@ public class PasswordCommandOptions { @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) public Boolean jsonDataModel = false; + @Parameter(names = {"--organization-id"}, description = "The body web service organizationId parameter", required = false, arity = 1) + public String organizationId; + @Parameter(names = {"--user", "-u"}, description = "The body web service user parameter", required = false, arity = 1) public String user; @@ -85,6 +138,38 @@ public class PasswordCommandOptions { } + @Parameters(commandNames = {"search"}, commandDescription ="User search method") + public class SearchCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) + public String include; + + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) + public String exclude; + + @Parameter(names = {"--limit"}, description = "Number of results to be returned", required = false, arity = 1) + public Integer limit; + + @Parameter(names = {"--skip"}, description = "Number of results to skip", required = false, arity = 1) + public Integer skip; + + @Parameter(names = {"--count"}, description = "Get the total number of results matching the query. Deactivated by default.", required = false, help = true, arity = 0) + public boolean count = false; + + @Parameter(names = {"--organization"}, description = "Organization id", required = false, arity = 1) + public String organization; + + @Parameter(names = {"--id"}, description = "Comma separated list user IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = false, arity = 1) + public String id; + + @Parameter(names = {"--authentication-id"}, description = "Authentication origin ID", required = false, arity = 1) + public String authenticationId; + + } + @Parameters(commandNames = {"info"}, commandDescription ="Return the user information including its projects and studies") public class InfoCommandOptions { @@ -97,6 +182,9 @@ public class InfoCommandOptions { @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) public String exclude; + @Parameter(names = {"--organization"}, description = "Organization id", required = false, arity = 1) + public String organization; + @Parameter(names = {"--users"}, description = "Comma separated list of user IDs", required = true, arity = 1) public String users; @@ -167,29 +255,6 @@ public class ResetPasswordCommandOptions { } - @Parameters(commandNames = {"projects"}, commandDescription ="Retrieve the projects of the user") - public class ProjectsCommandOptions { - - @ParametersDelegate - public CommonCommandOptions commonOptions = commonCommandOptions; - - @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) - public String include; - - @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) - public String exclude; - - @Parameter(names = {"--limit"}, description = "Number of results to be returned", required = false, arity = 1) - public Integer limit; - - @Parameter(names = {"--skip"}, description = "Number of results to skip", required = false, arity = 1) - public Integer skip; - - @Parameter(names = {"--user", "-u"}, description = "User ID", required = true, arity = 1) - public String user; - - } - @Parameters(commandNames = {"update"}, commandDescription ="Update some user attributes") public class UpdateCommandOptions { @@ -220,12 +285,6 @@ public class UpdateCommandOptions { @Parameter(names = {"--email"}, description = "The body web service email parameter", required = false, arity = 1) public String email; - @Parameter(names = {"--organization"}, description = "The body web service organization parameter", required = false, arity = 1) - public String organization; - - @DynamicParameter(names = {"--attributes"}, description = "The body web service attributes parameter. Use: --attributes key=value", required = false) - public java.util.Map attributes = new HashMap<>(); //Dynamic parameters must be initialized; - } } \ No newline at end of file diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/utils/CliCompletionMain.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/utils/CliCompletionMain.java index 7e5984ce1ce..345965a5339 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/utils/CliCompletionMain.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/utils/CliCompletionMain.java @@ -26,7 +26,7 @@ public class CliCompletionMain { - public static void main(String[] args) { + public static void main(String[] args) throws IOException { try { String output = System.getProperty("app.home") + "/bin/utils"; if (args != null && args.length > 0) { @@ -44,8 +44,9 @@ public static void main(String[] args) { InternalCliOptionsParser internalCliOptionsParser = new InternalCliOptionsParser(); CommandLineUtils.generateBashAutoComplete(internalCliOptionsParser.getJCommander(), output + "/opencga-internal", "opencga-internal", Collections.singletonList("-D")); - } catch (IOException e) { + } catch (RuntimeException | IOException e) { e.printStackTrace(); + throw e; } } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/StorageMigrationTool.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/StorageMigrationTool.java index 16914627d5b..4952f9d9812 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/StorageMigrationTool.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/StorageMigrationTool.java @@ -64,7 +64,7 @@ protected final List getVariantStorageProjects() throws Exception { protected final List getVariantStorageStudies() throws Exception { Set studies = new LinkedHashSet<>(); VariantStorageManager variantStorageManager = getVariantStorageManager(); - for (Study study : catalogManager.getStudyManager().search(new Query(), new QueryOptions(QueryOptions.INCLUDE, + for (Study study : catalogManager.getStudyManager().searchInOrganization(organizationId, new Query(), new QueryOptions(QueryOptions.INCLUDE, Arrays.asList("fqn")), token).getResults()) { if (variantStorageManager.exists(study.getFqn(), token)) { studies.add(study.getFqn()); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_0_3/catalog/java/Migration1.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_0_3/catalog/java/Migration1.java new file mode 100644 index 00000000000..425fd72353f --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_0_3/catalog/java/Migration1.java @@ -0,0 +1,14 @@ +package org.opencb.opencga.app.migrations.v2.v2_0_3.catalog.java; + +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "recalculate_roles", description = "Recalculate roles from Family #1763", version = "2.0.3", date = 20210528, + deprecatedSince = "3.0.0") +public class Migration1 extends MigrationTool { + + @Override + protected void run() throws CatalogException { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_0_5/catalog/java/initialiseGroups.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_0_5/catalog/java/initialiseGroups.java new file mode 100644 index 00000000000..0c0273e7820 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_0_5/catalog/java/initialiseGroups.java @@ -0,0 +1,15 @@ +package org.opencb.opencga.app.migrations.v2.v2_0_5.catalog.java; + + +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "initialise_groups", description = "Initialise userIds list from groups #1791", version = "2.0.5", date = 20210621, + deprecatedSince = "3.0.0") +public class initialiseGroups extends MigrationTool { + + @Override + protected void run() throws CatalogException { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_0_6/catalog/removeDeletedFileReferencesFromSample.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_0_6/catalog/removeDeletedFileReferencesFromSample.java new file mode 100644 index 00000000000..b99e0a9f1d5 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_0_6/catalog/removeDeletedFileReferencesFromSample.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_0_6.catalog; + + +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "remove_file_references_from_sample", description = "Remove deleted file references from samples #1815", version = "2.0.6", + date = 20210901, deprecatedSince = "3.0.0") +public class removeDeletedFileReferencesFromSample extends MigrationTool { + + @Override + protected void run() throws CatalogException { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_0/catalog/AddAnnotationSetsInClinicalAnalysisMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_0/catalog/AddAnnotationSetsInClinicalAnalysisMigration.java new file mode 100644 index 00000000000..06a78900ab0 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_0/catalog/AddAnnotationSetsInClinicalAnalysisMigration.java @@ -0,0 +1,20 @@ +package org.opencb.opencga.app.migrations.v2.v2_12_0.catalog; + + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_annotation_sets_to_clinical_analysis" , + description = "Add private annotation fields to ClinicalAnalysis documents #TASK-5198", + version = "2.12.0", + domain = Migration.MigrationDomain.CATALOG, + language = Migration.MigrationLanguage.JAVA, + date = 20231116, + deprecatedSince = "3.0.0" +) +public class AddAnnotationSetsInClinicalAnalysisMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_0/catalog/CompleteClinicalReportDataModelMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_0/catalog/CompleteClinicalReportDataModelMigration.java new file mode 100644 index 00000000000..96f702d96f4 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_0/catalog/CompleteClinicalReportDataModelMigration.java @@ -0,0 +1,19 @@ +package org.opencb.opencga.app.migrations.v2.v2_12_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "complete_clinical_report_data_model" , + description = "Complete Clinical Report data model #TASK-5198", + version = "2.12.0", + domain = Migration.MigrationDomain.CATALOG, + language = Migration.MigrationLanguage.JAVA, + date = 20231128, + deprecatedSince = "3.0.0" +) +public class CompleteClinicalReportDataModelMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_4/catalog/FixStatusIndexesMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_4/catalog/FixStatusIndexesMigration.java new file mode 100644 index 00000000000..e1953673252 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_4/catalog/FixStatusIndexesMigration.java @@ -0,0 +1,46 @@ +package org.opencb.opencga.app.migrations.v2.v2_12_4.catalog; + +import org.bson.Document; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +import java.util.Arrays; + +@Migration(id = "fix_status_indexes" , + description = "Replace 'status.name' indexes for 'status.id'", + version = "2.12.4", + domain = Migration.MigrationDomain.CATALOG, + language = Migration.MigrationLanguage.JAVA, + date = 20240328 +) +public class FixStatusIndexesMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + Document statusIndex = new Document() + .append("status.name", 1) + .append("studyUid", 1); + dropIndex(Arrays.asList(OrganizationMongoDBAdaptorFactory.JOB_COLLECTION, OrganizationMongoDBAdaptorFactory.FILE_COLLECTION, + OrganizationMongoDBAdaptorFactory.SAMPLE_COLLECTION, OrganizationMongoDBAdaptorFactory.SAMPLE_ARCHIVE_COLLECTION, + OrganizationMongoDBAdaptorFactory.COHORT_COLLECTION, OrganizationMongoDBAdaptorFactory.INDIVIDUAL_COLLECTION, + OrganizationMongoDBAdaptorFactory.INDIVIDUAL_ARCHIVE_COLLECTION, OrganizationMongoDBAdaptorFactory.FAMILY_COLLECTION, + OrganizationMongoDBAdaptorFactory.FAMILY_ARCHIVE_COLLECTION, OrganizationMongoDBAdaptorFactory.PANEL_COLLECTION, + OrganizationMongoDBAdaptorFactory.PANEL_ARCHIVE_COLLECTION), statusIndex); + + Document internalStatusIndex = new Document() + .append("internal.status.name", 1) + .append("studyUid", 1); + dropIndex(Arrays.asList(OrganizationMongoDBAdaptorFactory.JOB_COLLECTION, OrganizationMongoDBAdaptorFactory.FILE_COLLECTION, + OrganizationMongoDBAdaptorFactory.SAMPLE_COLLECTION, OrganizationMongoDBAdaptorFactory.SAMPLE_ARCHIVE_COLLECTION, + OrganizationMongoDBAdaptorFactory.COHORT_COLLECTION, OrganizationMongoDBAdaptorFactory.INDIVIDUAL_COLLECTION, + OrganizationMongoDBAdaptorFactory.INDIVIDUAL_ARCHIVE_COLLECTION, OrganizationMongoDBAdaptorFactory.FAMILY_COLLECTION, + OrganizationMongoDBAdaptorFactory.FAMILY_ARCHIVE_COLLECTION, OrganizationMongoDBAdaptorFactory.PANEL_COLLECTION, + OrganizationMongoDBAdaptorFactory.PANEL_ARCHIVE_COLLECTION, OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, + OrganizationMongoDBAdaptorFactory.INTERPRETATION_COLLECTION, OrganizationMongoDBAdaptorFactory.INTERPRETATION_ARCHIVE_COLLECTION), + internalStatusIndex); + + catalogManager.installIndexes(token); + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_5/storage/AddAllelesColumnToPhoenix.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_5/storage/AddAllelesColumnToPhoenix.java similarity index 95% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_5/storage/AddAllelesColumnToPhoenix.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_5/storage/AddAllelesColumnToPhoenix.java index c71f0483427..d641207a04e 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_5/storage/AddAllelesColumnToPhoenix.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_5/storage/AddAllelesColumnToPhoenix.java @@ -1,4 +1,4 @@ -package org.opencb.opencga.app.migrations.v2_12_5.storage; +package org.opencb.opencga.app.migrations.v2.v2_12_5.storage; import org.opencb.opencga.app.migrations.StorageMigrationTool; import org.opencb.opencga.catalog.migration.Migration; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_5/storage/DetectIllegalConcurrentFileLoadingsMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_5/storage/DetectIllegalConcurrentFileLoadingsMigration.java similarity index 99% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_5/storage/DetectIllegalConcurrentFileLoadingsMigration.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_5/storage/DetectIllegalConcurrentFileLoadingsMigration.java index c407cd82bbd..aba2f366b77 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_5/storage/DetectIllegalConcurrentFileLoadingsMigration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_5/storage/DetectIllegalConcurrentFileLoadingsMigration.java @@ -1,4 +1,4 @@ -package org.opencb.opencga.app.migrations.v2_12_5.storage; +package org.opencb.opencga.app.migrations.v2.v2_12_5.storage; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; @@ -267,7 +267,7 @@ private void invalidateFileIndex(String study, Integer fileId, ObjectMap event, .first(); catalogManager.getFileManager().update(study, file.getId(), new FileUpdateParams().setAttributes(new ObjectMap("TASK-6078", thisEvent)), QueryOptions.empty(), token); - catalogManager.getFileManager().updateFileInternalVariantIndex(file, new FileInternalVariantIndex() + catalogManager.getFileManager().updateFileInternalVariantIndex(study, file, new FileInternalVariantIndex() .setStatus(new VariantIndexStatus(IndexStatus.INVALID, "Invalid status - TASK-6078 - affected_invalid_sample - " + "File must be deleted and then indexed")), token); } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_6/SyncCohortsAndSamplesMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_6/SyncCohortsAndSamplesMigration.java similarity index 91% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_6/SyncCohortsAndSamplesMigration.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_6/SyncCohortsAndSamplesMigration.java index edfdbb7c828..6f6f5cbf18d 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_6/SyncCohortsAndSamplesMigration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_6/SyncCohortsAndSamplesMigration.java @@ -1,4 +1,4 @@ -package org.opencb.opencga.app.migrations.v2_12_6; +package org.opencb.opencga.app.migrations.v2.v2_12_6; import com.mongodb.client.MongoCollection; import com.mongodb.client.model.Filters; @@ -10,7 +10,7 @@ import org.opencb.opencga.catalog.db.api.CohortDBAdaptor; import org.opencb.opencga.catalog.db.api.SampleDBAdaptor; import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptor; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; import org.opencb.opencga.catalog.migration.Migration; import org.opencb.opencga.catalog.migration.MigrationTool; @@ -28,10 +28,10 @@ public class SyncCohortsAndSamplesMigration extends MigrationTool { @Override protected void run() throws Exception { - MongoCollection sampleCollection = getMongoCollection(MongoDBAdaptorFactory.SAMPLE_COLLECTION); - MongoCollection sampleArchiveCollection = getMongoCollection(MongoDBAdaptorFactory.SAMPLE_ARCHIVE_COLLECTION); + MongoCollection sampleCollection = getMongoCollection(OrganizationMongoDBAdaptorFactory.SAMPLE_COLLECTION); + MongoCollection sampleArchiveCollection = getMongoCollection(OrganizationMongoDBAdaptorFactory.SAMPLE_ARCHIVE_COLLECTION); - queryMongo(MongoDBAdaptorFactory.COHORT_COLLECTION, new Document(), + queryMongo(OrganizationMongoDBAdaptorFactory.COHORT_COLLECTION, new Document(), Projections.include(CohortDBAdaptor.QueryParams.ID.key(), CohortDBAdaptor.QueryParams.SAMPLES.key()), cohortDoc -> { String cohortId = cohortDoc.getString(CohortDBAdaptor.QueryParams.ID.key()); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/AddMissingIndexes.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/AddMissingIndexes.java new file mode 100644 index 00000000000..7efdea6a56a --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/AddMissingIndexes.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_1_0.catalog.java; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_missing_indexes", + description = "Add missing indexes", version = "2.1.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + patch = 1, + date = 20210928, + deprecatedSince = "3.0.0") +public class AddMissingIndexes extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/AddPanelsToInterpretations.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/AddPanelsToInterpretations.java new file mode 100644 index 00000000000..a35f794d130 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/AddPanelsToInterpretations.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_1_0.catalog.java; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_panels_to_interpretations", + description = "Add panels to Interpretations #1802", version = "2.1.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20210713, + deprecatedSince = "3.0.0") +public class AddPanelsToInterpretations extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/catalog/java/CreateAuditIndexes.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/CreateAuditIndexes.java similarity index 70% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/catalog/java/CreateAuditIndexes.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/CreateAuditIndexes.java index fcf48708bcc..48cd8ce2f3b 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/catalog/java/CreateAuditIndexes.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/CreateAuditIndexes.java @@ -1,4 +1,4 @@ -package org.opencb.opencga.app.migrations.v2_1_0.catalog.java; +package org.opencb.opencga.app.migrations.v2.v2_1_0.catalog.java; import org.opencb.opencga.catalog.migration.Migration; import org.opencb.opencga.catalog.migration.MigrationTool; @@ -7,13 +7,12 @@ language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, patch = 1, - date = 20210622) + date = 20210622, + deprecatedSince = "3.0.0") public class CreateAuditIndexes extends MigrationTool { @Override protected void run() throws Exception { - // This will actually create all missing indexes - catalogManager.installIndexes(token); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/PanelLock.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/PanelLock.java new file mode 100644 index 00000000000..747feb3bbcb --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/PanelLock.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_1_0.catalog.java; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_panel_lock", + description = "Add new panelLock to ClinicalAnalysis #1802", version = "2.1.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + patch = 1, + date = 20210713, + deprecatedSince = "3.0.0") +public class PanelLock extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/RemoveRCVersionsFromMigrationRuns.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/RemoveRCVersionsFromMigrationRuns.java new file mode 100644 index 00000000000..58625266d9b --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/RemoveRCVersionsFromMigrationRuns.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_1_0.catalog.java; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "remove_rc_version_from_migration_runs", + description = "Remove RC versions from migration runs stored in catalog", version = "2.1.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + patch = 1, + date = 20210723, + deprecatedSince = "3.0.0") +public class RemoveRCVersionsFromMigrationRuns extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/RenameDatastoreConfigurationToOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/RenameDatastoreConfigurationToOptions.java new file mode 100644 index 00000000000..452e99d627f --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/RenameDatastoreConfigurationToOptions.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_1_0.catalog.java; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "rename_datastore_configuration_to_options", description = "Rename project.internal.datastores.variant.configuration to options", version = "2.1.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + patch = 1, + date = 20210617, + deprecatedSince = "3.0.0") +public class RenameDatastoreConfigurationToOptions extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/StudyClinicalConfigurationRelocation.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/StudyClinicalConfigurationRelocation.java new file mode 100644 index 00000000000..18e6272d7a6 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/StudyClinicalConfigurationRelocation.java @@ -0,0 +1,19 @@ +package org.opencb.opencga.app.migrations.v2.v2_1_0.catalog.java; + + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "move_study_clinical_config_to_internal", + description = "Move Study ClinicalConfiguration to internal.configuration", version = "2.1.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + patch = 1, + date = 20210708, + deprecatedSince = "3.0.0") +public class StudyClinicalConfigurationRelocation extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/VariantFileStatsRelocation.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/VariantFileStatsRelocation.java new file mode 100644 index 00000000000..f37d46cb9f2 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/VariantFileStatsRelocation.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_1_0.catalog.java; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "move_variant_file_stats_to_qc", description = "Move opencga_file_variant_stats annotation set from variable sets to " + + "FileQualityControl", version = "2.1.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.STORAGE, + patch = 1, + date = 20210614, + deprecatedSince = "3.0.0") +public class VariantFileStatsRelocation extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/catalog/javascript/Migration1.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/javascript/Migration1.java similarity index 69% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/catalog/javascript/Migration1.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/javascript/Migration1.java index 6851372d4bc..0530e369b04 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/catalog/javascript/Migration1.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/javascript/Migration1.java @@ -1,18 +1,14 @@ -package org.opencb.opencga.app.migrations.v2_1_0.catalog.javascript; +package org.opencb.opencga.app.migrations.v2.v2_1_0.catalog.javascript; import org.opencb.opencga.catalog.migration.Migration; import org.opencb.opencga.catalog.migration.MigrationException; import org.opencb.opencga.catalog.migration.MigrationTool; -import java.nio.file.Path; - @Migration(id = "build_rga_indexes", description = "Create index for sample RGA status #1693", version = "2.1.0", - language = Migration.MigrationLanguage.JAVASCRIPT, date = 20210528) + language = Migration.MigrationLanguage.JAVASCRIPT, date = 20210528, deprecatedSince = "3.0.0") public class Migration1 extends MigrationTool { @Override protected void run() throws MigrationException { - Path path = appHome.resolve("misc/migration/v2.1.0/migration1.js"); - runJavascript(path); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/catalog/javascript/Migration2.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/javascript/Migration2.java similarity index 69% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/catalog/javascript/Migration2.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/javascript/Migration2.java index e9d2f7cb1e5..a0c0938de73 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/catalog/javascript/Migration2.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/javascript/Migration2.java @@ -1,18 +1,14 @@ -package org.opencb.opencga.app.migrations.v2_1_0.catalog.javascript; +package org.opencb.opencga.app.migrations.v2.v2_1_0.catalog.javascript; import org.opencb.opencga.catalog.migration.Migration; import org.opencb.opencga.catalog.migration.MigrationException; import org.opencb.opencga.catalog.migration.MigrationTool; -import java.nio.file.Path; - @Migration(id = "init_userId_group_arrays", description = "Initialise all userIds arrays from groups #1735", version = "2.1.0", - language = Migration.MigrationLanguage.JAVASCRIPT, date = 20210528) + language = Migration.MigrationLanguage.JAVASCRIPT, date = 20210528, deprecatedSince = "3.0.0") public class Migration2 extends MigrationTool { @Override protected void run() throws MigrationException { - Path path = appHome.resolve("misc/migration/v2.1.0/migration2.js"); - runJavascript(path); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/catalog/javascript/Migration3.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/javascript/Migration3.java similarity index 69% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/catalog/javascript/Migration3.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/javascript/Migration3.java index b6d247da8fb..9b21dcdbc9c 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/catalog/javascript/Migration3.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/javascript/Migration3.java @@ -1,18 +1,14 @@ -package org.opencb.opencga.app.migrations.v2_1_0.catalog.javascript; +package org.opencb.opencga.app.migrations.v2.v2_1_0.catalog.javascript; import org.opencb.opencga.catalog.migration.Migration; import org.opencb.opencga.catalog.migration.MigrationException; import org.opencb.opencga.catalog.migration.MigrationTool; -import java.nio.file.Path; - @Migration(id = "init_ca_panel_arrays", description = "Initialise panels array in Clinical Analysis #1759", version = "2.1.0", - language = Migration.MigrationLanguage.JAVASCRIPT, date = 20210528) + language = Migration.MigrationLanguage.JAVASCRIPT, date = 20210528, deprecatedSince = "3.0.0") public class Migration3 extends MigrationTool { @Override protected void run() throws MigrationException { - Path path = appHome.resolve("misc/migration/v2.1.0/migration3.js"); - runJavascript(path); } } \ No newline at end of file diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/catalog/javascript/Migration4.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/javascript/Migration4.java similarity index 68% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/catalog/javascript/Migration4.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/javascript/Migration4.java index 871097d3ddf..ef72a501f44 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/catalog/javascript/Migration4.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/javascript/Migration4.java @@ -1,18 +1,14 @@ -package org.opencb.opencga.app.migrations.v2_1_0.catalog.javascript; +package org.opencb.opencga.app.migrations.v2.v2_1_0.catalog.javascript; import org.opencb.opencga.catalog.migration.Migration; import org.opencb.opencga.catalog.migration.MigrationException; import org.opencb.opencga.catalog.migration.MigrationTool; -import java.nio.file.Path; - @Migration(id = "update_qc_file_sample_fields", description = "Update QC fields from Sample and File #1730", version = "2.1.0", - language = Migration.MigrationLanguage.JAVASCRIPT, date = 20210531, patch = 4) + language = Migration.MigrationLanguage.JAVASCRIPT, date = 20210531, patch = 4, deprecatedSince = "3.0.0") public class Migration4 extends MigrationTool { @Override protected void run() throws MigrationException { - Path path = appHome.resolve("misc/migration/v2.1.0/migration4.js"); - runJavascript(path); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/storage/AddCellbaseConfigurationToProject.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/storage/AddCellbaseConfigurationToProject.java new file mode 100644 index 00000000000..5dac53d4688 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/storage/AddCellbaseConfigurationToProject.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_1_0.storage; + + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_cellbase_configuration_to_project", description = "Add cellbase configuration from storage-configuration.yml to project.internal.cellbase", version = "2.1.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.STORAGE, + patch = 3, + date = 20210616, + deprecatedSince = "3.0.0") +public class AddCellbaseConfigurationToProject extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/storage/DefaultSampleIndexConfiguration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/storage/DefaultSampleIndexConfiguration.java new file mode 100644 index 00000000000..7b33bd7d709 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/storage/DefaultSampleIndexConfiguration.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_1_0.storage; + + +import org.opencb.opencga.app.migrations.StorageMigrationTool; +import org.opencb.opencga.catalog.migration.Migration; + +@Migration(id = "default_sample_index_configuration", description = "Add a default backward compatible sample index configuration", version = "2.1.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.STORAGE, + patch = 7, + date = 20210721, + deprecatedSince = "3.0.0") // Needs to run after StudyClinicalConfigurationRelocation +public class DefaultSampleIndexConfiguration extends StorageMigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/storage/NewClinicalSignificanceFields.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/storage/NewClinicalSignificanceFields.java new file mode 100644 index 00000000000..b4d243ee279 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/storage/NewClinicalSignificanceFields.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_1_0.storage; + +import org.opencb.opencga.app.migrations.StorageMigrationTool; +import org.opencb.opencga.catalog.migration.Migration; + + +@Migration(id = "new_clinical_significance_fields", description = "Add new clinical significance fields and combinations for variant storage and solr", version = "2.1.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.STORAGE, + patch = 1, + date = 20210708, + deprecatedSince = "3.0.0") +public class NewClinicalSignificanceFields extends StorageMigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/AddMissingClinicalAudit.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/AddMissingClinicalAudit.java new file mode 100644 index 00000000000..7008388f42e --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/AddMissingClinicalAudit.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_missing_create_interpretation_in_clinical_audit", + description = "Add missing CREATE_INTERPRETATION audits in ClinicalAnalysis", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211227, deprecatedSince = "3.0.0") +public class AddMissingClinicalAudit extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/AddNameFieldInCohort.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/AddNameFieldInCohort.java new file mode 100644 index 00000000000..1f23c1b331c --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/AddNameFieldInCohort.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_name_field_in_cohort_1902", + description = "Add new name field to Cohort #1902", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220228, deprecatedSince = "3.0.0") +public class AddNameFieldInCohort extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/AddNewAllowedBiotype.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/AddNewAllowedBiotype.java new file mode 100644 index 00000000000..1aa51b53014 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/AddNewAllowedBiotype.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_new_allowed_biotype", + description = "Add new allowed biotype 'guide_RNA', #1856", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + patch = 2, + date = 20211209, deprecatedSince = "3.0.0") +public class AddNewAllowedBiotype extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/AddNewFileInternalIndex_1850.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/AddNewFileInternalIndex_1850.java new file mode 100644 index 00000000000..099dd960790 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/AddNewFileInternalIndex_1850.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "new_file_internal_index_1850", + description = "Add new FileInternalVariant and FileInternalAlignment index #1850", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211127, deprecatedSince = "3.0.0") +public class AddNewFileInternalIndex_1850 extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/AddSampleInternalVariant_1851.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/AddSampleInternalVariant_1851.java new file mode 100644 index 00000000000..21218e3222d --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/AddSampleInternalVariant_1851.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_sample_internal_variant_1851", + description = "Add new SampleInternalVariant #1851", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211129, deprecatedSince = "3.0.0") +public class AddSampleInternalVariant_1851 extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/ChangeInterpretationMethods.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/ChangeInterpretationMethods.java new file mode 100644 index 00000000000..7731a4510fe --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/ChangeInterpretationMethods.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "change_interpretation_method", + description = "Remove list of methods from Interpretations #1841", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211112, deprecatedSince = "3.0.0") +public class ChangeInterpretationMethods extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/ClinicalVariantEvidenceMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/ClinicalVariantEvidenceMigration.java new file mode 100644 index 00000000000..6de2592fbaa --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/ClinicalVariantEvidenceMigration.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_clinical_variant_evidence_review", + description = "Add new ClinicalVariantEvidenceReview object, #1874", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220112, deprecatedSince = "3.0.0") +public class ClinicalVariantEvidenceMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/DeleteUnusedVariableSets.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/DeleteUnusedVariableSets.java new file mode 100644 index 00000000000..7ddea73508c --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/DeleteUnusedVariableSets.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "delete_unused_variablesets", + description = "Delete unused VariableSets, #1859", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + patch = 2, + date = 20211210, deprecatedSince = "3.0.0") +public class DeleteUnusedVariableSets extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/FixFamilyReferences.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/FixFamilyReferences.java new file mode 100644 index 00000000000..fd160750475 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/FixFamilyReferences.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "fix_family_references_in_individual", + description = "Fix Family references, #TASK-489", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220324, deprecatedSince = "3.0.0") +public class FixFamilyReferences extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/FixNonExistingMoIFromPanels.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/FixNonExistingMoIFromPanels.java new file mode 100644 index 00000000000..d642d8499af --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/FixNonExistingMoIFromPanels.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "fix_non_existing_mois_from_panels", + description = "Remove non-existing MOIs from Panels", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + patch = 2, + date = 20220111, deprecatedSince = "3.0.0") +public class FixNonExistingMoIFromPanels extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/ImproveClinicalAnalysisQualityControl.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/ImproveClinicalAnalysisQualityControl.java new file mode 100644 index 00000000000..b7ff196615b --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/ImproveClinicalAnalysisQualityControl.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "improve_ca_quality_control", + description = "Quality control normalize comments and fields #1826", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211001, deprecatedSince = "3.0.0") +public class ImproveClinicalAnalysisQualityControl extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/ImproveFileQualityControl.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/ImproveFileQualityControl.java new file mode 100644 index 00000000000..16043d38b26 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/ImproveFileQualityControl.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "improve_file_quality_control", + description = "Quality control normalize comments and fields #1826", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211001, deprecatedSince = "3.0.0") +public class ImproveFileQualityControl extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/ImproveIndividualQualityControl.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/ImproveIndividualQualityControl.java new file mode 100644 index 00000000000..b3350e099f3 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/ImproveIndividualQualityControl.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "improve_individual_quality_control", + description = "Quality control normalize comments and fields #1826", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211001, deprecatedSince = "3.0.0") +public class ImproveIndividualQualityControl extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/ImproveVariableSetNamesAndDescriptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/ImproveVariableSetNamesAndDescriptions.java new file mode 100644 index 00000000000..39b2df0d2a2 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/ImproveVariableSetNamesAndDescriptions.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "improveVariableSetNamesAndDescriptions", + description = "Improve VariableSet names and descriptions", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211210) +public class ImproveVariableSetNamesAndDescriptions extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RemoveFileDocsFromCA.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RemoveFileDocsFromCA.java new file mode 100644 index 00000000000..af85465d69e --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RemoveFileDocsFromCA.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "remove_file_docs_from_clinical_analyses", + description = "Store references of File in Clinical Analysis and not full File documents #1837", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211102, deprecatedSince = "3.0.0") +public class RemoveFileDocsFromCA extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RemoveParallelIndexes.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RemoveParallelIndexes.java new file mode 100644 index 00000000000..7b00543f9a5 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RemoveParallelIndexes.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "remove_parallel_indexes", + description = "Remove parallel array indexes #CU-20jc4tx", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220310, deprecatedSince = "3.0.0") +public class RemoveParallelIndexes extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RenameFamilyQualityControlFields.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RenameFamilyQualityControlFields.java new file mode 100644 index 00000000000..821fb020a28 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RenameFamilyQualityControlFields.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "rename_family_quality_control_fields", + description = "Rename FamilyQualityControl fields #1844", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211119, deprecatedSince = "3.0.0") +public class RenameFamilyQualityControlFields extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RenameFileQualityControlFields.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RenameFileQualityControlFields.java new file mode 100644 index 00000000000..c3c5d4ff7f8 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RenameFileQualityControlFields.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "rename_file_quality_control_fields", + description = "Rename FileQualityControl fields #1844", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211119, deprecatedSince = "3.0.0") +public class RenameFileQualityControlFields extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RenameIndividualQualityControlFields.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RenameIndividualQualityControlFields.java new file mode 100644 index 00000000000..6df5fc82131 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RenameIndividualQualityControlFields.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "rename_individual_quality_control_fields", + description = "Rename IndividualQualityControl fields #1844", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211119, deprecatedSince = "3.0.0") +public class RenameIndividualQualityControlFields extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RenameInterpretationFindingStats.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RenameInterpretationFindingStats.java new file mode 100644 index 00000000000..bd4ad966774 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RenameInterpretationFindingStats.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "rename_interpretation_stats_field", + description = "Rename interpretation stats field #1819", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211115, deprecatedSince = "3.0.0") +public class RenameInterpretationFindingStats extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RenameSampleQualityControlFields.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RenameSampleQualityControlFields.java new file mode 100644 index 00000000000..0c5b36cdf93 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RenameSampleQualityControlFields.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "rename_sample_quality_control_fields", + description = "Rename SampleQualityControl fields #1844", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + patch = 2, + date = 20211119, deprecatedSince = "3.0.0") +public class RenameSampleQualityControlFields extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addDefaultVariantCallerInterpretationStudyConfiguration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addDefaultVariantCallerInterpretationStudyConfiguration.java new file mode 100644 index 00000000000..9a43d99ac53 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addDefaultVariantCallerInterpretationStudyConfiguration.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_variant_caller_interpretation_configuration", + description = "Add default variant caller Interpretation Study configuration #1822", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20210916, deprecatedSince = "3.0.0") +public class addDefaultVariantCallerInterpretationStudyConfiguration extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddInternalLastModified.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddInternalLastModified.java new file mode 100644 index 00000000000..28e32235bb0 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddInternalLastModified.java @@ -0,0 +1,10 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addInternalLastModified; + +import org.opencb.opencga.catalog.migration.MigrationTool; + +public abstract class AddInternalLastModified extends MigrationTool { + + protected void addInternalModificationDate(String collection) { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addInternalLastModified/AddModificationDateToClinicalInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToClinicalInternal.java similarity index 63% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addInternalLastModified/AddModificationDateToClinicalInternal.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToClinicalInternal.java index 80a9b017a5a..c127e7b2695 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addInternalLastModified/AddModificationDateToClinicalInternal.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToClinicalInternal.java @@ -1,15 +1,13 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog.addInternalLastModified; +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addInternalLastModified; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; import org.opencb.opencga.catalog.migration.Migration; @Migration(id = "add_modificationDate_to_clinical.internal", description = "Add internal modificationDate to Clinical #1810", version = "2.2.0", language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, - date = 20210812) + date = 20210812, deprecatedSince = "3.0.0") public class AddModificationDateToClinicalInternal extends AddInternalLastModified { @Override protected void run() throws Exception { - addInternalModificationDate(MongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addInternalLastModified/AddModificationDateToCohortInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToCohortInternal.java similarity index 63% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addInternalLastModified/AddModificationDateToCohortInternal.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToCohortInternal.java index 48e631628d7..0aaccf98b74 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addInternalLastModified/AddModificationDateToCohortInternal.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToCohortInternal.java @@ -1,15 +1,13 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog.addInternalLastModified; +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addInternalLastModified; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; import org.opencb.opencga.catalog.migration.Migration; @Migration(id = "add_modificationDate_to_cohort.internal", description = "Add internal modificationDate to Cohort #1810", version = "2.2.0", language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, - date = 20210812) + date = 20210812, deprecatedSince = "3.0.0") public class AddModificationDateToCohortInternal extends AddInternalLastModified { @Override protected void run() throws Exception { - addInternalModificationDate(MongoDBAdaptorFactory.COHORT_COLLECTION); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addInternalLastModified/AddModificationDateToFamilyInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToFamilyInternal.java similarity index 63% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addInternalLastModified/AddModificationDateToFamilyInternal.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToFamilyInternal.java index 3970c7624a5..914b6351a0c 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addInternalLastModified/AddModificationDateToFamilyInternal.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToFamilyInternal.java @@ -1,15 +1,13 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog.addInternalLastModified; +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addInternalLastModified; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; import org.opencb.opencga.catalog.migration.Migration; @Migration(id = "add_modificationDate_to_family.internal", description = "Add internal modificationDate to Family #1810", version = "2.2.0", language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, - date = 20210812) + date = 20210812, deprecatedSince = "3.0.0") public class AddModificationDateToFamilyInternal extends AddInternalLastModified { @Override protected void run() throws Exception { - addInternalModificationDate(MongoDBAdaptorFactory.FAMILY_COLLECTION); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addInternalLastModified/AddModificationDateToFileInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToFileInternal.java similarity index 63% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addInternalLastModified/AddModificationDateToFileInternal.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToFileInternal.java index 48a09ae0d08..7ffc3cf7f37 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addInternalLastModified/AddModificationDateToFileInternal.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToFileInternal.java @@ -1,15 +1,13 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog.addInternalLastModified; +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addInternalLastModified; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; import org.opencb.opencga.catalog.migration.Migration; @Migration(id = "add_modificationDate_to_file.internal", description = "Add internal modificationDate to File #1810", version = "2.2.0", language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, - date = 20210812) + date = 20210812, deprecatedSince = "3.0.0") public class AddModificationDateToFileInternal extends AddInternalLastModified { @Override protected void run() throws Exception { - addInternalModificationDate(MongoDBAdaptorFactory.FILE_COLLECTION); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addInternalLastModified/AddModificationDateToIndividualInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToIndividualInternal.java similarity index 64% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addInternalLastModified/AddModificationDateToIndividualInternal.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToIndividualInternal.java index 90490f950fa..05ca7bc522d 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addInternalLastModified/AddModificationDateToIndividualInternal.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToIndividualInternal.java @@ -1,15 +1,13 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog.addInternalLastModified; +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addInternalLastModified; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; import org.opencb.opencga.catalog.migration.Migration; @Migration(id = "add_modificationDate_to_individual.internal", description = "Add internal modificationDate to Individual #1810", version = "2.2.0", language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, - date = 20210812) + date = 20210812, deprecatedSince = "3.0.0") public class AddModificationDateToIndividualInternal extends AddInternalLastModified { @Override protected void run() throws Exception { - addInternalModificationDate(MongoDBAdaptorFactory.INDIVIDUAL_COLLECTION); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addInternalLastModified/AddModificationDateToInterpretationInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToInterpretationInternal.java similarity index 64% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addInternalLastModified/AddModificationDateToInterpretationInternal.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToInterpretationInternal.java index e6b667be27f..9340231dfde 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addInternalLastModified/AddModificationDateToInterpretationInternal.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToInterpretationInternal.java @@ -1,15 +1,13 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog.addInternalLastModified; +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addInternalLastModified; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; import org.opencb.opencga.catalog.migration.Migration; @Migration(id = "add_modificationDate_to_interpretation.internal", description = "Add internal modificationDate to Interpretation #1810", version = "2.2.0", language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, - date = 20210812) + date = 20210812, deprecatedSince = "3.0.0") public class AddModificationDateToInterpretationInternal extends AddInternalLastModified { @Override protected void run() throws Exception { - addInternalModificationDate(MongoDBAdaptorFactory.INTERPRETATION_COLLECTION); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addInternalLastModified/AddModificationDateToJobInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToJobInternal.java similarity index 63% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addInternalLastModified/AddModificationDateToJobInternal.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToJobInternal.java index f1a42462b79..543e02f328e 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addInternalLastModified/AddModificationDateToJobInternal.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToJobInternal.java @@ -1,15 +1,13 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog.addInternalLastModified; +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addInternalLastModified; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; import org.opencb.opencga.catalog.migration.Migration; @Migration(id = "add_modificationDate_to_job.internal", description = "Add internal modificationDate to Job #1810", version = "2.2.0", language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, - date = 20210812) + date = 20210812, deprecatedSince = "3.0.0") public class AddModificationDateToJobInternal extends AddInternalLastModified { @Override protected void run() throws Exception { - addInternalModificationDate(MongoDBAdaptorFactory.JOB_COLLECTION); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToProjectInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToProjectInternal.java new file mode 100644 index 00000000000..82a3142a222 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToProjectInternal.java @@ -0,0 +1,14 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addInternalLastModified; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_modificationDate_to_project.internal", description = "Add internal modificationDate to Project #1810", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, + date = 20210812, deprecatedSince = "3.0.0") +public class AddModificationDateToProjectInternal extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addInternalLastModified/AddModificationDateToSampleInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToSampleInternal.java similarity index 63% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addInternalLastModified/AddModificationDateToSampleInternal.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToSampleInternal.java index 081614b5b33..875bbaa26a7 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addInternalLastModified/AddModificationDateToSampleInternal.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToSampleInternal.java @@ -1,15 +1,13 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog.addInternalLastModified; +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addInternalLastModified; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; import org.opencb.opencga.catalog.migration.Migration; @Migration(id = "add_modificationDate_to_sample.internal", description = "Add internal modificationDate to Sample #1810", version = "2.2.0", language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, - date = 20210812) + date = 20210812, deprecatedSince = "3.0.0") public class AddModificationDateToSampleInternal extends AddInternalLastModified { @Override protected void run() throws Exception { - addInternalModificationDate(MongoDBAdaptorFactory.SAMPLE_COLLECTION); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addInternalLastModified/AddModificationDateToStudyInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToStudyInternal.java similarity index 63% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addInternalLastModified/AddModificationDateToStudyInternal.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToStudyInternal.java index a2c0a521853..987a742cb5d 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addInternalLastModified/AddModificationDateToStudyInternal.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToStudyInternal.java @@ -1,15 +1,13 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog.addInternalLastModified; +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addInternalLastModified; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; import org.opencb.opencga.catalog.migration.Migration; @Migration(id = "add_modificationDate_to_study.internal", description = "Add internal modificationDate to Study #1810", version = "2.2.0", language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, - date = 20210812) + date = 20210812, deprecatedSince = "3.0.0") public class AddModificationDateToStudyInternal extends AddInternalLastModified { @Override protected void run() throws Exception { - addInternalModificationDate(MongoDBAdaptorFactory.STUDY_COLLECTION); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInterpretationStats.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInterpretationStats.java new file mode 100644 index 00000000000..ac1c112d0a1 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInterpretationStats.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_interpretation_stats", + description = "Add interpretation stats #1819", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + patch = 2, + date = 20210908, deprecatedSince = "3.0.0") +public class addInterpretationStats extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDate.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDate.java new file mode 100644 index 00000000000..efd1107ba7b --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDate.java @@ -0,0 +1,10 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addRegistrationDate; + +import org.opencb.opencga.catalog.migration.MigrationTool; + +public abstract class AddRegistrationDate extends MigrationTool { + + protected void addRegistrationDate(String collection) { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToClinicalInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToClinicalInternal.java similarity index 63% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToClinicalInternal.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToClinicalInternal.java index c2281afbb23..14a48ed90be 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToClinicalInternal.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToClinicalInternal.java @@ -1,15 +1,13 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog.addRegistrationDate; +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addRegistrationDate; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; import org.opencb.opencga.catalog.migration.Migration; @Migration(id = "add_registrationDate_to_clinical.internal", description = "Add registrationDate to Clinical #1804", version = "2.2.0", language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, - date = 20210720) + date = 20210720, deprecatedSince = "3.0.0") public class AddRegistrationDateToClinicalInternal extends AddRegistrationDate { @Override protected void run() throws Exception { - addRegistrationDate(MongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToCohortInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToCohortInternal.java similarity index 64% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToCohortInternal.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToCohortInternal.java index 27663ff964e..a7dcf145b33 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToCohortInternal.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToCohortInternal.java @@ -1,15 +1,13 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog.addRegistrationDate; +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addRegistrationDate; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; import org.opencb.opencga.catalog.migration.Migration; @Migration(id = "add_registrationDate_to_cohort.internal", description = "Add registrationDate to Cohort #1804", version = "2.2.0", language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, - date = 20210720) + date = 20210720, deprecatedSince = "3.0.0") public class AddRegistrationDateToCohortInternal extends AddRegistrationDate { @Override protected void run() throws Exception { - addRegistrationDate(MongoDBAdaptorFactory.COHORT_COLLECTION); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToFamilyInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToFamilyInternal.java similarity index 64% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToFamilyInternal.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToFamilyInternal.java index 18929017fee..66f375ce955 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToFamilyInternal.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToFamilyInternal.java @@ -1,15 +1,13 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog.addRegistrationDate; +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addRegistrationDate; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; import org.opencb.opencga.catalog.migration.Migration; @Migration(id = "add_registrationDate_to_family.internal", description = "Add registrationDate to Family #1804", version = "2.2.0", language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, - date = 20210720) + date = 20210720, deprecatedSince = "3.0.0") public class AddRegistrationDateToFamilyInternal extends AddRegistrationDate { @Override protected void run() throws Exception { - addRegistrationDate(MongoDBAdaptorFactory.FAMILY_COLLECTION); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToFileInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToFileInternal.java similarity index 64% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToFileInternal.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToFileInternal.java index 4c88b518840..cfa29de842a 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToFileInternal.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToFileInternal.java @@ -1,15 +1,13 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog.addRegistrationDate; +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addRegistrationDate; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; import org.opencb.opencga.catalog.migration.Migration; @Migration(id = "add_registrationDate_to_file.internal", description = "Add registrationDate to File #1804", version = "2.2.0", language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, - date = 20210720) + date = 20210720, deprecatedSince = "3.0.0") public class AddRegistrationDateToFileInternal extends AddRegistrationDate { @Override protected void run() throws Exception { - addRegistrationDate(MongoDBAdaptorFactory.FILE_COLLECTION); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToIndividualInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToIndividualInternal.java similarity index 64% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToIndividualInternal.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToIndividualInternal.java index a60e9e1255c..58d233f6d7f 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToIndividualInternal.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToIndividualInternal.java @@ -1,15 +1,13 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog.addRegistrationDate; +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addRegistrationDate; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; import org.opencb.opencga.catalog.migration.Migration; @Migration(id = "add_registrationDate_to_individual.internal", description = "Add registrationDate to Individual #1804", version = "2.2.0", language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, - date = 20210720) + date = 20210720, deprecatedSince = "3.0.0") public class AddRegistrationDateToIndividualInternal extends AddRegistrationDate { @Override protected void run() throws Exception { - addRegistrationDate(MongoDBAdaptorFactory.INDIVIDUAL_COLLECTION); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToInterpretationInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToInterpretationInternal.java similarity index 64% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToInterpretationInternal.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToInterpretationInternal.java index 8e442278407..115b4e4fc6c 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToInterpretationInternal.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToInterpretationInternal.java @@ -1,15 +1,13 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog.addRegistrationDate; +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addRegistrationDate; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; import org.opencb.opencga.catalog.migration.Migration; @Migration(id = "add_registrationDate_to_interpretation.internal", description = "Add registrationDate to Interpretation #1804", version = "2.2.0", language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, - date = 20210720) + date = 20210720, deprecatedSince = "3.0.0") public class AddRegistrationDateToInterpretationInternal extends AddRegistrationDate { @Override protected void run() throws Exception { - addRegistrationDate(MongoDBAdaptorFactory.INTERPRETATION_COLLECTION); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToJobInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToJobInternal.java similarity index 63% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToJobInternal.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToJobInternal.java index d14c3413836..c56124cdda6 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToJobInternal.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToJobInternal.java @@ -1,15 +1,13 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog.addRegistrationDate; +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addRegistrationDate; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; import org.opencb.opencga.catalog.migration.Migration; @Migration(id = "add_registrationDate_to_job.internal", description = "Add registrationDate to Job #1804", version = "2.2.0", language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, - date = 20210720) + date = 20210720, deprecatedSince = "3.0.0") public class AddRegistrationDateToJobInternal extends AddRegistrationDate { @Override protected void run() throws Exception { - addRegistrationDate(MongoDBAdaptorFactory.JOB_COLLECTION); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToProjectInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToProjectInternal.java new file mode 100644 index 00000000000..ae3f65b70d9 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToProjectInternal.java @@ -0,0 +1,14 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addRegistrationDate; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_registrationDate_to_project.internal", description = "Add registrationDate to Project #1804", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, + date = 20210720, deprecatedSince = "3.0.0") +public class AddRegistrationDateToProjectInternal extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToSampleInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToSampleInternal.java similarity index 64% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToSampleInternal.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToSampleInternal.java index e0f2b0a1332..e317c667a98 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToSampleInternal.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToSampleInternal.java @@ -1,15 +1,13 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog.addRegistrationDate; +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addRegistrationDate; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; import org.opencb.opencga.catalog.migration.Migration; @Migration(id = "add_registrationDate_to_sample.internal", description = "Add registrationDate to Sample #1804", version = "2.2.0", language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, - date = 20210720) + date = 20210720, deprecatedSince = "3.0.0") public class AddRegistrationDateToSampleInternal extends AddRegistrationDate { @Override protected void run() throws Exception { - addRegistrationDate(MongoDBAdaptorFactory.SAMPLE_COLLECTION); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToStudyInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToStudyInternal.java similarity index 64% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToStudyInternal.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToStudyInternal.java index 3a61791eb1d..7d036f65bc4 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToStudyInternal.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToStudyInternal.java @@ -1,15 +1,13 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog.addRegistrationDate; +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addRegistrationDate; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; import org.opencb.opencga.catalog.migration.Migration; @Migration(id = "add_registrationDate_to_study.internal", description = "Add registrationDate to Study #1804", version = "2.2.0", language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, - date = 20210720) + date = 20210720, deprecatedSince = "3.0.0") public class AddRegistrationDateToStudyInternal extends AddRegistrationDate { @Override protected void run() throws Exception { - addRegistrationDate(MongoDBAdaptorFactory.STUDY_COLLECTION); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRgaIndexToStudyInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRgaIndexToStudyInternal.java new file mode 100644 index 00000000000..d995f4ba97f --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRgaIndexToStudyInternal.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_rga_index_summary_to_study_internal", + description = "Add RGA Index information to Study Internal #", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + patch = 1, + date = 20210719, deprecatedSince = "3.0.0") +public class addRgaIndexToStudyInternal extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1795/AddFamilyIdsInIndividual.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1795/AddFamilyIdsInIndividual.java new file mode 100644 index 00000000000..25e93347812 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1795/AddFamilyIdsInIndividual.java @@ -0,0 +1,15 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.issue_1795; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationException; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_familyIds_in_individual", description = "Add new list of familyIds in Individual #1795", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, date = 20210630, deprecatedSince = "3.0.0") +public class AddFamilyIdsInIndividual extends MigrationTool { + + @Override + protected void run() throws MigrationException { + } +} + diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1796/AddCohortIdsInSample.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1796/AddCohortIdsInSample.java new file mode 100644 index 00000000000..b6d1d3ff298 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1796/AddCohortIdsInSample.java @@ -0,0 +1,15 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.issue_1796; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationException; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_cohortIds_in_sample", description = "Add new list of cohortIds in Sample #1796", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, date = 20210706, deprecatedSince = "3.0.0") +public class AddCohortIdsInSample extends MigrationTool { + + @Override + protected void run() throws MigrationException { + } +} + diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteClinicalStatusDataModel.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteClinicalStatusDataModel.java new file mode 100644 index 00000000000..fcb2313cb39 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteClinicalStatusDataModel.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.issue_1849; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "complete_clinical_status_models", + description = "Complete Clinical Status data models #1849", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211126, + deprecatedSince = "3.0.0") +public class CompleteClinicalStatusDataModel extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteCohortStatusDataModel.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteCohortStatusDataModel.java new file mode 100644 index 00000000000..692e58cc397 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteCohortStatusDataModel.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.issue_1849; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "complete_cohort_status_models", + description = "Complete Cohort Status data models #1849", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211126, deprecatedSince = "3.0.0") +public class CompleteCohortStatusDataModel extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteFamilyStatusDataModel.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteFamilyStatusDataModel.java new file mode 100644 index 00000000000..5c1bfbfc2aa --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteFamilyStatusDataModel.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.issue_1849; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "complete_family_status_models", + description = "Complete Family Status data models #1849", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211126, deprecatedSince = "3.0.0") +public class CompleteFamilyStatusDataModel extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteFileStatusDataModel.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteFileStatusDataModel.java new file mode 100644 index 00000000000..c1c84058b5c --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteFileStatusDataModel.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.issue_1849; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "complete_file_status_models", + description = "Complete File Status data models #1849", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211126, deprecatedSince = "3.0.0") +public class CompleteFileStatusDataModel extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteIndividualStatusDataModel.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteIndividualStatusDataModel.java new file mode 100644 index 00000000000..cb7fda578ca --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteIndividualStatusDataModel.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.issue_1849; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "complete_individual_status_models", + description = "Complete Individual Status data models #1849", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211126, deprecatedSince = "3.0.0") +public class CompleteIndividualStatusDataModel extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteInterpretationStatusDataModel.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteInterpretationStatusDataModel.java new file mode 100644 index 00000000000..dad8a984810 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteInterpretationStatusDataModel.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.issue_1849; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "complete_interpretation_status_models", + description = "Complete Interpretation Status data models #1849", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211126, deprecatedSince = "3.0.0") +public class CompleteInterpretationStatusDataModel extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteJobStatusDataModel.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteJobStatusDataModel.java new file mode 100644 index 00000000000..4a08d4695e7 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteJobStatusDataModel.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.issue_1849; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "complete_job_status_models", + description = "Complete Job Status data models #1849", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211126, deprecatedSince = "3.0.0") +public class CompleteJobStatusDataModel extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompletePanelStatusDataModel.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompletePanelStatusDataModel.java new file mode 100644 index 00000000000..8311dcc2153 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompletePanelStatusDataModel.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.issue_1849; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "complete_panel_status_models", + description = "Complete Panel Status data models #1849", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211126, deprecatedSince = "3.0.0") +public class CompletePanelStatusDataModel extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteProjectStatusDataModel.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteProjectStatusDataModel.java new file mode 100644 index 00000000000..09ce8a4fc88 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteProjectStatusDataModel.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.issue_1849; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "complete_project_status_models", + description = "Complete Project Status data models #1849", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211126, deprecatedSince = "3.0.0") +public class CompleteProjectStatusDataModel extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteSampleStatusDataModel.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteSampleStatusDataModel.java new file mode 100644 index 00000000000..bb9b3e54086 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteSampleStatusDataModel.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.issue_1849; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "complete_sample_status_models", + description = "Complete Sample Status data models #1849", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211126, deprecatedSince = "3.0.0") +public class CompleteSampleStatusDataModel extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteStudyStatusDataModel.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteStudyStatusDataModel.java new file mode 100644 index 00000000000..8b119d92cc4 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteStudyStatusDataModel.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.issue_1849; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "complete_study_status_models", + description = "Complete Study Status data models #1849", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211126, deprecatedSince = "3.0.0") +public class CompleteStudyStatusDataModel extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteUserStatusDataModel.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteUserStatusDataModel.java new file mode 100644 index 00000000000..937e6d3f9fb --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteUserStatusDataModel.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.issue_1849; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "complete_user_status_models", + description = "Complete User Status data models #1849", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211126, deprecatedSince = "3.0.0") +public class CompleteUserStatusDataModel extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issues_1853_1855/IndividualOntologyTermMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issues_1853_1855/IndividualOntologyTermMigration.java new file mode 100644 index 00000000000..9a5eb6cec84 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issues_1853_1855/IndividualOntologyTermMigration.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.issues_1853_1855; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "individual_ontology_term_migration_#1853", + description = "Change sex and ethnicity types for OntologyTermAnnotation #1855", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211203, deprecatedSince = "3.0.0") +public class IndividualOntologyTermMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issues_1853_1855/NewStudyDataModelFields.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issues_1853_1855/NewStudyDataModelFields.java new file mode 100644 index 00000000000..e6659fb4ad7 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issues_1853_1855/NewStudyDataModelFields.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.issues_1853_1855; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "new_study_data_model_fields_#1853", + description = "New Study 'sources', 'type' and 'additionalInfo' fields #1853", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211202, deprecatedSince = "3.0.0") +public class NewStudyDataModelFields extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issues_1853_1855/SampleProcessingAndCollectionChanges.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issues_1853_1855/SampleProcessingAndCollectionChanges.java new file mode 100644 index 00000000000..a054a6a0566 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issues_1853_1855/SampleProcessingAndCollectionChanges.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.issues_1853_1855; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "sample_source_treatments_#1854", + description = "Sample source, treatments, processing and collection changes #1854", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211201, patch = 2, deprecatedSince = "3.0.0") +public class SampleProcessingAndCollectionChanges extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/renameVariableSetFieldFromVariable.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/renameVariableSetFieldFromVariable.java new file mode 100644 index 00000000000..bd3f00937e9 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/renameVariableSetFieldFromVariable.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "rename_variableset_field", + description = "Rename Variable variableSet field to variables #1823", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20210920, deprecatedSince = "3.0.0") +public class renameVariableSetFieldFromVariable extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/storage/SynchronizeCatalogStorage.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/storage/SynchronizeCatalogStorage.java new file mode 100644 index 00000000000..609d50c915a --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/storage/SynchronizeCatalogStorage.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.storage; + +import org.opencb.opencga.app.migrations.StorageMigrationTool; +import org.opencb.opencga.catalog.migration.Migration; + +@Migration(id = "synchronize_catalog_storage_1850_1851", + description = "Synchronize catalog storage #1850 #1851", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.STORAGE, + date = 20220118, deprecatedSince = "3.0.0") +public class SynchronizeCatalogStorage extends StorageMigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/storage/UpdateSampleIndexStatus.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/storage/UpdateSampleIndexStatus.java new file mode 100644 index 00000000000..2965c326760 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/storage/UpdateSampleIndexStatus.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.storage; + +import org.opencb.opencga.app.migrations.StorageMigrationTool; +import org.opencb.opencga.catalog.migration.Migration; + +@Migration(id = "update_sample_index_status_1901", + description = " Improve sample-index multi-schema management. #1901", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.STORAGE, + date = 20220302, deprecatedSince = "3.0.0") +public class UpdateSampleIndexStatus extends StorageMigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_1/catalog/AddMissingBiotypes.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_1/catalog/AddMissingBiotypes.java new file mode 100644 index 00000000000..7173a402e96 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_1/catalog/AddMissingBiotypes.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_1.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_missing_biotypes", + description = "Add missing biotypes, #TASK-625", version = "2.2.1", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220412, deprecatedSince = "3.0.0") +public class AddMissingBiotypes extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_1/catalog/AddPanelGeneNameIndex.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_1/catalog/AddPanelGeneNameIndex.java similarity index 55% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_1/catalog/AddPanelGeneNameIndex.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_1/catalog/AddPanelGeneNameIndex.java index 24445dddd2b..0cd9ba5ff1b 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_1/catalog/AddPanelGeneNameIndex.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_1/catalog/AddPanelGeneNameIndex.java @@ -1,7 +1,5 @@ -package org.opencb.opencga.app.migrations.v2_2_1.catalog; +package org.opencb.opencga.app.migrations.v2.v2_2_1.catalog; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; import org.opencb.opencga.catalog.migration.Migration; import org.opencb.opencga.catalog.migration.MigrationTool; @@ -9,15 +7,11 @@ description = "Add Panel gene.name index #TASK-602", version = "2.2.1", language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, - date = 20220406) + date = 20220406, deprecatedSince = "3.0.0") public class AddPanelGeneNameIndex extends MigrationTool { @Override protected void run() throws Exception { - Document newIndex = new Document() - .append("genes.name", 1) - .append("studyUid", 1); - createIndex(MongoDBAdaptorFactory.PANEL_COLLECTION, newIndex); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_1/catalog/InitSharedProjectField.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_1/catalog/InitSharedProjectField.java new file mode 100644 index 00000000000..49b4b252f60 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_1/catalog/InitSharedProjectField.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_1.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "init_shared_project", + description = "Initialise sharedProjects #TASK-702", version = "2.2.1", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220502, deprecatedSince = "3.0.0") +public class InitSharedProjectField extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_1/storage/SampleIndexConfigurationAddMissingStatus.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_1/storage/SampleIndexConfigurationAddMissingStatus.java new file mode 100644 index 00000000000..cd11e50f8aa --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_1/storage/SampleIndexConfigurationAddMissingStatus.java @@ -0,0 +1,15 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_1.storage; + +import org.opencb.opencga.app.migrations.StorageMigrationTool; +import org.opencb.opencga.catalog.migration.Migration; + +@Migration(id = "sample_index_configuration_add_missing_status", + description = "Sample index configuration add missing status #TASK-512", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.STORAGE, + date = 20220328, deprecatedSince = "3.0.0") +public class SampleIndexConfigurationAddMissingStatus extends StorageMigrationTool { + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/AddLockedToInterpretation.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/AddLockedToInterpretation.java new file mode 100644 index 00000000000..52a95ccf66e --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/AddLockedToInterpretation.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_3_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_locked_to_interpretation_TASK-552", + description = "Add new 'locked' field to Interpretation #TASK-552", version = "2.3.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220401, + deprecatedSince = "3.0.0") +public class AddLockedToInterpretation extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/AddPanelInternalField.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/AddPanelInternalField.java new file mode 100644 index 00000000000..e2edbf68060 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/AddPanelInternalField.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_3_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_panel_internal_field-TASK_734", + description = "Add 'internal' to Panel data model #TASK-734", version = "2.3.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220509, + deprecatedSince = "3.0.0") +public class AddPanelInternalField extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/AutoIncrementVersion.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/AutoIncrementVersion.java new file mode 100644 index 00000000000..8cbd5a59eef --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/AutoIncrementVersion.java @@ -0,0 +1,19 @@ +package org.opencb.opencga.app.migrations.v2.v2_3_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "autoincrement_version_TASK-323", + description = "Reorganise data in new collections (autoincrement version) #TASK-323", version = "2.3.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + offline = true, + date = 20220512, + deprecatedSince = "3.0.0") +public class AutoIncrementVersion extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/InitSampleIndexConfiguration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/InitSampleIndexConfiguration.java new file mode 100644 index 00000000000..81d3f1e8628 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/InitSampleIndexConfiguration.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_3_0.catalog; + +import org.opencb.opencga.app.migrations.StorageMigrationTool; +import org.opencb.opencga.catalog.migration.Migration; + +@Migration(id = "init_sampleIndexConfiguration-TASK550", + description = "Initialise SampleIndexConfiguration in Study internal #TASK-550", version = "2.3.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220502, + deprecatedSince = "3.0.0") +public class InitSampleIndexConfiguration extends StorageMigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/MoveCellbaseFromProjectInternalToProject.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/MoveCellbaseFromProjectInternalToProject.java new file mode 100644 index 00000000000..9f4f30213fb --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/MoveCellbaseFromProjectInternalToProject.java @@ -0,0 +1,19 @@ +package org.opencb.opencga.app.migrations.v2.v2_3_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "move_cellbase_from_project_internal_to_project_TASK-354", + description = "Move cellbase from project.internal to project #TASK-354", version = "2.3.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220422, + deprecatedSince = "3.0.0") + +public class MoveCellbaseFromProjectInternalToProject extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/RenameDeletedCollections.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/RenameDeletedCollections.java new file mode 100644 index 00000000000..2a1532c6da4 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/RenameDeletedCollections.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_3_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "rename_deleted_collections_TASK-359", + description = "Rename deleted collections #TASK-359", version = "2.3.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220502, + deprecatedSince = "3.0.0") +public class RenameDeletedCollections extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/RenameDeprecatedVariantStorageToolId.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/RenameDeprecatedVariantStorageToolId.java new file mode 100644 index 00000000000..b39a56cdbec --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/RenameDeprecatedVariantStorageToolId.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_3_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "rename_variant_storage_tool_ids_TASK-705", + description = "Rename variant storage tool ids #TASK-705", version = "2.3.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220506, + deprecatedSince = "3.0.0") +public class RenameDeprecatedVariantStorageToolId extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/UpdateJWTSecretKey.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/UpdateJWTSecretKey.java new file mode 100644 index 00000000000..be328bbeb6a --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/UpdateJWTSecretKey.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_3_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "update_jwt_secret_key_TASK-807", + description = "Update JWT secret key #TASK-807", version = "2.3.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220513, + deprecatedSince = "3.0.0") +public class UpdateJWTSecretKey extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/storage/AddMissingColumnsToPhoenix.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/storage/AddMissingColumnsToPhoenix.java new file mode 100644 index 00000000000..df2e308a727 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/storage/AddMissingColumnsToPhoenix.java @@ -0,0 +1,14 @@ +package org.opencb.opencga.app.migrations.v2.v2_3_0.storage; + +import org.opencb.opencga.app.migrations.StorageMigrationTool; +import org.opencb.opencga.catalog.migration.Migration; + +@Migration(id="add_missing_columns_to_phoenix_TASK-789", description = "Add missing 1000G columns to phoenix #TASK-789", + version = "2.3.0", domain = Migration.MigrationDomain.STORAGE, date = 20220614, deprecatedSince = "3.0.0" +) +public class AddMissingColumnsToPhoenix extends StorageMigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_1/catalog/MoveCellbaseFromProjectInternalToProjectBugFix.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_1/catalog/MoveCellbaseFromProjectInternalToProjectBugFix.java new file mode 100644 index 00000000000..2c645980391 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_1/catalog/MoveCellbaseFromProjectInternalToProjectBugFix.java @@ -0,0 +1,19 @@ +package org.opencb.opencga.app.migrations.v2.v2_3_1.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "bugfix_move_cellbase_from_project_internal_to_project_TASK-1100", + description = "Bugfix: Move cellbase from project.internal to project #TASK-1100", version = "2.3.1", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220620, + deprecatedSince = "3.0.0") + +public class MoveCellbaseFromProjectInternalToProjectBugFix extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_0/catalog/AddClinicalStatusType_607.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_0/catalog/AddClinicalStatusType_607.java new file mode 100644 index 00000000000..8f3886c50e8 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_0/catalog/AddClinicalStatusType_607.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_4_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_clinical_status_type_TASK-607", + description = "Automatically close cases depending on the new clinical status type #TASK-607", version = "2.4.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220610, + deprecatedSince = "3.0.0") +public class AddClinicalStatusType_607 extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_0/catalog/AddPrivateDueDate_803.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_0/catalog/AddPrivateDueDate_803.java new file mode 100644 index 00000000000..6dcab05694a --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_0/catalog/AddPrivateDueDate_803.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_4_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_private_dueDate_TASK-803", + description = "Add private _dueDate to ClinicalAnalysis #TASK-803", version = "2.4.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220530, + deprecatedSince = "3.0.0") +public class AddPrivateDueDate_803 extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_0/catalog/IndexPanelSource.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_0/catalog/IndexPanelSource.java new file mode 100644 index 00000000000..af93d04de7d --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_0/catalog/IndexPanelSource.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_4_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "index_panel_source_TASK-473", + description = "Index panel source #TASK-473", version = "2.4.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220530, + deprecatedSince = "3.0.0") +public class IndexPanelSource extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_10/catalog/AddMissingEndDateOnFinishedJobs.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_10/catalog/AddMissingEndDateOnFinishedJobs.java new file mode 100644 index 00000000000..0759d4c7bf1 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_10/catalog/AddMissingEndDateOnFinishedJobs.java @@ -0,0 +1,20 @@ +package org.opencb.opencga.app.migrations.v2.v2_4_10.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_missing_endDate_on_finished_jobs" , + description = "Add missing end date on finished jobs", + version = "2.4.10", + domain = Migration.MigrationDomain.CATALOG, + language = Migration.MigrationLanguage.JAVA, + date = 20221026, + deprecatedSince = "3.0.0" +) +public class AddMissingEndDateOnFinishedJobs extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_11/SynchronizeCatalogStorageStatuses.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_11/SynchronizeCatalogStorageStatuses.java similarity index 56% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_11/SynchronizeCatalogStorageStatuses.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_11/SynchronizeCatalogStorageStatuses.java index 653c62944d9..e2fb99933cf 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_11/SynchronizeCatalogStorageStatuses.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_11/SynchronizeCatalogStorageStatuses.java @@ -1,6 +1,5 @@ -package org.opencb.opencga.app.migrations.v2_4_11; +package org.opencb.opencga.app.migrations.v2.v2_4_11; -import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; import org.opencb.opencga.app.migrations.StorageMigrationTool; import org.opencb.opencga.catalog.migration.Migration; @@ -8,16 +7,12 @@ description = "Synchronize catalog internal statuses from storage, #TASK-1304", version = "2.4.11", language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.STORAGE, - date = 20221117) + date = 20221117, + deprecatedSince = "3.0.0") public class SynchronizeCatalogStorageStatuses extends StorageMigrationTool { @Override protected void run() throws Exception { - VariantStorageManager manager = getVariantStorageManager(); - for (String study : getVariantStorageStudies()) { - logger.info("Synchronize study '{}'", study); - manager.synchronizeCatalogStudyFromStorage(study, token); - } } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_11/catalog/SignatureFittingsMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_11/catalog/SignatureFittingsMigration.java new file mode 100644 index 00000000000..ec0b49baae3 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_11/catalog/SignatureFittingsMigration.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_4_11.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "signature_fittings" , + description = "Replace fitting for fittings in Signature", + version = "2.4.11", + domain = Migration.MigrationDomain.CATALOG, + language = Migration.MigrationLanguage.JAVA, + date = 20221109, + deprecatedSince = "3.0.0" +) +public class SignatureFittingsMigration extends MigrationTool { + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_2/catalog/AddClinicalDiscussion.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_2/catalog/AddClinicalDiscussion.java new file mode 100644 index 00000000000..8e5825e3de7 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_2/catalog/AddClinicalDiscussion.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_4_2.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_clinical_discussion_TASK-1472", + description = "Add ClinicalDiscussion #TASK-1472", version = "2.4.2", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220727, + deprecatedSince = "3.0.0") +public class AddClinicalDiscussion extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_2/catalog/RecoverProbandSamplesInCases.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_2/catalog/RecoverProbandSamplesInCases.java new file mode 100644 index 00000000000..aba0267d65b --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_2/catalog/RecoverProbandSamplesInCases.java @@ -0,0 +1,19 @@ +package org.opencb.opencga.app.migrations.v2.v2_4_2.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "recover_proband_samples_in_cases_TASK-1470", + description = "Recover lost samples in clinical collection #TASK-1470", version = "2.4.2", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220725, + patch = 2, + deprecatedSince = "3.0.0") +public class RecoverProbandSamplesInCases extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_3/catalog/MigrateToClinicalAcmg.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_3/catalog/MigrateToClinicalAcmg.java new file mode 100644 index 00000000000..7951810b105 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_3/catalog/MigrateToClinicalAcmg.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_4_3.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "migrate_to_clinical_acmg_TASK-1194", + description = "Migrate to ClinicalAcmg #TASK-1194", version = "2.4.3", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220809, + deprecatedSince = "3.0.0") +public class MigrateToClinicalAcmg extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_3/catalog/ShortenIndexedFieldVariables.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_3/catalog/ShortenIndexedFieldVariables.java new file mode 100644 index 00000000000..2d4e2cb14a8 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_3/catalog/ShortenIndexedFieldVariables.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_4_3.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "shorten_indexed_field_variables-1386", + description = "Shorten indexed field variables #TASK-1386", version = "2.4.3", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220809, + deprecatedSince = "3.0.0") +public class ShortenIndexedFieldVariables extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_4/catalog/AvoidCaseStatusIdNullMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_4/catalog/AvoidCaseStatusIdNullMigration.java new file mode 100644 index 00000000000..e26df11c03c --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_4/catalog/AvoidCaseStatusIdNullMigration.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_4_4.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "avoidCaseStatusIdNull-1938", + description = "Avoid nullable status id in Clinical Analysis #TASK-1938", version = "2.4.4", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220914, + deprecatedSince = "3.0.0") +public class AvoidCaseStatusIdNullMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_4/catalog/RemoveBiotypeAllowedKeysMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_4/catalog/RemoveBiotypeAllowedKeysMigration.java new file mode 100644 index 00000000000..07a19a12fdb --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_4/catalog/RemoveBiotypeAllowedKeysMigration.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_4_4.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "remove_biotype_allowed_keys_TASK-1849", + description = "Remove biotype and consequence type allowed keys from variable sets #TASK-1849", version = "2.4.4", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220905, + deprecatedSince = "3.0.0") +public class RemoveBiotypeAllowedKeysMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_4/storage/InvalidateVariantArchivesMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_4/storage/InvalidateVariantArchivesMigration.java new file mode 100644 index 00000000000..069e1a86e6e --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_4/storage/InvalidateVariantArchivesMigration.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_4_4.storage; + +import org.opencb.opencga.app.migrations.StorageMigrationTool; +import org.opencb.opencga.catalog.migration.Migration; + +@Migration(id = "invalidate_variant_archives_migration-633", + description = "Invalidate variant archives from variant storage hadoop #TASK-633", version = "2.4.4", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.STORAGE, + date = 20220825, + deprecatedSince = "3.0.0") +public class InvalidateVariantArchivesMigration extends StorageMigrationTool { + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_6_0/storage/AddCellbaseDataRelease.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_6_0/storage/AddCellbaseDataRelease.java new file mode 100644 index 00000000000..06f1d871a80 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_6_0/storage/AddCellbaseDataRelease.java @@ -0,0 +1,20 @@ +package org.opencb.opencga.app.migrations.v2.v2_6_0.storage; + +import org.opencb.opencga.app.migrations.StorageMigrationTool; +import org.opencb.opencga.catalog.migration.Migration; + +@Migration(id = "add_cellbase_data_release" , + description = "Add default cellbase data release if missing", + version = "2.6.0", + domain = Migration.MigrationDomain.STORAGE, + language = Migration.MigrationLanguage.JAVA, + patch = 2, + date = 20230104, + deprecatedSince = "3.0.0" +) +public class AddCellbaseDataRelease extends StorageMigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_7_0/catalog/CancerRoleAndModeOfInheritanceMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_7_0/catalog/CancerRoleAndModeOfInheritanceMigration.java new file mode 100644 index 00000000000..ce425e3434b --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_7_0/catalog/CancerRoleAndModeOfInheritanceMigration.java @@ -0,0 +1,20 @@ +package org.opencb.opencga.app.migrations.v2.v2_7_0.catalog; + + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "panel_roles_and_modesOfInheritance" , + description = "Change panel cancer.role and modeOfInheritance for cancer.roles[] and modesOfInheritance[]", + version = "2.7.0", + domain = Migration.MigrationDomain.CATALOG, + language = Migration.MigrationLanguage.JAVA, + date = 20230117, + deprecatedSince = "3.0.0" +) +public class CancerRoleAndModeOfInheritanceMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_8_0/catalog/CalculatePedigreeGraphMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_8_0/catalog/CalculatePedigreeGraphMigration.java new file mode 100644 index 00000000000..0be802202c4 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_8_0/catalog/CalculatePedigreeGraphMigration.java @@ -0,0 +1,19 @@ +package org.opencb.opencga.app.migrations.v2.v2_8_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "calculate_pedigree_graph" , + description = "Calculate Pedigree Graph for all the families", + version = "2.8.0", + domain = Migration.MigrationDomain.CATALOG, + language = Migration.MigrationLanguage.JAVA, + date = 20230313, + deprecatedSince = "3.0.0" +) +public class CalculatePedigreeGraphMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_8_6/storage/MarkUnknownLargestVariantLength.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_8_6/storage/MarkUnknownLargestVariantLength.java new file mode 100644 index 00000000000..e3b093906c4 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_8_6/storage/MarkUnknownLargestVariantLength.java @@ -0,0 +1,20 @@ +package org.opencb.opencga.app.migrations.v2.v2_8_6.storage; + +import org.opencb.opencga.app.migrations.StorageMigrationTool; +import org.opencb.opencga.catalog.migration.Migration; + +@Migration(id = "mark_unknown_largest_variant_length" , + description = "Mark as unknown largest variant length", + version = "2.8.6", + domain = Migration.MigrationDomain.STORAGE, + language = Migration.MigrationLanguage.JAVA, + patch = 1, + date = 20230927, + deprecatedSince = "3.0.0" +) +public class MarkUnknownLargestVariantLength extends StorageMigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_9_0/catalog/RenameCellBaseToken2ApiKey.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_9_0/catalog/RenameCellBaseToken2ApiKey.java new file mode 100644 index 00000000000..7dbd590c27e --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_9_0/catalog/RenameCellBaseToken2ApiKey.java @@ -0,0 +1,20 @@ +package org.opencb.opencga.app.migrations.v2.v2_9_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "rename_cellbase_token_2_api_key" , + description = "Rename CellBase Token to ApiKey", + version = "2.9.0", + domain = Migration.MigrationDomain.CATALOG, + language = Migration.MigrationLanguage.JAVA, + date = 20230829, + deprecatedSince = "3.0.0" +) +public class RenameCellBaseToken2ApiKey extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_0_3/catalog/java/Migration1.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_0_3/catalog/java/Migration1.java deleted file mode 100644 index f7f11450cc2..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_0_3/catalog/java/Migration1.java +++ /dev/null @@ -1,65 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_0_3.catalog.java; - -import org.apache.commons.lang3.time.StopWatch; -import org.opencb.commons.datastore.core.Query; -import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.managers.FamilyManager; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; -import org.opencb.opencga.core.api.ParamConstants; -import org.opencb.opencga.core.models.family.Family; -import org.opencb.opencga.core.models.project.Project; -import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.response.OpenCGAResult; - -import java.util.concurrent.TimeUnit; - -@Migration(id = "recalculate_roles", description = "Recalculate roles from Family #1763", version = "2.0.3", date = 20210528) -public class Migration1 extends MigrationTool { - - @Override - protected void run() throws CatalogException { - // Add automatically roles to all the family members - StopWatch stopWatch = new StopWatch(); - - QueryOptions familyUpdateOptions = new QueryOptions(ParamConstants.FAMILY_UPDATE_ROLES_PARAM, true); - QueryOptions queryOptions = new QueryOptions(FamilyManager.INCLUDE_FAMILY_IDS) - .append(QueryOptions.LIMIT, 1000); - for (Project project : catalogManager.getProjectManager().search(new Query(), new QueryOptions(), token).getResults()) { - if (project.getStudies() != null) { - for (Study study : project.getStudies()) { - stopWatch.start(); - - int skip = 0; - boolean more = true; - do { - queryOptions.put(QueryOptions.SKIP, skip); - - OpenCGAResult search = catalogManager.getFamilyManager().search(study.getFqn(), new Query(), queryOptions, - token); - if (search.getNumResults() < 1000) { - more = false; - } - skip += search.getNumResults(); - - for (Family family : search.getResults()) { - try { - catalogManager.getFamilyManager().update(study.getFqn(), family.getId(), null, familyUpdateOptions, token); - logger.info("Updated roles from family '{}' - '{}'", study.getFqn(), family.getId()); - } catch (CatalogException e) { - continue; - } - } - - logger.info("Updated {} families in study '{}' in {} seconds", skip, study.getFqn(), - stopWatch.getTime(TimeUnit.SECONDS)); - } while (more); - - stopWatch.stop(); - stopWatch.reset(); - } - } - } - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_0_5/catalog/java/initialiseGroups.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_0_5/catalog/java/initialiseGroups.java deleted file mode 100644 index bdaf040d50e..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_0_5/catalog/java/initialiseGroups.java +++ /dev/null @@ -1,42 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_0_5.catalog.java; - - -import org.apache.commons.lang3.time.StopWatch; -import org.opencb.commons.datastore.core.Query; -import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; -import org.opencb.opencga.core.models.project.Project; -import org.opencb.opencga.core.models.study.Group; -import org.opencb.opencga.core.models.study.Study; - -@Migration(id = "initialise_groups", description = "Initialise userIds list from groups #1791", version = "2.0.5", date = 20210621) -public class initialiseGroups extends MigrationTool { - - @Override - protected void run() throws CatalogException { - StopWatch stopWatch = new StopWatch(); - // Initialise all list of users from groups - for (Project project : catalogManager.getProjectManager().search(new Query(), new QueryOptions(), token).getResults()) { - if (project.getStudies() != null) { - for (Study study : project.getStudies()) { - stopWatch.start(); - - for (Group group : study.getGroups()) { - if (group.getUserIds() == null) { - logger.info("Fixing group '{}' from study '{}'", group.getId(), study.getFqn()); - catalogManager.getStudyManager().deleteGroup(study.getFqn(), group.getId(), token); - - // Create it again. The new implementation takes now good care of the list of userIds - catalogManager.getStudyManager().createGroup(study.getFqn(), group, token); - } - } - - stopWatch.stop(); - stopWatch.reset(); - } - } - } - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_0_6/catalog/removeDeletedFileReferencesFromSample.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_0_6/catalog/removeDeletedFileReferencesFromSample.java deleted file mode 100644 index ff08748d412..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_0_6/catalog/removeDeletedFileReferencesFromSample.java +++ /dev/null @@ -1,79 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_0_6.catalog; - - -import com.mongodb.client.MongoCollection; -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.Updates; -import org.bson.Document; -import org.bson.conversions.Bson; -import org.opencb.opencga.catalog.db.api.FileDBAdaptor; -import org.opencb.opencga.catalog.db.api.SampleDBAdaptor; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import java.util.*; - -@Migration(id = "remove_file_references_from_sample", description = "Remove deleted file references from samples #1815", version = "2.0.6", - date = 20210901) -public class removeDeletedFileReferencesFromSample extends MigrationTool { - - @Override - protected void run() throws CatalogException { - // Obtain all file ids that have been deleted - Set deletedFileIds = new HashSet<>(); - queryMongo(MongoDBAdaptorFactory.DELETED_FILE_COLLECTION, new Document(), Projections.include("id"), - (file) -> deletedFileIds.add(file.getString("id")) - ); - - if (deletedFileIds.isEmpty()) { - return; - } - - List> batches = getBatches(deletedFileIds); - Set foundFiles = new HashSet<>(); - for (List batch : batches) { - queryMongo(MongoDBAdaptorFactory.FILE_COLLECTION, new Document(FileDBAdaptor.QueryParams.ID.key(), batch), - Projections.include("id"), (file) -> foundFiles.add(file.getString("id"))); - } - // Remove from the original list of deleted files the one that also exist in the main file collection (they've been replaced for - // new ones with same ids) - deletedFileIds.removeAll(foundFiles); - - if (deletedFileIds.isEmpty()) { - return; - } - - // Check and remove any of the deletedFileIds from the sample references - batches = getBatches(deletedFileIds); - MongoCollection sampleCollection = getMongoCollection(MongoDBAdaptorFactory.SAMPLE_COLLECTION); - for (List batch : batches) { - Bson update = Updates.pullAll(SampleDBAdaptor.QueryParams.FILE_IDS.key(), batch); - sampleCollection.updateMany(new Document(SampleDBAdaptor.QueryParams.FILE_IDS.key(), batch), update); - } - } - - private List> getBatches(Set fileIds) { - // Generate batches of 50 ids - int size = 50; - - List> batches = new LinkedList<>(); - List tmpList = null; - int counter = 0; - for (String fileId : fileIds) { - if (counter % size == 0) { - if (counter > 0) { - batches.add(tmpList); - } - tmpList = new ArrayList<>(); - } - tmpList.add(fileId); - - counter++; - } - batches.add(tmpList); - - return batches; - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_0/catalog/AddAnnotationSetsInClinicalAnalysisMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_0/catalog/AddAnnotationSetsInClinicalAnalysisMigration.java deleted file mode 100644 index 0ab4fceb81a..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_0/catalog/AddAnnotationSetsInClinicalAnalysisMigration.java +++ /dev/null @@ -1,43 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_12_0.catalog; - - -import com.mongodb.client.MongoCollection; -import com.mongodb.client.model.Filters; -import com.mongodb.client.result.UpdateResult; -import org.bson.Document; -import org.bson.conversions.Bson; -import org.opencb.opencga.catalog.db.mongodb.AnnotationMongoDBAdaptor; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import java.util.Arrays; -import java.util.Collections; - -@Migration(id = "add_annotation_sets_to_clinical_analysis" , - description = "Add private annotation fields to ClinicalAnalysis documents #TASK-5198", - version = "2.12.0", - domain = Migration.MigrationDomain.CATALOG, - language = Migration.MigrationLanguage.JAVA, - date = 20231116 -) -public class AddAnnotationSetsInClinicalAnalysisMigration extends MigrationTool { - - @Override - protected void run() throws Exception { - Bson query = Filters.exists(AnnotationMongoDBAdaptor.AnnotationSetParams.ANNOTATION_SETS.key(), false); - Document update = new Document("$set", new Document() - .append(AnnotationMongoDBAdaptor.AnnotationSetParams.ANNOTATION_SETS.key(), Collections.emptyList()) - .append(AnnotationMongoDBAdaptor.AnnotationSetParams.INTERNAL_ANNOTATION_SETS.key(), Collections.emptyList()) - .append(AnnotationMongoDBAdaptor.AnnotationSetParams.PRIVATE_VARIABLE_SET_MAP.key(), Collections.emptyMap()) - .append(AnnotationMongoDBAdaptor.AnnotationSetParams.PRIVATE_INTERNAL_VARIABLE_SET_MAP.key(), Collections.emptyMap()) - ); - // Initialise private fields in all Clinical Analysis documents - for (String collection : Arrays.asList(MongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, - MongoDBAdaptorFactory.DELETED_CLINICAL_ANALYSIS_COLLECTION)) { - MongoCollection mongoCollection = getMongoCollection(collection); - UpdateResult updateResult = mongoCollection.updateMany(query, update); - logger.info("{} clinical analysis documents updated from the {} collection", updateResult.getModifiedCount(), collection); - } - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_0/catalog/CompleteClinicalReportDataModelMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_0/catalog/CompleteClinicalReportDataModelMigration.java deleted file mode 100644 index f7e6123b223..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_0/catalog/CompleteClinicalReportDataModelMigration.java +++ /dev/null @@ -1,62 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_12_0.catalog; - -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptor; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import java.util.Arrays; -import java.util.Collections; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "complete_clinical_report_data_model" , - description = "Complete Clinical Report data model #TASK-5198", - version = "2.12.0", - domain = Migration.MigrationDomain.CATALOG, - language = Migration.MigrationLanguage.JAVA, - date = 20231128 -) -public class CompleteClinicalReportDataModelMigration extends MigrationTool { - - @Override - protected void run() throws Exception { - migrateCollection( - Arrays.asList(MongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, MongoDBAdaptorFactory.DELETED_CLINICAL_ANALYSIS_COLLECTION), - Filters.exists("analyst"), - Projections.include(Arrays.asList("analyst", "report")), - (document, bulk) -> { - Document analyst = document.get("analyst", Document.class); - analyst.remove("assignedBy"); - analyst.remove("date"); - - Document report = document.get("report", Document.class); - if (report != null) { - report.put("comments", Collections.emptyList()); - report.put("supportingEvidences", Collections.emptyList()); - report.put("files", Collections.emptyList()); - } - - MongoDBAdaptor.UpdateDocument updateDocument = new MongoDBAdaptor.UpdateDocument(); - updateDocument.getSet().put("report", report); - updateDocument.getSet().put("request", new Document()); - updateDocument.getSet().put("responsible", new Document() - .append("id", analyst.get("id")) - .append("name", analyst.get("name")) - .append("email", analyst.get("email")) - ); - updateDocument.getSet().put("analysts", Collections.singletonList(analyst)); - updateDocument.getUnset().add("analyst"); - - bulk.add(new UpdateOneModel<>( - eq("_id", document.get("_id")), - updateDocument.toFinalUpdateDocument() - ) - ); - }); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_4/catalog/FixStatusIndexesMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_4/catalog/FixStatusIndexesMigration.java deleted file mode 100644 index bce2551a0f5..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_4/catalog/FixStatusIndexesMigration.java +++ /dev/null @@ -1,46 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_12_4.catalog; - -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import java.util.Arrays; - -@Migration(id = "fix_status_indexes" , - description = "Replace 'status.name' indexes for 'status.id'", - version = "2.12.4", - domain = Migration.MigrationDomain.CATALOG, - language = Migration.MigrationLanguage.JAVA, - date = 20240328 -) -public class FixStatusIndexesMigration extends MigrationTool { - - @Override - protected void run() throws Exception { - Document statusIndex = new Document() - .append("status.name", 1) - .append("studyUid", 1); - dropIndex(Arrays.asList(MongoDBAdaptorFactory.JOB_COLLECTION, MongoDBAdaptorFactory.FILE_COLLECTION, - MongoDBAdaptorFactory.SAMPLE_COLLECTION, MongoDBAdaptorFactory.SAMPLE_ARCHIVE_COLLECTION, - MongoDBAdaptorFactory.COHORT_COLLECTION, MongoDBAdaptorFactory.INDIVIDUAL_COLLECTION, - MongoDBAdaptorFactory.INDIVIDUAL_ARCHIVE_COLLECTION, MongoDBAdaptorFactory.FAMILY_COLLECTION, - MongoDBAdaptorFactory.FAMILY_ARCHIVE_COLLECTION, MongoDBAdaptorFactory.PANEL_COLLECTION, - MongoDBAdaptorFactory.PANEL_ARCHIVE_COLLECTION), statusIndex); - - Document internalStatusIndex = new Document() - .append("internal.status.name", 1) - .append("studyUid", 1); - dropIndex(Arrays.asList(MongoDBAdaptorFactory.JOB_COLLECTION, MongoDBAdaptorFactory.FILE_COLLECTION, - MongoDBAdaptorFactory.SAMPLE_COLLECTION, MongoDBAdaptorFactory.SAMPLE_ARCHIVE_COLLECTION, - MongoDBAdaptorFactory.COHORT_COLLECTION, MongoDBAdaptorFactory.INDIVIDUAL_COLLECTION, - MongoDBAdaptorFactory.INDIVIDUAL_ARCHIVE_COLLECTION, MongoDBAdaptorFactory.FAMILY_COLLECTION, - MongoDBAdaptorFactory.FAMILY_ARCHIVE_COLLECTION, MongoDBAdaptorFactory.PANEL_COLLECTION, - MongoDBAdaptorFactory.PANEL_ARCHIVE_COLLECTION, MongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, - MongoDBAdaptorFactory.INTERPRETATION_COLLECTION, MongoDBAdaptorFactory.INTERPRETATION_ARCHIVE_COLLECTION), - internalStatusIndex); - - catalogManager.installIndexes(token); - } - -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/catalog/java/AddMissingIndexes.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/catalog/java/AddMissingIndexes.java deleted file mode 100644 index f0773de6bf5..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/catalog/java/AddMissingIndexes.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_1_0.catalog.java; - -import com.mongodb.client.MongoCollection; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -@Migration(id = "add_missing_indexes", - description = "Add missing indexes", version = "2.1.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - patch = 1, - date = 20210928) -public class AddMissingIndexes extends MigrationTool { - - @Override - protected void run() throws Exception { - catalogManager.installIndexes(token); - - // Delete old indexes - MongoCollection sampleCollection = getMongoCollection(MongoDBAdaptorFactory.SAMPLE_COLLECTION); - sampleCollection.dropIndex("studyUid_1__acl_1__lastOfVersion_1"); - - // Delete old indexes - MongoCollection interpretationCollection = getMongoCollection(MongoDBAdaptorFactory.INTERPRETATION_COLLECTION); - interpretationCollection.dropIndex("analyst.name_1_studyUid_1"); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/catalog/java/AddPanelsToInterpretations.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/catalog/java/AddPanelsToInterpretations.java deleted file mode 100644 index 3a853fb05dd..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/catalog/java/AddPanelsToInterpretations.java +++ /dev/null @@ -1,115 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_1_0.catalog.java; - -import com.mongodb.client.MongoCollection; -import com.mongodb.client.MongoCursor; -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.bson.conversions.Bson; -import org.opencb.opencga.catalog.db.api.ClinicalAnalysisDBAdaptor; -import org.opencb.opencga.catalog.db.api.InterpretationDBAdaptor; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; -import org.opencb.opencga.core.common.BatchUtils; - -import java.util.*; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "add_panels_to_interpretations", - description = "Add panels to Interpretations #1802", version = "2.1.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20210713) -public class AddPanelsToInterpretations extends MigrationTool { - - @Override - protected void run() throws Exception { - MongoCollection clinicalCollection = getMongoCollection(MongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION); - - /* - 1. Check if there is any orphan interpretation (attached to an unknown clinical analysis) - */ - // Obtain all clinicalAnalysisIds (study - [clinicalIds]) - Map> clinicalMap = new HashMap<>(); - queryMongo(MongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, new Document(), - Projections.include(Arrays.asList("id", "studyUid")), (doc) -> { - String id = doc.getString("id"); - String studyUid = String.valueOf(doc.getLong("studyUid")); - - if (!clinicalMap.containsKey(studyUid)) { - clinicalMap.put(studyUid, new HashSet<>()); - } - clinicalMap.get(studyUid).add(id); - }); - - List interpretationsToRemove = new LinkedList<>(); - // Iterate over all interpretations and remove orphan ones - queryMongo(MongoDBAdaptorFactory.INTERPRETATION_COLLECTION, new Document(), - Projections.include(Arrays.asList("_id", "clinicalAnalysisId", "studyUid")), (doc) -> { - Object interpretationId = doc.get("_id"); - String clinicalId = doc.getString("clinicalAnalysisId"); - String studyUid = String.valueOf(doc.getLong("studyUid")); - - if (!clinicalMap.containsKey(studyUid) || !clinicalMap.get(studyUid).contains(clinicalId)) { - interpretationsToRemove.add(interpretationId); - } - }); - - if (!interpretationsToRemove.isEmpty()) { - MongoCollection interpretationCollection = getMongoCollection(MongoDBAdaptorFactory.INTERPRETATION_COLLECTION); - List> bulkList = BatchUtils.splitBatches(interpretationsToRemove, 100); - logger.info("Removing " + interpretationsToRemove.size() + " orphan interpretations..."); - for (List list : bulkList) { - interpretationCollection.deleteMany(Filters.in("_id", list)); - } - } - - /* - 2. Add panels to Interpretations - */ - migrateCollection(MongoDBAdaptorFactory.INTERPRETATION_COLLECTION, - new Document(InterpretationDBAdaptor.QueryParams.PANELS.key(), new Document("$exists", false)), - Projections.include("_id", InterpretationDBAdaptor.QueryParams.ID.key(), - InterpretationDBAdaptor.QueryParams.CLINICAL_ANALYSIS_ID.key(), - InterpretationDBAdaptor.QueryParams.STUDY_UID.key()), - (interpretation, bulk) -> { - - // Get panels from related Clinical Analysis - String interpretationId = interpretation.getString(InterpretationDBAdaptor.QueryParams.ID.key()); - String clinicalId = interpretation.getString(InterpretationDBAdaptor.QueryParams.CLINICAL_ANALYSIS_ID.key()); - long studyUid = interpretation.getLong(InterpretationDBAdaptor.QueryParams.STUDY_UID.key()); - - Bson query = Filters.and( - Filters.eq(ClinicalAnalysisDBAdaptor.QueryParams.STUDY_UID.key(), studyUid), - Filters.eq(ClinicalAnalysisDBAdaptor.QueryParams.ID.key(), clinicalId) - ); - Bson projection = Projections.include("_id", ClinicalAnalysisDBAdaptor.QueryParams.PANELS.key()); - - try (MongoCursor cursor = clinicalCollection.find(query).projection(projection).cursor()) { - if (cursor.hasNext()) { - Document clinicalDocument = cursor.next(); - if (cursor.hasNext()) { - - throw new RuntimeException("Found more than one Clinical Analysis '" + clinicalId + "' for Interpretation '" - + interpretationId + "' from " + "studyUid " + studyUid); - } - - List panels = clinicalDocument.getList(ClinicalAnalysisDBAdaptor.QueryParams.PANELS.key(), Document.class); - // Set same panels from ClinicalAnalysis to the Interpretation - bulk.add(new UpdateOneModel<>( - eq("_id", interpretation.get("_id")), - new Document("$set", new Document(InterpretationDBAdaptor.QueryParams.PANELS.key(), panels)) - ) - ); - } else { - throw new RuntimeException("Could not find Clinical Analysis '" + clinicalId + "' for Interpretation '" - + interpretationId + "' from " + "studyUid " + studyUid); - } - } - } - ); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/catalog/java/PanelLock.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/catalog/java/PanelLock.java deleted file mode 100644 index 6cf27668e0b..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/catalog/java/PanelLock.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_1_0.catalog.java; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.db.api.ClinicalAnalysisDBAdaptor; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "add_panel_lock", - description = "Add new panelLock to ClinicalAnalysis #1802", version = "2.1.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - patch = 1, - date = 20210713) -public class PanelLock extends MigrationTool { - - @Override - protected void run() throws Exception { - migrateCollection(MongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, - new Document(ClinicalAnalysisDBAdaptor.QueryParams.PANEL_LOCK.key(), new Document("$exists", false)), - Projections.include("_id"), - (clinical, bulk) -> { - bulk.add(new UpdateOneModel<>( - eq("_id", clinical.get("_id")), - new Document("$set", new Document(ClinicalAnalysisDBAdaptor.QueryParams.PANEL_LOCK.key(), false)) - ) - ); - } - ); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/catalog/java/RemoveRCVersionsFromMigrationRuns.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/catalog/java/RemoveRCVersionsFromMigrationRuns.java deleted file mode 100644 index 762d27686b9..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/catalog/java/RemoveRCVersionsFromMigrationRuns.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_1_0.catalog.java; - -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import static com.mongodb.client.model.Filters.eq; -import static com.mongodb.client.model.Updates.set; - -@Migration(id = "remove_rc_version_from_migration_runs", - description = "Remove RC versions from migration runs stored in catalog", version = "2.1.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - patch = 1, - date = 20210723) -public class RemoveRCVersionsFromMigrationRuns extends MigrationTool { - - @Override - protected void run() throws Exception { - migrateCollection(MongoDBAdaptorFactory.MIGRATION_COLLECTION, new Document(), new Document(), (document, bulk) -> { - String version = document.getString("version"); - if (version.toUpperCase().contains("-RC")) { - String newVersion = version.split("-RC")[0]; - logger.info("Change version for migration " + document.getString("id") + " : " + version + " -> " + newVersion); - bulk.add(new UpdateOneModel<>(eq("_id", document.get("_id")), set("version", newVersion))); - } - }); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/catalog/java/RenameDatastoreConfigurationToOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/catalog/java/RenameDatastoreConfigurationToOptions.java deleted file mode 100644 index b7cf705cf1f..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/catalog/java/RenameDatastoreConfigurationToOptions.java +++ /dev/null @@ -1,52 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_1_0.catalog.java; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import com.mongodb.client.model.Updates; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import java.util.List; - -import static com.mongodb.client.model.Filters.and; -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "rename_datastore_configuration_to_options", description = "Rename project.internal.datastores.variant.configuration to options", version = "2.1.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - patch = 1, - date = 20210617) -public class RenameDatastoreConfigurationToOptions extends MigrationTool { - - @Override - protected void run() throws Exception { - migrateCollection(MongoDBAdaptorFactory.USER_COLLECTION, new Document(), Projections.include("_id", "projects.internal", "projects.uuid"), - (user, bulk) -> { - List projects = user.getList("projects", Document.class); - for (Document project : projects) { - Document configuration = get(project, "internal", "datastores", "variant", "configuration"); - Document options = get(project, "internal", "datastores", "variant", "options"); - if (configuration != null) - // Do not overwrite options if present! - if (options == null) { - bulk.add(new UpdateOneModel<>( - and(eq("_id", user.get("_id")), eq("projects.uuid", project.get("uuid"))), - Updates.set("projects.$.internal.datastores.variant.options", configuration))); - } - } - } - ); - } - - protected Document get(Document document, String... keys) { - for (String key : keys) { - document = document.get(key, Document.class); - if (document == null) { - return null; - } - } - return document; - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/catalog/java/StudyClinicalConfigurationRelocation.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/catalog/java/StudyClinicalConfigurationRelocation.java deleted file mode 100644 index e2aa0583d21..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/catalog/java/StudyClinicalConfigurationRelocation.java +++ /dev/null @@ -1,51 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_1_0.catalog.java; - - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "move_study_clinical_config_to_internal", - description = "Move Study ClinicalConfiguration to internal.configuration", version = "2.1.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - patch = 1, - date = 20210708) -public class StudyClinicalConfigurationRelocation extends MigrationTool { - - @Override - protected void run() throws Exception { - migrateCollection(MongoDBAdaptorFactory.STUDY_COLLECTION, - new Document("configuration", new Document("$exists", true)), - Projections.include("_id", "configuration", "internal"), - (study, bulk) -> { - Document configuration = study.get("configuration", Document.class); - Document internal = study.get("internal", Document.class); - - Document variantEngineConfiguration = internal.get("variantEngineConfiguration", Document.class); - - // Remove variantEngineConfiguration from internal - internal.remove("variantEngineConfiguration"); - - // Add variantEngineConfiguration to configuration as variantEngine - configuration.put("variantEngine", variantEngineConfiguration); - - // Add configuration to internal - internal.put("configuration", configuration); - - bulk.add(new UpdateOneModel<>( - eq("_id", study.get("_id")), - new Document() - .append("$set", new Document("internal", internal)) - .append("$unset", new Document("configuration", "")) - ) - ); - } - ); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/catalog/java/VariantFileStatsRelocation.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/catalog/java/VariantFileStatsRelocation.java deleted file mode 100644 index ee6c7daaaee..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/catalog/java/VariantFileStatsRelocation.java +++ /dev/null @@ -1,70 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_1_0.catalog.java; - -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.opencb.biodata.models.variant.metadata.VariantSetStats; -import org.opencb.commons.datastore.core.Query; -import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.catalog.db.api.DBIterator; -import org.opencb.opencga.catalog.db.api.FileDBAdaptor; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; -import org.opencb.opencga.core.models.common.AnnotationSet; -import org.opencb.opencga.core.models.file.File; -import org.opencb.opencga.core.models.file.FileQualityControl; -import org.opencb.opencga.core.models.file.FileUpdateParams; -import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.models.variant.VariantFileQualityControl; - -import java.util.Arrays; - -@Migration(id = "move_variant_file_stats_to_qc", description = "Move opencga_file_variant_stats annotation set from variable sets to " + - "FileQualityControl", version = "2.1.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.STORAGE, - patch = 1, - date = 20210614) -public class VariantFileStatsRelocation extends MigrationTool { - - private static final String FILE_VARIANT_STATS_VARIABLE_SET = "opencga_file_variant_stats"; - - @Override - protected void run() throws Exception { - for (Study study : catalogManager.getStudyManager().search(new Query(), new QueryOptions(), token).getResults()) { - Query query = new Query(FileDBAdaptor.QueryParams.FORMAT.key(), File.Format.VCF); - QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, - Arrays.asList( - FileDBAdaptor.QueryParams.ID.key(), - FileDBAdaptor.QueryParams.QUALITY_CONTROL.key(), - FileDBAdaptor.QueryParams.ANNOTATION_SETS.key() - )); - - ObjectMapper objectMapper = new ObjectMapper(new JsonFactory()); - logger.info("Updating files from study {}", study.getFqn()); - try (DBIterator it = catalogManager.getFileManager().iterator(study.getFqn(), query, new QueryOptions(options), token)) { - while (it.hasNext()) { - File file = it.next(); - if (file.getQualityControl() != null - && file.getQualityControl().getVariant() != null - && file.getQualityControl().getVariant().getVariantSetMetrics() != null) { - logger.info("Variant stats from file {} already relocated.", file.getId()); - continue; - } - VariantSetStats variantSetStats = null; - for (AnnotationSet annotationSet : file.getAnnotationSets()) { - if (annotationSet.getVariableSetId().equals(FILE_VARIANT_STATS_VARIABLE_SET)) { - variantSetStats = objectMapper.convertValue(annotationSet.getAnnotations(), VariantSetStats.class); - } - } - if (variantSetStats != null) { - logger.info("Relocating variant stats from file {}", file.getId()); - catalogManager.getFileManager().update(study.getFqn(), file.getId(), - new FileUpdateParams().setQualityControl( - new FileQualityControl().setVariant(new VariantFileQualityControl(variantSetStats, null))), - new QueryOptions(options), token); - } - } - } - } - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/storage/AddCellbaseConfigurationToProject.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/storage/AddCellbaseConfigurationToProject.java deleted file mode 100644 index 21959475f7a..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/storage/AddCellbaseConfigurationToProject.java +++ /dev/null @@ -1,36 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_1_0.storage; - - -import org.opencb.commons.datastore.core.Query; -import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; -import org.opencb.opencga.core.config.storage.StorageConfiguration; -import org.opencb.opencga.core.models.project.Project; - -import java.util.Collections; - -import static org.opencb.opencga.core.api.ParamConstants.USER_PROJECT_SEPARATOR; - -@Migration(id = "add_cellbase_configuration_to_project", description = "Add cellbase configuration from storage-configuration.yml to project.internal.cellbase", version = "2.1.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.STORAGE, - patch = 3, - date = 20210616) -public class AddCellbaseConfigurationToProject extends MigrationTool { - - @Override - protected void run() throws Exception { - StorageConfiguration storageConfiguration = readStorageConfiguration(); -// StorageEngineFactory engineFactory = StorageEngineFactory.get(storageConfiguration); -// VariantStorageManager variantStorageManager = new VariantStorageManager(catalogManager, engineFactory); - - for (Project project : catalogManager.getProjectManager().search(new Query(), QueryOptions.empty(), token).getResults()) { - if (project.getInternal() == null || project.getCellbase() == null) { - String userToken = catalogManager.getUserManager().getNonExpiringToken(project.getFqn().split(USER_PROJECT_SEPARATOR)[0], Collections.emptyMap(), token); - catalogManager.getProjectManager() - .setCellbaseConfiguration(project.getFqn(), storageConfiguration.getCellbase(), false, userToken); - } - } - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/storage/DefaultSampleIndexConfiguration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/storage/DefaultSampleIndexConfiguration.java deleted file mode 100644 index 301b510266b..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/storage/DefaultSampleIndexConfiguration.java +++ /dev/null @@ -1,102 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_1_0.storage; - - -import org.apache.commons.collections4.CollectionUtils; -import org.opencb.commons.datastore.core.Query; -import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; -import org.opencb.opencga.app.migrations.StorageMigrationTool; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.core.config.storage.IndexFieldConfiguration; -import org.opencb.opencga.core.config.storage.SampleIndexConfiguration; -import org.opencb.opencga.core.models.project.DataStore; -import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.models.study.StudyVariantEngineConfiguration; -import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; -import org.opencb.opencga.storage.core.variant.VariantStorageEngine; - -import java.time.Instant; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; - -@Migration(id = "default_sample_index_configuration", description = "Add a default backward compatible sample index configuration", version = "2.1.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.STORAGE, - patch = 7, - date = 20210721) // Needs to run after StudyClinicalConfigurationRelocation -public class DefaultSampleIndexConfiguration extends StorageMigrationTool { - - @Override - protected void run() throws Exception { - VariantStorageManager variantStorageManager = getVariantStorageManager(); - - for (Study study : catalogManager.getStudyManager().search(new Query(), new QueryOptions(QueryOptions.INCLUDE, Arrays.asList("fqn", "internal")), token).getResults()) { - if (variantStorageManager.exists(study.getFqn(), token)) { - DataStore dataStore = variantStorageManager.getDataStore(study.getFqn(), token); - VariantStorageEngine engine = getVariantStorageEngineFactory() - .getVariantStorageEngine(dataStore.getStorageEngine(), dataStore.getDbName()); - StudyMetadata metadata = engine.getMetadataManager().getStudyMetadata(study.getFqn()); - if (study.getInternal().getConfiguration().getVariantEngine() == null) { - study.getInternal().getConfiguration().setVariantEngine(new StudyVariantEngineConfiguration()); - } - - boolean updateConfiguration = false; - - SampleIndexConfiguration sampleIndexConfiguration; - if (CollectionUtils.isEmpty(metadata.getSampleIndexConfigurations())) { - updateConfiguration = true; - sampleIndexConfiguration = SampleIndexConfiguration.backwardCompatibleConfiguration(); - sampleIndexConfiguration.validate(); - } else { - logger.info("Study {} already has a SampleIndex configuration", study.getFqn()); - sampleIndexConfiguration = metadata.getSampleIndexConfigurationLatest().getConfiguration(); - - if (study.getInternal().getConfiguration().getVariantEngine().getSampleIndex() == null) { - // Missing sample index configuration in catalog - updateConfiguration = true; - } else { - SampleIndexConfiguration catalogSampleIndexConfiguration - = study.getInternal().getConfiguration().getVariantEngine().getSampleIndex(); - // Different sample index configuration - if (!catalogSampleIndexConfiguration.equals(sampleIndexConfiguration)) { - updateConfiguration = true; - } - } - if (sampleIndexConfiguration.getAnnotationIndexConfiguration().getTranscriptFlagIndexConfiguration() == null) { - logger.info("Missing transcriptFlag"); - updateConfiguration = true; - sampleIndexConfiguration.getAnnotationIndexConfiguration().setTranscriptFlagIndexConfiguration( - new IndexFieldConfiguration(IndexFieldConfiguration.Source.ANNOTATION, "transcriptFlag", - IndexFieldConfiguration.Type.CATEGORICAL_MULTI_VALUE, "invalid_transcript_flag_index")); - } - if (sampleIndexConfiguration.getAnnotationIndexConfiguration().getTranscriptCombination() == null) { - // If null, this combination was not computed. - sampleIndexConfiguration.getAnnotationIndexConfiguration().setTranscriptCombination(false); - updateConfiguration = true; - } - } - - if (!updateConfiguration) { - logger.info("Skip study"); - continue; - } - - engine.getMetadataManager().updateStudyMetadata(study.getFqn(), studyMetadata -> { - if (CollectionUtils.isEmpty(studyMetadata.getSampleIndexConfigurations())) { - studyMetadata.setSampleIndexConfigurations(new ArrayList<>()); - studyMetadata.getSampleIndexConfigurations().add( - new StudyMetadata.SampleIndexConfigurationVersioned(sampleIndexConfiguration, 1, Date.from(Instant.now()), - StudyMetadata.SampleIndexConfigurationVersioned.Status.ACTIVE)); - } else { - int size = studyMetadata.getSampleIndexConfigurations().size(); - studyMetadata.getSampleIndexConfigurations().get(size - 1).setConfiguration(sampleIndexConfiguration); - } - return studyMetadata; - }); - catalogManager.getStudyManager() - .setVariantEngineConfigurationSampleIndex(study.getFqn(), sampleIndexConfiguration, token); - } - } - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/storage/NewClinicalSignificanceFields.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/storage/NewClinicalSignificanceFields.java deleted file mode 100644 index dc2db8de001..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_1_0/storage/NewClinicalSignificanceFields.java +++ /dev/null @@ -1,67 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_1_0.storage; - -import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.analysis.variant.operations.VariantAnnotationRebuilderOperationTool; -import org.opencb.opencga.app.migrations.StorageMigrationTool; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationRun; -import org.opencb.opencga.core.api.ParamConstants; -import org.opencb.opencga.core.models.common.Enums; -import org.opencb.opencga.core.models.job.Job; -import org.opencb.opencga.core.models.job.JobReferenceParam; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Map; - - -@Migration(id = "new_clinical_significance_fields", description = "Add new clinical significance fields and combinations for variant storage and solr", version = "2.1.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.STORAGE, - patch = 1, - date = 20210708) -public class NewClinicalSignificanceFields extends StorageMigrationTool { - - @Override - protected void run() throws Exception { -// getVariantStorageManager().getStorageConfiguration().getVariantEngine().getEngine().equals("hadoop"); - MigrationRun migrationRun = getMigrationRun(); - Map jobs = new HashMap<>(); - for (JobReferenceParam jobReference : migrationRun.getJobs()) { - Job job = catalogManager.getJobManager().get(jobReference.getStudyId(), jobReference.getId(), new QueryOptions(), token) - .first(); - jobs.put(job.getParams().get(ParamConstants.PROJECT_PARAM).toString(), job); - } - for (String project : getVariantStorageProjects()) { - Job job = jobs.get(project); - if (job != null) { - int patch = Integer.parseInt(job.getParams().get("patch").toString()); - String status = job.getInternal().getStatus().getId(); - if (status.equals(Enums.ExecutionStatus.DONE)) { - if (patch == getAnnotation().patch()) { - // Skip this project. Already migrated - logger.info("Project {} already migrated", project); - continue; - } else { - logger.info("Rerun job, as the patch has changed."); - } - } else if (status.equals(Enums.ExecutionStatus.ERROR) || status.equals(Enums.ExecutionStatus.ABORTED)) { - logger.info("Retry migration job for project {}", project); - } else { - logger.info("Job {} for migrating project {} in status {}. Wait for completion", job.getId(), project, status); - continue; - } - getMigrationRun().removeJob(job); - } - - ObjectMap params = new ObjectMap() - .append(ParamConstants.PROJECT_PARAM, project) - .append("patch", getAnnotation().patch()); - getMigrationRun().addJob(catalogManager.getJobManager().submitProject(project, VariantAnnotationRebuilderOperationTool.ID, Enums.Priority.MEDIUM, params, - null, null, null, new ArrayList<>(), token).first()); -// VariantStorageEngine storageEngine = getVariantStorageEngineByProject(project); -// getVariantStorageManager() - } - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/AddMissingClinicalAudit.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/AddMissingClinicalAudit.java deleted file mode 100644 index 7c7a91336cc..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/AddMissingClinicalAudit.java +++ /dev/null @@ -1,114 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog; - -import com.mongodb.client.MongoCollection; -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.Updates; -import org.apache.commons.lang3.StringUtils; -import org.bson.Document; -import org.opencb.biodata.models.clinical.ClinicalAudit; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptor; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.db.mongodb.converters.ClinicalAnalysisConverter; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; -import org.opencb.opencga.core.common.TimeUtils; -import org.opencb.opencga.core.models.clinical.ClinicalAnalysis; - -import java.util.*; - -@Migration(id = "add_missing_create_interpretation_in_clinical_audit", - description = "Add missing CREATE_INTERPRETATION audits in ClinicalAnalysis", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20211227) -public class AddMissingClinicalAudit extends MigrationTool { - - private final static String SEPARATOR = "____"; - - @Override - protected void run() throws Exception { - Map> clinicalInterpretationsMap = new HashMap<>(); - - // Extract map of ClinicalAnalysisId - List - queryMongo(MongoDBAdaptorFactory.INTERPRETATION_COLLECTION, new Document(MongoDBAdaptor.LAST_OF_VERSION, true), - Projections.include(Arrays.asList("id", "clinicalAnalysisId", "studyUid")), (doc) -> { - String id = doc.getString("id"); - String clinicalAnalysisId = doc.getString("clinicalAnalysisId"); - String studyUid = String.valueOf(doc.getLong("studyUid")); - String key = clinicalAnalysisId + SEPARATOR + studyUid; - if (!clinicalInterpretationsMap.containsKey(key)) { - clinicalInterpretationsMap.put(key, new LinkedList<>()); - } - clinicalInterpretationsMap.get(key).add(id); - }); - - // Calculate map of ClinicalAnalysisId - MaximumInterpretationIdCount - Map clinicalAuditCount = extractLastInterpretationCounter(clinicalInterpretationsMap); - - String time = TimeUtils.getTime(); - ClinicalAnalysisConverter converter = new ClinicalAnalysisConverter(); - MongoCollection clinicalCollection = getMongoCollection(MongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION); - for (Map.Entry entry : clinicalAuditCount.entrySet()) { - logger.debug("Processing entry key '{}'...", entry.getKey()); - String[] split = entry.getKey().split(SEPARATOR); - String clinicalAnalysisId = split[0]; - long studyUid = Long.parseLong(split[1]); - - Integer createInterpretationCount = entry.getValue(); - - queryMongo(MongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, - new Document("studyUid", studyUid).append("id", clinicalAnalysisId), - Projections.include("audit"), - (doc) -> { - ClinicalAnalysis clinicalAnalysis = converter.convertToDataModelType(doc); - - int count = 0; - for (ClinicalAudit clinicalAudit : clinicalAnalysis.getAudit()) { - if (clinicalAudit.getAction().equals(ClinicalAudit.Action.CREATE_INTERPRETATION)) { - count++; - } - } - - if (createInterpretationCount > count) { - List auditList = new ArrayList<>(clinicalAnalysis.getAudit()); - - while (count < createInterpretationCount) { - auditList.add(new ClinicalAudit("opencga", ClinicalAudit.Action.CREATE_INTERPRETATION, - "Migration add_missing_create_interpretation_in_clinical_audit", time)); - count++; - } - - List auditDocumentList = convertToDocument(auditList); - clinicalCollection.updateOne(new Document("studyUid", studyUid).append("id", clinicalAnalysisId), - Updates.set("audit", auditDocumentList)); - } - }); - } - } - - private Map extractLastInterpretationCounter(Map> clinicalInterpretationsMap) { - Map clinicalAuditCount = new HashMap<>(); - - for (Map.Entry> entry : clinicalInterpretationsMap.entrySet()) { - int size = entry.getValue().size(); - - for (String interpretationId : entry.getValue()) { - String[] split = StringUtils.split(interpretationId, "."); - String value = split[split.length - 1]; - try { - int intValue = Integer.parseInt(value); - if (intValue > size) { - size = intValue; - } - } catch (NumberFormatException e) { - // Ignore error - continue; - } - } - - clinicalAuditCount.put(entry.getKey(), size); - } - - return clinicalAuditCount; - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/AddNameFieldInCohort.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/AddNameFieldInCohort.java deleted file mode 100644 index 1701f31ddb9..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/AddNameFieldInCohort.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog; - -import com.mongodb.client.MongoCollection; -import com.mongodb.client.model.IndexOptions; -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "add_name_field_in_cohort_1902", - description = "Add new name field to Cohort #1902", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20220228) -public class AddNameFieldInCohort extends MigrationTool { - - @Override - protected void run() throws Exception { - // Add new index - MongoCollection collection = getMongoCollection(MongoDBAdaptorFactory.COHORT_COLLECTION); - collection.createIndex(new Document().append("name", 1).append("studyUid", 1), new IndexOptions().background(true)); - - // Set name = id - migrateCollection(MongoDBAdaptorFactory.COHORT_COLLECTION, - new Document("name", new Document("$exists", false)), - Projections.include("_id", "id"), - (doc, bulk) -> bulk.add(new UpdateOneModel<>( - eq("_id", doc.get("_id")), - new Document("$set", new Document("name", doc.get("id"))) - ) - )); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/AddNewAllowedBiotype.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/AddNewAllowedBiotype.java deleted file mode 100644 index 50336e72f60..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/AddNewAllowedBiotype.java +++ /dev/null @@ -1,61 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.db.mongodb.converters.StudyConverter; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; -import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.models.study.Variable; -import org.opencb.opencga.core.models.study.VariableSet; - -import java.util.ArrayList; -import java.util.List; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "add_new_allowed_biotype", - description = "Add new allowed biotype 'guide_RNA', #1856", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - patch = 2, - date = 20211209) -public class AddNewAllowedBiotype extends MigrationTool { - - @Override - protected void run() throws Exception { - StudyConverter converter = new StudyConverter(); - migrateCollection(MongoDBAdaptorFactory.STUDY_COLLECTION, - new Document(), - Projections.include("_id", "variableSets"), - (doc, bulk) -> { - Study study = converter.convertToDataModelType(doc); - for (VariableSet variableSet : study.getVariableSets()) { - if ("opencga_sample_variant_stats".equals(variableSet.getId())) { - for (Variable variable : variableSet.getVariables()) { - if ("biotypeCount".equals(variable.getId())) { - List allowedKeys = new ArrayList<>(variable.getAllowedKeys()); - if (!allowedKeys.contains("guide_RNA")) { - allowedKeys.add("guide_RNA"); - variable.setAllowedKeys(allowedKeys); - } - if (!allowedKeys.contains("vault_RNA")) { - allowedKeys.add("vault_RNA"); - variable.setAllowedKeys(allowedKeys); - } - } - } - } - } - - List variableSetDocumentList = convertToDocument(study.getVariableSets()); - bulk.add(new UpdateOneModel<>( - eq("_id", doc.get("_id")), - new Document("$set", new Document("variableSets", variableSetDocumentList))) - ); - } - ); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/AddNewFileInternalIndex_1850.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/AddNewFileInternalIndex_1850.java deleted file mode 100644 index 1cc50d4057d..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/AddNewFileInternalIndex_1850.java +++ /dev/null @@ -1,152 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog; - -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.managers.FileManager; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; -import org.opencb.opencga.core.models.common.InternalStatus; -import org.opencb.opencga.core.models.file.File; -import org.opencb.opencga.core.models.file.FileInternalAlignment; -import org.opencb.opencga.core.models.file.FileInternalVariant; -import org.opencb.opencga.core.response.OpenCGAResult; - -import java.util.concurrent.atomic.AtomicInteger; - -import static com.mongodb.client.model.Filters.*; -import static com.mongodb.client.model.Projections.include; - -@Migration(id = "new_file_internal_index_1850", - description = "Add new FileInternalVariant and FileInternalAlignment index #1850", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20211127) -public class AddNewFileInternalIndex_1850 extends MigrationTool { - - @Override - protected void run() throws Exception { - final AtomicInteger filesWithNewInternalField = new AtomicInteger(); - final AtomicInteger filesWithoutNewInternalField = new AtomicInteger(); - - migrateCollection(MongoDBAdaptorFactory.FILE_COLLECTION, - and( - in("bioformat", File.Bioformat.VARIANT.toString(), File.Bioformat.ALIGNMENT.toString()), - exists("internal") - ), - include("_id", "internal", "bioformat"), - (doc, bulk) -> { - File.Bioformat bioformat = File.Bioformat.valueOf(doc.getString("bioformat")); - Document internal = doc.get("internal", Document.class); - Document index = internal.get("index", Document.class); - - if (internal.containsKey("variant") || internal.containsKey("alignment")) { - // Skip file. It already has the new internal field. - filesWithNewInternalField.incrementAndGet(); - return; - } else { - filesWithoutNewInternalField.incrementAndGet(); - } - - FileInternalVariant variant = FileInternalVariant.init(); - FileInternalAlignment alignment = FileInternalAlignment.init(); - - if (index != null) { - Number release = index.get("release", Number.class); - if (release != null) { - variant.getIndex().setRelease(release.intValue()); - } - - Document status = index.get("status", Document.class); - if (status != null) { - InternalStatus internalStatus = null; - if (bioformat == File.Bioformat.VARIANT) { - internalStatus = variant.getIndex().getStatus(); - } else if (bioformat == File.Bioformat.ALIGNMENT) { - // Initialise new Alignment internal status - internalStatus = new InternalStatus(InternalStatus.READY); - alignment.getIndex().setStatus(internalStatus); - } - - if (internalStatus != null) { - String statusId = status.getString("name"); - String description = status.getString("description"); - String date = status.getString("date"); - - if (statusId != null) { - internalStatus.setId(statusId); - internalStatus.setName(statusId); - } - if (description != null) { - internalStatus.setDescription(description); - } - if (date != null) { - internalStatus.setDate(date); - } - } - } - - Document transformedFile = index.get("transformedFile", Document.class); - if (transformedFile != null) { - Number uid = transformedFile.get("id", Number.class); - Number metadataUid = transformedFile.get("metadataId", Number.class); - - if (uid != null && uid.longValue() > 0) { - variant.getIndex().getTransform().setFileId(getFileId(uid.longValue())); - } - if (metadataUid != null && metadataUid.longValue() > 0) { - variant.getIndex().getTransform().setMetadataFileId(getFileId(metadataUid.longValue())); - } - } - - Document localFileIndex = index.get("localFileIndex", Document.class); - if (localFileIndex != null) { - Number fileUid = localFileIndex.get("fileId", Number.class); - String indexer = localFileIndex.getString("indexer"); - - if (fileUid != null && fileUid.longValue() > 0) { - alignment.getIndex().setFileId(getFileId(fileUid.longValue())); - } - if (indexer != null) { - alignment.getIndex().setIndexer(indexer); - } - } - } - - Document variantDoc = convertToDocument(variant); - Document alignmentDoc = convertToDocument(alignment); - - bulk.add(new UpdateOneModel<>( - eq("_id", doc.get("_id")), - new Document() - .append("$set", new Document() - .append("internal.variant", variantDoc) - .append("internal.alignment", alignmentDoc) - ) - .append("$unset", new Document("internal.index", "")) - ) - ); - }); - logger.info("Finish add new file internal index field."); - logger.info(" - Total files inspected : {}", filesWithNewInternalField.intValue() + filesWithoutNewInternalField.intValue()); - logger.info(" - Files updated : {}", filesWithoutNewInternalField.intValue()); - logger.info(" - Files already updated (skipped) : {}", filesWithNewInternalField.intValue()); - } - - private String getFileId(long fileUid) { - OpenCGAResult fileOpenCGAResult; - try { - fileOpenCGAResult = dbAdaptorFactory.getCatalogFileDBAdaptor().get(fileUid, FileManager.INCLUDE_FILE_IDS); - } catch (CatalogException e) { - throw new RuntimeException(e.getMessage(), e); - } - - if (fileOpenCGAResult.getNumResults() == 0) { - throw new RuntimeException("Could not find 'id' for file 'uid': " + fileUid); - } - return fileOpenCGAResult.first().getId(); - - } - -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/AddSampleInternalVariant_1851.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/AddSampleInternalVariant_1851.java deleted file mode 100644 index 795ae1b757b..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/AddSampleInternalVariant_1851.java +++ /dev/null @@ -1,47 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; -import org.opencb.opencga.core.models.sample.SampleInternal; -import org.opencb.opencga.core.models.sample.SampleInternalVariant; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "add_sample_internal_variant_1851", - description = "Add new SampleInternalVariant #1851", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20211129) -public class AddSampleInternalVariant_1851 extends MigrationTool { - - @Override - protected void run() throws Exception { - migrateCollection(MongoDBAdaptorFactory.SAMPLE_COLLECTION, - new Document("internal.variant", new Document("$exists", false)), - Projections.include("_id", "internal"), - (doc, bulk) -> { - Document internal = doc.get("internal", Document.class); - - if (internal != null) { - SampleInternalVariant sampleInternalVariant = SampleInternalVariant.init(); - Document sampleInternalVariantDoc = convertToDocument(sampleInternalVariant); - internal.put("variant", sampleInternalVariantDoc); - } else { - SampleInternal sampleInternal = SampleInternal.init(); - Document sampleInternalDoc = convertToDocument(sampleInternal); - doc.put("internal", sampleInternalDoc); - } - - bulk.add(new UpdateOneModel<>( - eq("_id", doc.get("_id")), - new Document("$set", new Document("internal", doc.get("internal"))) - ) - ); - }); - } - -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/ChangeInterpretationMethods.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/ChangeInterpretationMethods.java deleted file mode 100644 index f68d80d80e6..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/ChangeInterpretationMethods.java +++ /dev/null @@ -1,89 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.db.api.InterpretationDBAdaptor; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptor; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import java.util.Collections; -import java.util.List; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "change_interpretation_method", - description = "Remove list of methods from Interpretations #1841", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20211112) -public class ChangeInterpretationMethods extends MigrationTool { - - private void updateClinicalVariant(List findings, Object filters) { - if (findings == null) { - return; - } - - for (Document finding : findings) { - finding.remove("interpretationMethodNames"); - if (filters != null) { - finding.put("filters", filters); - } - } - } - - @Override - protected void run() throws Exception { - migrateCollection(MongoDBAdaptorFactory.INTERPRETATION_COLLECTION, - new Document(), - Projections.include("_id", "methods", InterpretationDBAdaptor.QueryParams.PRIMARY_FINDINGS.key(), - InterpretationDBAdaptor.QueryParams.SECONDARY_FINDINGS.key()), - (interpretationDoc, bulk) -> { - MongoDBAdaptor.UpdateDocument updateDocument = new MongoDBAdaptor.UpdateDocument(); - List methods = interpretationDoc.getList("methods", Document.class); - - Object filters = null; - if (methods != null) { - updateDocument.getUnset().add("methods"); - - Document methodDocument; - if (methods.size() > 0) { - // Edit method - methodDocument = methods.get(0); - filters = methodDocument.get("filters"); - methodDocument.remove("filters"); - methodDocument.remove("panels"); - } else { - // Initialise - methodDocument = new Document() - .append("name", "") - .append("dependencies", Collections.emptyList()); - } - methodDocument.put("version", ""); - methodDocument.put("commit", ""); - - updateDocument.getSet().put(InterpretationDBAdaptor.QueryParams.METHOD.key(), methodDocument); - } - - List primaryFindings = interpretationDoc.getList(InterpretationDBAdaptor.QueryParams.PRIMARY_FINDINGS.key(), Document.class); - if (primaryFindings != null) { - updateClinicalVariant(primaryFindings, filters); - updateDocument.getSet().put(InterpretationDBAdaptor.QueryParams.PRIMARY_FINDINGS.key(), primaryFindings); - } - List secondaryFindings = interpretationDoc.getList(InterpretationDBAdaptor.QueryParams.SECONDARY_FINDINGS.key(), Document.class); - if (secondaryFindings != null) { - updateClinicalVariant(secondaryFindings, filters); - updateDocument.getSet().put(InterpretationDBAdaptor.QueryParams.SECONDARY_FINDINGS.key(), secondaryFindings); - } - - Document update = updateDocument.toFinalUpdateDocument(); - if (!update.isEmpty()) { - bulk.add(new UpdateOneModel<>( - eq("_id", interpretationDoc.get("_id")), - update)); - } - }); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/ClinicalVariantEvidenceMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/ClinicalVariantEvidenceMigration.java deleted file mode 100644 index d1fc2f27767..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/ClinicalVariantEvidenceMigration.java +++ /dev/null @@ -1,95 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog; - -import com.mongodb.client.MongoCollection; -import com.mongodb.client.MongoCursor; -import com.mongodb.client.model.Projections; -import org.apache.commons.lang3.StringUtils; -import org.bson.BsonMaximumSizeExceededException; -import org.bson.Document; -import org.opencb.commons.ProgressLogger; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import java.util.Collections; -import java.util.List; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "add_clinical_variant_evidence_review", - description = "Add new ClinicalVariantEvidenceReview object, #1874", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20220112) -public class ClinicalVariantEvidenceMigration extends MigrationTool { - - @Override - protected void run() throws Exception { - MongoCollection collection = getMongoCollection(MongoDBAdaptorFactory.INTERPRETATION_COLLECTION); - - int bsonMaximumSizeExceeded = 0; - ProgressLogger progressLogger = new ProgressLogger("Execute Interpretation update").setBatchSize(100); - try (MongoCursor it = collection - .find(new Document("primaryFindings.evidences.review", new Document("$exists", false))) - .projection(Projections.include("_id", "primaryFindings", "secondaryFindings")).cursor()) { - while (it.hasNext()) { - Document doc = it.next(); - - List primaryFindings = doc.getList("primaryFindings", Document.class); - List secondaryFindings = doc.getList("secondaryFindings", Document.class); - - Document update = new Document(); - if (!primaryFindings.isEmpty()) { - applyChanges(primaryFindings); - update.put("primaryFindings", primaryFindings); - } - if (!secondaryFindings.isEmpty()) { - applyChanges(secondaryFindings); - update.put("secondaryFindings", secondaryFindings); - } - - if (!update.isEmpty()) { - try { - collection.updateOne(eq("_id", doc.get("_id")), new Document("$set", update)); - progressLogger.increment(1); - } catch (BsonMaximumSizeExceededException e) { - bsonMaximumSizeExceeded++; - } - } - } - } - - if (bsonMaximumSizeExceeded > 0) { - logger.warn("{} Interpretations could not be updated because of the size of the document", bsonMaximumSizeExceeded); - } - } - - private void applyChanges(List clinicalVariants) { - if (clinicalVariants.isEmpty()) { - return; - } - - for (Document clinicalVariant : clinicalVariants) { - List evidences = clinicalVariant.getList("evidences", Document.class); - if (evidences.isEmpty()) { - continue; - } - for (Document evidence : evidences) { - Document review = evidence.get("review", Document.class); - // review field should not exist - if (review == null) { - String justification = evidence.getString("justification"); - evidence.remove("justification"); - - Document clinicalEvidenceReview = new Document() - .append("select", false) - .append("acmg", Collections.emptyList()); - if (StringUtils.isNotEmpty(justification)) { - clinicalEvidenceReview.put("discussion", justification); - } - evidence.put("review", clinicalEvidenceReview); - } - } - } - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/DeleteUnusedVariableSets.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/DeleteUnusedVariableSets.java deleted file mode 100644 index 291acb39b6b..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/DeleteUnusedVariableSets.java +++ /dev/null @@ -1,95 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog; - -import com.mongodb.client.MongoCollection; -import com.mongodb.client.MongoCursor; -import com.mongodb.client.model.Projections; -import org.bson.Document; -import org.bson.conversions.Bson; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.db.mongodb.converters.StudyConverter; -import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; -import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.models.study.VariableSet; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -@Migration(id = "delete_unused_variablesets", - description = "Delete unused VariableSets, #1859", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - patch = 2, - date = 20211210) -public class DeleteUnusedVariableSets extends MigrationTool { - - @Override - protected void run() throws Exception { - StudyConverter converter = new StudyConverter(); - MongoCursor studyIterator = getMongoCollection(MongoDBAdaptorFactory.STUDY_COLLECTION) - .find(new Document()) - .projection(Projections.include("fqn", "variableSets")).iterator(); - - List studyList = new ArrayList<>(); - while (studyIterator.hasNext()) { - Document studyDoc = studyIterator.next(); - Study study = converter.convertToDataModelType(studyDoc); - studyList.add(study); - } - - for (Study study : studyList) { - logger.info("Deleting VariableSets from study '{}'", study.getFqn()); - delete(study, "opencga_alignment_samtools_flagstat"); - delete(study, "opencga_alignment_stats"); - delete(study, "opencga_sample_qc"); - delete(study, "opencga_file_variant_stats"); - } - } - - private void deleteAnnotationSets(VariableSet variableSet, String collection) { - MongoCollection mongoCollection = getMongoCollection(collection); - Bson query = new Document() - .append("$or", Arrays.asList( - new Document("customAnnotationSets.vs", variableSet.getUid()), - new Document("customInternalAnnotationSets.vs", variableSet.getUid()) - )); - Bson update = new Document() - .append("$pull", new Document() - .append("customAnnotationSets", new Document("vs", variableSet.getUid())) - .append("customInternalAnnotationSets", new Document("vs", variableSet.getUid())) - ) - .append("$unset", new Document() - .append("_vsMap." + variableSet.getUid(), "") - .append("_ivsMap." + variableSet.getUid(), "") - ); - logger.info("Removing AnnotationSets using the VariableSet '{}' from '{}' collection...", variableSet.getId(), collection); - mongoCollection.updateMany(query, update); - } - - private void delete(Study study, String variableSetId) { - // Find variable set - VariableSet variableSet = null; - for (VariableSet tmpVariableSet : study.getVariableSets()) { - if (tmpVariableSet.getId().equals(variableSetId)) { - variableSet = tmpVariableSet; - } - } - if (variableSet == null) { - return; - } - deleteAnnotationSets(variableSet, MongoDBAdaptorFactory.FILE_COLLECTION); - deleteAnnotationSets(variableSet, MongoDBAdaptorFactory.SAMPLE_COLLECTION); - deleteAnnotationSets(variableSet, MongoDBAdaptorFactory.INDIVIDUAL_COLLECTION); - deleteAnnotationSets(variableSet, MongoDBAdaptorFactory.FAMILY_COLLECTION); - deleteAnnotationSets(variableSet, MongoDBAdaptorFactory.COHORT_COLLECTION); - - try { - catalogManager.getStudyManager().deleteVariableSet(study.getFqn(), variableSetId, true, token); - } catch (CatalogException e) { - logger.warn(e.getMessage()); - } - } - -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/FixFamilyReferences.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/FixFamilyReferences.java deleted file mode 100644 index 5ffe8471598..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/FixFamilyReferences.java +++ /dev/null @@ -1,69 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import java.util.*; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "fix_family_references_in_individual", - description = "Fix Family references, #TASK-489", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20220324) -public class FixFamilyReferences extends MigrationTool { - - @Override - protected void run() throws Exception { - // Get all the family ids: studyUid - {familyIds} - Map> familyIds = new HashMap<>(); - queryMongo(MongoDBAdaptorFactory.FAMILY_COLLECTION, new Document(), - Projections.include(Arrays.asList("id", "studyUid")), (doc) -> { - String studyId = doc.get("studyUid", Number.class).toString(); - if (!familyIds.containsKey(studyId)) { - familyIds.put(studyId, new HashSet<>()); - } - familyIds.get(studyId).add(doc.getString("id")); - }); - - migrateCollection(MongoDBAdaptorFactory.INDIVIDUAL_COLLECTION, - new Document("familyIds", new Document("$ne", Collections.emptyList())), - Projections.include("familyIds", "studyUid"), - (individual, bulk) -> { - String studyId = individual.get("studyUid", Number.class).toString(); - - if (!familyIds.containsKey(studyId)) { - return; - } - - boolean changed = false; - List newIds = new ArrayList<>(); - List iFamilyIds = individual.getList("familyIds", String.class); - - // Store all existing familyIds in newIds so non-existing ones are removed - for (String iFamilyId : iFamilyIds) { - if (familyIds.get(studyId).contains(iFamilyId)) { - newIds.add(iFamilyId); - } else { - // Found a non-existing familyId, so we'll need to apply the update ! - changed = true; - } - } - - if (changed) { - bulk.add(new UpdateOneModel<>( - eq("_id", individual.get("_id")), - new Document("$set", new Document("familyIds", newIds)) - ) - ); - } - } - ); - } - -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/FixNonExistingMoIFromPanels.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/FixNonExistingMoIFromPanels.java deleted file mode 100644 index 90951267b66..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/FixNonExistingMoIFromPanels.java +++ /dev/null @@ -1,110 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.apache.commons.lang3.StringUtils; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "fix_non_existing_mois_from_panels", - description = "Remove non-existing MOIs from Panels", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - patch = 2, - date = 20220111) -public class FixNonExistingMoIFromPanels extends MigrationTool { - - private void fixMoI(List documentList) { - List newMoi = new ArrayList<>(); - - for (Document document : documentList) { - String modeOfInheritance = document.getString("modeOfInheritance"); - if (StringUtils.isNotEmpty(modeOfInheritance)) { - switch (modeOfInheritance) { - case "AUTOSOMAL_DOMINANT_AND_RECESSIVE": - case "AUTOSOMAL_DOMINANT_AND_MORE_SEVERE_RECESSIVE": - // Make this moi dominant - document.put("modeOfInheritance", "AUTOSOMAL_DOMINANT"); - - // And generate a new one being recessive - Document newDoc = new Document(document); - newDoc.put("modeOfInheritance", "AUTOSOMAL_RECESSIVE"); - newMoi.add(newDoc); - break; - case "AUTOSOMAL_DOMINANT_MATERNALLY_IMPRINTED": - case "AUTOSOMAL_DOMINANT_NOT_IMPRINTED": - case "AUTOSOMAL_DOMINANT_PATERNALLY_IMPRINTED": - document.put("modeOfInheritance", "AUTOSOMAL_DOMINANT"); - break; - - // Valid MOI terms - case "AUTOSOMAL_DOMINANT": - case "AUTOSOMAL_RECESSIVE": - case "X_LINKED_DOMINANT": - case "X_LINKED_RECESSIVE": - case "Y_LINKED": - case "MITOCHONDRIAL": - case "DE_NOVO": - case "MENDELIAN_ERROR": - case "COMPOUND_HETEROZYGOUS": - case "UNKNOWN": - break; - - default: - logger.warn("Unexpected MOI term '{}' found. Setting it to unknown.", modeOfInheritance); - document.put("modeOfInheritance", "UNKNOWN"); - break; - } - } - } - - documentList.addAll(newMoi); - } - - private List getDocumentList(Document document, String field) { - List list = document.getList(field, Document.class); - if (list == null) { - return Collections.emptyList(); - } else { - return new ArrayList<>(list); - } - } - - @Override - protected void run() throws Exception { - migrateCollection(MongoDBAdaptorFactory.PANEL_COLLECTION, - new Document(), - Projections.include("_id", "variants", "genes", "strs", "regions"), - (panelDoc, bulk) -> { - if (panelDoc != null) { - List genes = getDocumentList(panelDoc, "genes"); - fixMoI(genes); - List variants = getDocumentList(panelDoc, "variants"); - fixMoI(variants); - List strs = getDocumentList(panelDoc, "strs"); - fixMoI(strs); - List regions = getDocumentList(panelDoc, "regions"); - fixMoI(regions); - - bulk.add(new UpdateOneModel<>( - eq("_id", panelDoc.get("_id")), - new Document("$set", new Document() - .append("genes", genes) - .append("variants", variants) - .append("strs", strs) - .append("regions", regions) - ))); - } - } - ); - } - -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/ImproveClinicalAnalysisQualityControl.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/ImproveClinicalAnalysisQualityControl.java deleted file mode 100644 index 4014fbf1f8e..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/ImproveClinicalAnalysisQualityControl.java +++ /dev/null @@ -1,98 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.apache.commons.lang3.StringUtils; -import org.bson.Document; -import org.opencb.opencga.catalog.db.api.ClinicalAnalysisDBAdaptor; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; -import org.opencb.opencga.core.common.TimeUtils; -import org.opencb.opencga.core.models.clinical.ClinicalAnalysisQualityControl; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.Date; -import java.util.List; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "improve_ca_quality_control", - description = "Quality control normalize comments and fields #1826", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20211001) -public class ImproveClinicalAnalysisQualityControl extends MigrationTool { - - @Override - protected void run() throws Exception { - - improvementsClinicalAnalysis(); - } - - private void improvementsClinicalAnalysis() { - migrateCollection(MongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, - new Document(ClinicalAnalysisDBAdaptor.QueryParams.QUALITY_CONTROL.key() + ".comments", new Document("$exists", false)), - Projections.include("_id", ClinicalAnalysisDBAdaptor.QueryParams.QUALITY_CONTROL.key()), - (clinicalAnalysisDoc, bulk) -> { - if (clinicalAnalysisDoc != null) { - Document qc = clinicalAnalysisDoc.get(ClinicalAnalysisDBAdaptor.QueryParams.QUALITY_CONTROL.key(), Document.class); - if (qc != null && qc.containsKey("summary")) { - String summary = qc.get("summary", String.class); - String text = qc.get("comment", String.class); - String user = qc.get("user", String.class); - Date date = null; - try { - if (qc.containsKey("date")) { - date = new Date(qc.get("date", Number.class).longValue()); - } - } catch (Exception e) { - // empty catch block - } - -// if EXCELLENT, GOOD, NORMAL, BAD, UNKNOWN - String newSummary = null; - switch (summary) { - case "EXCELLENT": - newSummary = ClinicalAnalysisQualityControl.QualityControlSummary.HIGH.name(); - break; - case "GOOD": - newSummary = ClinicalAnalysisQualityControl.QualityControlSummary.MEDIUM.name(); - break; - case "NORMAL": - newSummary = ClinicalAnalysisQualityControl.QualityControlSummary.MEDIUM.name(); - break; - case "BAD": - newSummary = ClinicalAnalysisQualityControl.QualityControlSummary.LOW.name(); - break; - case "UNKNOWN": - newSummary = ClinicalAnalysisQualityControl.QualityControlSummary.UNKNOWN.name(); - break; - default: - break; - } - - List comments = new ArrayList<>(); - if (StringUtils.isNotEmpty(text) && StringUtils.isNotEmpty(user)) { - Document comment = new Document() - .append("author", user) - .append("message", text) - .append("tags", Collections.emptyList()) - .append("date", date != null ? TimeUtils.getTime(date) : TimeUtils.getTime()); - comments.add(comment); - } - - Document finalQC = new Document() - .append("comments", comments) - .append("summary", newSummary); - bulk.add(new UpdateOneModel<>( - eq("_id", clinicalAnalysisDoc.get("_id")), - new Document("$set", - new Document(ClinicalAnalysisDBAdaptor.QueryParams.QUALITY_CONTROL.key(), finalQC)))); - } - } - } - ); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/ImproveFileQualityControl.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/ImproveFileQualityControl.java deleted file mode 100644 index 2d2734b977c..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/ImproveFileQualityControl.java +++ /dev/null @@ -1,60 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.biodata.formats.sequence.ascat.AscatMetrics; -import org.opencb.opencga.catalog.db.api.FileDBAdaptor; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.db.mongodb.converters.FileConverter; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; -import org.opencb.opencga.core.models.file.File; -import org.opencb.opencga.core.models.file.FileQualityControl; - -import java.util.Collections; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "improve_file_quality_control", - description = "Quality control normalize comments and fields #1826", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20211001) -public class ImproveFileQualityControl extends MigrationTool { - - @Override - protected void run() throws Exception { - // File - improvementsFile(); - } - - private void improvementsFile() { - FileConverter fileConverter = new FileConverter(); - migrateCollection(MongoDBAdaptorFactory.FILE_COLLECTION, - new Document(FileDBAdaptor.QueryParams.QUALITY_CONTROL.key() + ".comments", new Document("$exists", false)), - Projections.include("_id", FileDBAdaptor.QueryParams.QUALITY_CONTROL.key()), - (fileDoc, bulk) -> { - //List files = fileDoc.getList("files", String.class); - File file = fileConverter.convertToDataModelType(fileDoc); - FileQualityControl fqc = file.getQualityControl(); - if (fqc != null) { - if (fqc.getComments() == null) { - fqc.setComments(Collections.emptyList()); - } - if (fqc.getVariant() != null && fqc.getVariant().getAscatMetrics() == null) { - fqc.getVariant().setAscatMetrics(new AscatMetrics()); - } - - Document doc = fileConverter.convertToStorageType(file); - bulk.add(new UpdateOneModel<>( - eq("_id", fileDoc.get("_id")), - new Document("$set", new Document(FileDBAdaptor.QueryParams.QUALITY_CONTROL.key(), - doc.get(FileDBAdaptor.QueryParams.QUALITY_CONTROL.key()))) - ) - ); - } - } - ); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/ImproveIndividualQualityControl.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/ImproveIndividualQualityControl.java deleted file mode 100644 index b1691968ebe..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/ImproveIndividualQualityControl.java +++ /dev/null @@ -1,96 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.apache.commons.collections4.CollectionUtils; -import org.bson.Document; -import org.opencb.biodata.models.clinical.qc.InferredSexReport; -import org.opencb.biodata.models.clinical.qc.MendelianErrorReport; -import org.opencb.biodata.models.clinical.qc.SampleRelatednessReport; -import org.opencb.opencga.catalog.db.api.IndividualDBAdaptor; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.db.mongodb.converters.IndividualConverter; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; -import org.opencb.opencga.core.common.JacksonUtils; -import org.opencb.opencga.core.models.individual.Individual; -import org.opencb.opencga.core.models.individual.IndividualQualityControl; - -import java.util.Collections; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "improve_individual_quality_control", - description = "Quality control normalize comments and fields #1826", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20211001) -public class ImproveIndividualQualityControl extends MigrationTool { - - @Override - protected void run() throws Exception { - // File - - improvementsIndividual(); - } - - private void improvementsIndividual() { - ObjectMapper defaultObjectMapper = JacksonUtils.getDefaultObjectMapper(); - - IndividualConverter individualConverter = new IndividualConverter(); - migrateCollection(MongoDBAdaptorFactory.INDIVIDUAL_COLLECTION, - new Document(IndividualDBAdaptor.QueryParams.QUALITY_CONTROL.key() + ".inferredSexReports.sampleId", - new Document("$exists", false)), - Projections.include("_id", IndividualDBAdaptor.QueryParams.QUALITY_CONTROL.key()), - (individualDoc, bulk) -> { - Individual individual = individualConverter.convertToDataModelType(individualDoc); - IndividualQualityControl fqc = individual.getQualityControl(); - - if (fqc != null) { - Document qc = individualDoc.get(IndividualDBAdaptor.QueryParams.QUALITY_CONTROL.key(), Document.class); - if (qc != null) { - if (fqc.getInferredSexReports() != null) { - String sampleId = qc.get("sampleId", String.class); - for (InferredSexReport inferredSexReport : fqc.getInferredSexReports()) { - inferredSexReport.setSampleId(sampleId); - } - } - if (CollectionUtils.isEmpty(fqc.getMendelianErrorReports())) { - Document mendelianErrorReportDoc = qc.get("mendelianErrorReport", Document.class); - if (mendelianErrorReportDoc != null) { - try { - String mString = defaultObjectMapper.writeValueAsString(mendelianErrorReportDoc); - MendelianErrorReport mendelianErrorReport = defaultObjectMapper.readValue(mString, - MendelianErrorReport.class); - if (mendelianErrorReport.getNumErrors() == 0 && CollectionUtils.isEmpty(mendelianErrorReport.getSampleAggregation())) { - // Default values of Mendelian Error Report - fqc.setMendelianErrorReports(Collections.emptyList()); - } else { - fqc.setMendelianErrorReports(Collections.singletonList(mendelianErrorReport)); - } - } catch (JsonProcessingException e) { - logger.error("Could not parse Mendelian Error Report properly"); - } - } else { - fqc.setMendelianErrorReports(Collections.emptyList()); - } - } - } - if (fqc.getSampleRelatednessReport() == null) { - fqc.setSampleRelatednessReport(new SampleRelatednessReport()); - } - - Document doc = individualConverter.convertToStorageType(individual); - bulk.add(new UpdateOneModel<>( - eq("_id", individualDoc.get("_id")), - new Document("$set", new Document(IndividualDBAdaptor.QueryParams.QUALITY_CONTROL.key(), - doc.get(IndividualDBAdaptor.QueryParams.QUALITY_CONTROL.key()))) - ) - ); - } - } - ); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/ImproveVariableSetNamesAndDescriptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/ImproveVariableSetNamesAndDescriptions.java deleted file mode 100644 index f3b745a6a0b..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/ImproveVariableSetNamesAndDescriptions.java +++ /dev/null @@ -1,77 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.db.mongodb.converters.StudyConverter; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; -import org.opencb.opencga.core.common.JacksonUtils; -import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.models.study.VariableSet; -import org.reflections.Reflections; -import org.reflections.scanners.ResourcesScanner; - -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.regex.Pattern; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "improveVariableSetNamesAndDescriptions", - description = "Improve VariableSet names and descriptions", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20211210) -public class ImproveVariableSetNamesAndDescriptions extends MigrationTool { - - @Override - protected void run() throws Exception { - // Read default VariableSets - Map variableSetMap = new HashMap<>(); - Set variablesets = new Reflections(new ResourcesScanner(), "variablesets/").getResources(Pattern.compile(".*\\.json")); - for (String variableSetFile : variablesets) { - VariableSet vs; - try { - vs = JacksonUtils.getDefaultNonNullObjectMapper().readValue( - getClass().getClassLoader().getResourceAsStream(variableSetFile), VariableSet.class); - } catch (IOException e) { - logger.error("Could not parse variable set '{}'", variableSetFile, e); - continue; - } - if (vs != null) { - variableSetMap.put(vs.getId(), vs); - } - } - if (variableSetMap.isEmpty()) { - throw new RuntimeException("Java reflection didn't work"); - } - - StudyConverter converter = new StudyConverter(); - migrateCollection(MongoDBAdaptorFactory.STUDY_COLLECTION, - new Document(), - Projections.include("_id", "variableSets"), - (doc, bulk) -> { - Study study = converter.convertToDataModelType(doc); - for (VariableSet variableSet : study.getVariableSets()) { - if (variableSetMap.containsKey(variableSet.getId())) { - // Change names and descriptions - VariableSet vs = variableSetMap.get(variableSet.getId()); - variableSet.setName(vs.getName()); - variableSet.setDescription(vs.getDescription()); - } - } - - List variableSetDocumentList = convertToDocument(study.getVariableSets()); - bulk.add(new UpdateOneModel<>( - eq("_id", doc.get("_id")), - new Document("$set", new Document("variableSets", variableSetDocumentList))) - ); - } - ); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/RemoveFileDocsFromCA.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/RemoveFileDocsFromCA.java deleted file mode 100644 index 4c9c0b566ca..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/RemoveFileDocsFromCA.java +++ /dev/null @@ -1,41 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.db.api.ClinicalAnalysisDBAdaptor; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.db.mongodb.converters.ClinicalAnalysisConverter; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "remove_file_docs_from_clinical_analyses", - description = "Store references of File in Clinical Analysis and not full File documents #1837", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20211102) -public class RemoveFileDocsFromCA extends MigrationTool { - - @Override - protected void run() throws Exception { - ClinicalAnalysisConverter converter = new ClinicalAnalysisConverter(); - - migrateCollection(MongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, - new Document(ClinicalAnalysisDBAdaptor.QueryParams.FILES.key() + ".id", new Document("$exists", true)), - Projections.include("_id", ClinicalAnalysisDBAdaptor.QueryParams.FILES.key()), - (clinicalDoc, bulk) -> { - // Filter files to store only the fields expected - converter.validateFilesToUpdate(clinicalDoc); - - bulk.add(new UpdateOneModel<>( - eq("_id", clinicalDoc.get("_id")), - new Document("$set", new Document(ClinicalAnalysisDBAdaptor.QueryParams.FILES.key(), - clinicalDoc.get(ClinicalAnalysisDBAdaptor.QueryParams.FILES.key())))) - ); - - } - ); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/RemoveParallelIndexes.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/RemoveParallelIndexes.java deleted file mode 100644 index 6945df1f568..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/RemoveParallelIndexes.java +++ /dev/null @@ -1,36 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog; - -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -@Migration(id = "remove_parallel_indexes", - description = "Remove parallel array indexes #CU-20jc4tx", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20220310) -public class RemoveParallelIndexes extends MigrationTool { - - @Override - protected void run() throws Exception { - Document newIndex = new Document() - .append("studyUid", 1) - .append("_releaseFromVersion", 1) - .append("_lastOfRelease", 1); - createIndex(MongoDBAdaptorFactory.SAMPLE_COLLECTION, newIndex); - createIndex(MongoDBAdaptorFactory.INDIVIDUAL_COLLECTION, newIndex); - createIndex(MongoDBAdaptorFactory.FAMILY_COLLECTION, newIndex); - createIndex(MongoDBAdaptorFactory.PANEL_COLLECTION, newIndex); - - Document oldIndex = new Document() - .append("studyUid", 1) - .append("_releaseFromVersion", 1) - .append("_lastOfRelease", 1) - .append("_acl", 1); - dropIndex(MongoDBAdaptorFactory.SAMPLE_COLLECTION, oldIndex); - dropIndex(MongoDBAdaptorFactory.INDIVIDUAL_COLLECTION, oldIndex); - dropIndex(MongoDBAdaptorFactory.FAMILY_COLLECTION, oldIndex); - dropIndex(MongoDBAdaptorFactory.PANEL_COLLECTION, oldIndex); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/RenameFamilyQualityControlFields.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/RenameFamilyQualityControlFields.java deleted file mode 100644 index a88a058447f..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/RenameFamilyQualityControlFields.java +++ /dev/null @@ -1,44 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.db.api.FamilyDBAdaptor; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import java.util.Collections; -import java.util.List; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "rename_family_quality_control_fields", - description = "Rename FamilyQualityControl fields #1844", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20211119) -public class RenameFamilyQualityControlFields extends MigrationTool { - - @Override - protected void run() throws Exception { - - migrateCollection(MongoDBAdaptorFactory.FAMILY_COLLECTION, - new Document(), - Projections.include("_id", FamilyDBAdaptor.QueryParams.QUALITY_CONTROL.key()), - (doc, bulk) -> { - Document qc = doc.get(FamilyDBAdaptor.QueryParams.QUALITY_CONTROL.key(), Document.class); - if (qc != null) { - List fileIds = qc.getList("fileIds", String.class); - qc.remove("fileIds"); - qc.put("files", fileIds != null ? fileIds : Collections.emptyList()); - - bulk.add(new UpdateOneModel<>( - eq("_id", doc.get("_id")), - new Document("$set", new Document(FamilyDBAdaptor.QueryParams.QUALITY_CONTROL.key(), qc))) - ); - } - } - ); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/RenameFileQualityControlFields.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/RenameFileQualityControlFields.java deleted file mode 100644 index 55d21a24223..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/RenameFileQualityControlFields.java +++ /dev/null @@ -1,65 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.db.api.FileDBAdaptor; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import java.util.Collections; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "rename_file_quality_control_fields", - description = "Rename FileQualityControl fields #1844", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20211119) -public class RenameFileQualityControlFields extends MigrationTool { - - @Override - protected void run() throws Exception { - - migrateCollection(MongoDBAdaptorFactory.FILE_COLLECTION, - new Document(), - Projections.include("_id", FileDBAdaptor.QueryParams.QUALITY_CONTROL.key()), - (doc, bulk) -> { - Document qc = doc.get(FileDBAdaptor.QueryParams.QUALITY_CONTROL.key(), Document.class); - if (qc != null) { - qc.put("files", Collections.emptyList()); - - Document alignment = qc.get("alignment", Document.class); - if (alignment != null) { - Document fastQcMetrics = alignment.get("fastQcMetrics", Document.class); - if (fastQcMetrics != null) { - fastQcMetrics.put("files", fastQcMetrics.get("images")); - fastQcMetrics.remove("images"); - } - - Document samtoolsStats = alignment.get("samtoolsStats", Document.class); - if (samtoolsStats != null) { - samtoolsStats.put("files", samtoolsStats.get("images")); - samtoolsStats.remove("images"); - } - } - - Document variant = qc.get("variant", Document.class); - if (variant != null) { - Document ascat = variant.get("ascatMetrics", Document.class); - if (ascat != null) { - ascat.put("files", ascat.get("images")); - ascat.remove("images"); - } - } - - bulk.add(new UpdateOneModel<>( - eq("_id", doc.get("_id")), - new Document("$set", new Document(FileDBAdaptor.QueryParams.QUALITY_CONTROL.key(), qc))) - ); - } - } - ); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/RenameIndividualQualityControlFields.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/RenameIndividualQualityControlFields.java deleted file mode 100644 index 307d5f18617..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/RenameIndividualQualityControlFields.java +++ /dev/null @@ -1,44 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.db.api.IndividualDBAdaptor; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import java.util.Collections; -import java.util.List; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "rename_individual_quality_control_fields", - description = "Rename IndividualQualityControl fields #1844", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20211119) -public class RenameIndividualQualityControlFields extends MigrationTool { - - @Override - protected void run() throws Exception { - - migrateCollection(MongoDBAdaptorFactory.INDIVIDUAL_COLLECTION, - new Document(), - Projections.include("_id", IndividualDBAdaptor.QueryParams.QUALITY_CONTROL.key()), - (doc, bulk) -> { - Document qc = doc.get(IndividualDBAdaptor.QueryParams.QUALITY_CONTROL.key(), Document.class); - if (qc != null) { - List fileIds = qc.getList("fileIds", String.class); - qc.remove("fileIds"); - qc.put("files", fileIds != null ? fileIds : Collections.emptyList()); - - bulk.add(new UpdateOneModel<>( - eq("_id", doc.get("_id")), - new Document("$set", new Document(IndividualDBAdaptor.QueryParams.QUALITY_CONTROL.key(), qc))) - ); - } - } - ); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/RenameInterpretationFindingStats.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/RenameInterpretationFindingStats.java deleted file mode 100644 index 3df8a947dd8..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/RenameInterpretationFindingStats.java +++ /dev/null @@ -1,56 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.db.api.InterpretationDBAdaptor; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "rename_interpretation_stats_field", - description = "Rename interpretation stats field #1819", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20211115) -public class RenameInterpretationFindingStats extends MigrationTool { - - @Override - protected void run() throws Exception { - migrateCollection(MongoDBAdaptorFactory.INTERPRETATION_COLLECTION, - new Document(), - Projections.include("_id", InterpretationDBAdaptor.QueryParams.STATS.key()), - (interpretationDoc, bulk) -> { - Document stats = interpretationDoc.get("stats", Document.class); - if (stats == null) { - return; - } - Document primaryFindingsStats = stats.get("primaryFindings", Document.class); - if (primaryFindingsStats != null) { - Object statusCount = primaryFindingsStats.get("variantStatusCount"); - primaryFindingsStats.remove("variantStatusCount"); - if (statusCount != null) { - primaryFindingsStats.put("statusCount", statusCount); - } - } - Document secondaryFindingsStats = stats.get("secondaryFindings", Document.class); - if (secondaryFindingsStats != null) { - Object statusCount = secondaryFindingsStats.get("variantStatusCount"); - secondaryFindingsStats.remove("variantStatusCount"); - if (statusCount != null) { - secondaryFindingsStats.put("statusCount", statusCount); - } - } - - bulk.add(new UpdateOneModel<>( - eq("_id", interpretationDoc.get("_id")), - new Document("$set", new Document("stats", stats)) - ) - ); - } - ); - } - -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/RenameSampleQualityControlFields.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/RenameSampleQualityControlFields.java deleted file mode 100644 index afa6aabb4a6..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/RenameSampleQualityControlFields.java +++ /dev/null @@ -1,82 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.apache.commons.collections4.CollectionUtils; -import org.bson.Document; -import org.opencb.opencga.catalog.db.api.SampleDBAdaptor; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import java.util.Collections; -import java.util.List; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "rename_sample_quality_control_fields", - description = "Rename SampleQualityControl fields #1844", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - patch = 2, - date = 20211119) -public class RenameSampleQualityControlFields extends MigrationTool { - - @Override - protected void run() throws Exception { - - migrateCollection(MongoDBAdaptorFactory.SAMPLE_COLLECTION, - new Document(), - Projections.include("_id", SampleDBAdaptor.QueryParams.QUALITY_CONTROL.key()), - (sampleDoc, bulk) -> { - - Document qc = sampleDoc.get(SampleDBAdaptor.QueryParams.QUALITY_CONTROL.key(), Document.class); - if (qc != null) { - List files = qc.getList("files", String.class); - List fileIds = qc.getList("fileIds", String.class); - qc.remove("fileIds"); - - if (CollectionUtils.isEmpty(files) && CollectionUtils.isNotEmpty(fileIds)) { - qc.put("files", fileIds); - } else { - qc.put("files", Collections.emptyList()); - } - - Document variant = qc.get("variant", Document.class); - if (variant != null) { - // Change List genomePlots for GenomePlot genomePlot; - List plots = variant.getList("genomePlots", Document.class); - variant.remove("genomePlots"); - if (CollectionUtils.isNotEmpty(plots)) { - variant.put("genomePlot", plots.get(0)); - } - - // Add files - variant.put("files", Collections.emptyList()); - - // Check signatures - List signatures = variant.getList("signatures", Document.class); - if (signatures != null) { - for (Document signature : signatures) { - // Check SignatureFitting - Document fitting = signature.get("fitting", Document.class); - if (fitting != null) { - // Rename image for file - String image = fitting.get("image", String.class); - fitting.remove("image"); - fitting.put("file", image != null ? image : ""); - } - } - } - } - - bulk.add(new UpdateOneModel<>( - eq("_id", sampleDoc.get("_id")), - new Document("$set", new Document(SampleDBAdaptor.QueryParams.QUALITY_CONTROL.key(), qc))) - ); - } - } - ); - } - -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addDefaultVariantCallerInterpretationStudyConfiguration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addDefaultVariantCallerInterpretationStudyConfiguration.java deleted file mode 100644 index 438725e72d7..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addDefaultVariantCallerInterpretationStudyConfiguration.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog; - -import com.mongodb.client.MongoCollection; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import java.util.Collections; - -@Migration(id = "add_variant_caller_interpretation_configuration", - description = "Add default variant caller Interpretation Study configuration #1822", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20210916) -public class addDefaultVariantCallerInterpretationStudyConfiguration extends MigrationTool { - - @Override - protected void run() throws Exception { - // Get study collection instance - MongoCollection studyCollection = getMongoCollection(MongoDBAdaptorFactory.STUDY_COLLECTION); - - // Update - String field = "internal.configuration.clinical.interpretation.variantCallers"; - Document query = new Document(field, new Document("$exists", false)); - Document update = new Document("$set", new Document(field, Collections.emptyList())); - studyCollection.updateMany(query, update); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addInternalLastModified/AddInternalLastModified.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addInternalLastModified/AddInternalLastModified.java deleted file mode 100644 index d3f3f78554f..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addInternalLastModified/AddInternalLastModified.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog.addInternalLastModified; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import static com.mongodb.client.model.Filters.eq; - -public abstract class AddInternalLastModified extends MigrationTool { - - protected void addInternalModificationDate(String collection) { - migrateCollection(collection, - new Document("internal.lastModified", new Document("$exists", false)), - Projections.include("_id", "modificationDate", "internal"), - (doc, bulk) -> { - String modificationDate = doc.getString("modificationDate"); - - bulk.add(new UpdateOneModel<>( - eq("_id", doc.get("_id")), - new Document("$set", new Document("internal.lastModified", modificationDate)) - ) - ); - } - ); - } - -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addInternalLastModified/AddModificationDateToProjectInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addInternalLastModified/AddModificationDateToProjectInternal.java deleted file mode 100644 index 34144daceb5..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addInternalLastModified/AddModificationDateToProjectInternal.java +++ /dev/null @@ -1,47 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog.addInternalLastModified; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.db.api.ProjectDBAdaptor; -import org.opencb.opencga.catalog.db.api.UserDBAdaptor; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import java.util.List; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "add_modificationDate_to_project.internal", description = "Add internal modificationDate to Project #1810", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, - date = 20210812) -public class AddModificationDateToProjectInternal extends MigrationTool { - - @Override - protected void run() throws Exception { - - migrateCollection(MongoDBAdaptorFactory.USER_COLLECTION, - new Document() - .append("projects.internal", new Document("$exists", true)) - .append("projects.internal.modificationDate", new Document("$exists", false)), - Projections.include("_id", UserDBAdaptor.QueryParams.PROJECTS.key()), - (user, bulk) -> { - // Get projects - List projects = user.getList(UserDBAdaptor.QueryParams.PROJECTS.key(), Document.class); - - for (Document project : projects) { - String modificationDate = project.getString(ProjectDBAdaptor.QueryParams.MODIFICATION_DATE.key()); - Document internal = project.get(ProjectDBAdaptor.QueryParams.INTERNAL.key(), Document.class); - internal.put("modificationDate", modificationDate); - } - - bulk.add(new UpdateOneModel<>( - eq("_id", user.get("_id")), - new Document("$set", new Document(UserDBAdaptor.QueryParams.PROJECTS.key(), projects)) - ) - ); - } - ); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addInterpretationStats.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addInterpretationStats.java deleted file mode 100644 index 3a957b6ccd5..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addInterpretationStats.java +++ /dev/null @@ -1,53 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.biodata.models.clinical.interpretation.InterpretationStats; -import org.opencb.commons.datastore.mongodb.GenericDocumentComplexConverter; -import org.opencb.opencga.catalog.db.api.InterpretationDBAdaptor; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.db.mongodb.converters.InterpretationConverter; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; -import org.opencb.opencga.catalog.utils.InterpretationUtils; -import org.opencb.opencga.core.models.clinical.Interpretation; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "add_interpretation_stats", - description = "Add interpretation stats #1819", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - patch = 2, - date = 20210908) -public class addInterpretationStats extends MigrationTool { - - public addInterpretationStats() { - // Batch size in this case will be 50 - super(50); - } - - @Override - protected void run() throws Exception { - InterpretationConverter converter = new InterpretationConverter(); - GenericDocumentComplexConverter statsConverter = new GenericDocumentComplexConverter<>(InterpretationStats.class); - - migrateCollection(MongoDBAdaptorFactory.INTERPRETATION_COLLECTION, - new Document(InterpretationDBAdaptor.QueryParams.STATS.key(), new Document("$exists", false)), - Projections.include("_id", InterpretationDBAdaptor.QueryParams.PRIMARY_FINDINGS.key(), - InterpretationDBAdaptor.QueryParams.SECONDARY_FINDINGS.key()), - (interpretationDoc, bulk) -> { - Interpretation interpretation = converter.convertToDataModelType(interpretationDoc); - InterpretationStats interpretationStats = InterpretationUtils.calculateStats(interpretation); - Document statsDocument = statsConverter.convertToStorageType(interpretationStats); - - bulk.add(new UpdateOneModel<>( - eq("_id", interpretationDoc.get("_id")), - new Document("$set", new Document(InterpretationDBAdaptor.QueryParams.STATS.key(), statsDocument)) - ) - ); - } - ); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addRegistrationDate/AddRegistrationDate.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addRegistrationDate/AddRegistrationDate.java deleted file mode 100644 index 0e2f711c445..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addRegistrationDate/AddRegistrationDate.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog.addRegistrationDate; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import static com.mongodb.client.model.Filters.eq; - -public abstract class AddRegistrationDate extends MigrationTool { - - protected void addRegistrationDate(String collection) { - migrateCollection(collection, - new Document("internal.registrationDate", new Document("$exists", false)), - Projections.include("_id", "creationDate", "internal"), - (doc, bulk) -> { - String creationDate = doc.getString("creationDate"); - - bulk.add(new UpdateOneModel<>( - eq("_id", doc.get("_id")), - new Document("$set", new Document("internal.registrationDate", creationDate)) - ) - ); - } - ); - } - -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToProjectInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToProjectInternal.java deleted file mode 100644 index adf93911837..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToProjectInternal.java +++ /dev/null @@ -1,47 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog.addRegistrationDate; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.db.api.ProjectDBAdaptor; -import org.opencb.opencga.catalog.db.api.UserDBAdaptor; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import java.util.List; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "add_registrationDate_to_project.internal", description = "Add registrationDate to Project #1804", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, - date = 20210720) -public class AddRegistrationDateToProjectInternal extends MigrationTool { - - @Override - protected void run() throws Exception { - - migrateCollection(MongoDBAdaptorFactory.USER_COLLECTION, - new Document() - .append("projects.internal", new Document("$exists", true)) - .append("projects.internal.registrationDate", new Document("$exists", false)), - Projections.include("_id", UserDBAdaptor.QueryParams.PROJECTS.key()), - (user, bulk) -> { - // Get projects - List projects = user.getList(UserDBAdaptor.QueryParams.PROJECTS.key(), Document.class); - - for (Document project : projects) { - String creationDate = project.getString(ProjectDBAdaptor.QueryParams.CREATION_DATE.key()); - Document internal = project.get(ProjectDBAdaptor.QueryParams.INTERNAL.key(), Document.class); - internal.put("registrationDate", creationDate); - } - - bulk.add(new UpdateOneModel<>( - eq("_id", user.get("_id")), - new Document("$set", new Document(UserDBAdaptor.QueryParams.PROJECTS.key(), projects)) - ) - ); - } - ); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addRgaIndexToStudyInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addRgaIndexToStudyInternal.java deleted file mode 100644 index b9bd53caa1b..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/addRgaIndexToStudyInternal.java +++ /dev/null @@ -1,45 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "add_rga_index_summary_to_study_internal", - description = "Add RGA Index information to Study Internal #", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - patch = 1, - date = 20210719) -public class addRgaIndexToStudyInternal extends MigrationTool { - - @Override - protected void run() throws Exception { - - migrateCollection(MongoDBAdaptorFactory.STUDY_COLLECTION, - new Document("internal.index", new Document("$exists", false)), - Projections.include("_id", StudyDBAdaptor.QueryParams.CREATION_DATE.key(), "internal"), - (study, bulk) -> { - - String creationDate = study.getString(StudyDBAdaptor.QueryParams.CREATION_DATE.key()); - Document internal = study.get("internal", Document.class); - - internal.put("index", new Document("recessiveGene", new Document() - .append("status", "NOT_INDEXED") - .append("modificationDate", creationDate) - )); - - bulk.add(new UpdateOneModel<>( - eq("_id", study.get("_id")), - new Document("$set", new Document("internal", internal)) - ) - ); - } - ); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1795/AddFamilyIdsInIndividual.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1795/AddFamilyIdsInIndividual.java deleted file mode 100644 index 1e5f6978447..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1795/AddFamilyIdsInIndividual.java +++ /dev/null @@ -1,48 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog.issue_1795; - -import com.mongodb.client.MongoCollection; -import com.mongodb.client.MongoCursor; -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationException; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import java.util.ArrayList; -import java.util.List; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "add_familyIds_in_individual", description = "Add new list of familyIds in Individual #1795", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, date = 20210630) -public class AddFamilyIdsInIndividual extends MigrationTool { - - @Override - protected void run() throws MigrationException { - MongoCollection familyCollection = getMongoCollection(MongoDBAdaptorFactory.FAMILY_COLLECTION); - - migrateCollection(MongoDBAdaptorFactory.INDIVIDUAL_COLLECTION, - new Document("familyIds", new Document("$exists", false)), - Projections.include("_id", "uid"), - (individualDoc, bulk) -> { - MongoCursor familyIterator = familyCollection - .find(new Document("members.uid", individualDoc.get("uid"))) - .projection(new Document("id", 1)).iterator(); - - List familyIds = new ArrayList<>(); - while (familyIterator.hasNext()) { - Document familyDoc = familyIterator.next(); - familyIds.add(familyDoc.getString("id")); - } - - bulk.add(new UpdateOneModel<>( - eq("_id", individualDoc.get("_id")), - new Document("$set", new Document("familyIds", familyIds))) - ); - } - ); - } -} - diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1796/AddCohortIdsInSample.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1796/AddCohortIdsInSample.java deleted file mode 100644 index 5c741233462..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1796/AddCohortIdsInSample.java +++ /dev/null @@ -1,48 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog.issue_1796; - -import com.mongodb.client.MongoCollection; -import com.mongodb.client.MongoCursor; -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationException; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import java.util.ArrayList; -import java.util.List; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "add_cohortIds_in_sample", description = "Add new list of cohortIds in Sample #1796", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, date = 20210706) -public class AddCohortIdsInSample extends MigrationTool { - - @Override - protected void run() throws MigrationException { - MongoCollection cohortCollection = getMongoCollection(MongoDBAdaptorFactory.COHORT_COLLECTION); - - migrateCollection(MongoDBAdaptorFactory.SAMPLE_COLLECTION, - new Document("cohortIds", new Document("$exists", false)), - Projections.include("_id", "uid"), - (sampleDoc, bulk) -> { - MongoCursor cohortIterator = cohortCollection - .find(new Document("samples.uid", sampleDoc.get("uid"))) - .projection(new Document("id", 1)).iterator(); - - List cohortIds = new ArrayList<>(); - while (cohortIterator.hasNext()) { - Document cohortDoc = cohortIterator.next(); - cohortIds.add(cohortDoc.getString("id")); - } - - bulk.add(new UpdateOneModel<>( - eq("_id", sampleDoc.get("_id")), - new Document("$set", new Document("cohortIds", cohortIds))) - ); - } - ); - } -} - diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompleteClinicalStatusDataModel.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompleteClinicalStatusDataModel.java deleted file mode 100644 index ee832aab986..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompleteClinicalStatusDataModel.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog.issue_1849; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "complete_clinical_status_models", - description = "Complete Clinical Status data models #1849", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20211126) -public class CompleteClinicalStatusDataModel extends MigrationTool { - - @Override - protected void run() throws Exception { - migrateCollection(MongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, - new Document("status.name", new Document("$exists", false)), - Projections.include("_id", "status", "internal"), - (doc, bulk) -> { - CompleteStatusDataModelUtils.completeStatus(doc); - CompleteStatusDataModelUtils.completeInternalStatus(doc); - - bulk.add(new UpdateOneModel<>( - eq("_id", doc.get("_id")), - new Document("$set", new Document() - .append("status", doc.get("status")) - .append("internal", doc.get("internal")) - )) - ); - }); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompleteCohortStatusDataModel.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompleteCohortStatusDataModel.java deleted file mode 100644 index 68f3528db11..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompleteCohortStatusDataModel.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog.issue_1849; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "complete_cohort_status_models", - description = "Complete Cohort Status data models #1849", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20211126) -public class CompleteCohortStatusDataModel extends MigrationTool { - - @Override - protected void run() throws Exception { - migrateCollection(MongoDBAdaptorFactory.COHORT_COLLECTION, - new Document("status.id", new Document("$exists", false)), - Projections.include("_id", "status", "internal"), - (doc, bulk) -> { - CompleteStatusDataModelUtils.completeStatus(doc); - CompleteStatusDataModelUtils.completeInternalStatus(doc); - - bulk.add(new UpdateOneModel<>( - eq("_id", doc.get("_id")), - new Document("$set", new Document() - .append("status", doc.get("status")) - .append("internal", doc.get("internal")) - )) - ); - }); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompleteFamilyStatusDataModel.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompleteFamilyStatusDataModel.java deleted file mode 100644 index 030a96206ce..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompleteFamilyStatusDataModel.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog.issue_1849; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "complete_family_status_models", - description = "Complete Family Status data models #1849", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20211126) -public class CompleteFamilyStatusDataModel extends MigrationTool { - - @Override - protected void run() throws Exception { - migrateCollection(MongoDBAdaptorFactory.FAMILY_COLLECTION, - new Document("status.id", new Document("$exists", false)), - Projections.include("_id", "status", "internal"), - (doc, bulk) -> { - CompleteStatusDataModelUtils.completeStatus(doc); - CompleteStatusDataModelUtils.completeInternalStatus(doc); - - bulk.add(new UpdateOneModel<>( - eq("_id", doc.get("_id")), - new Document("$set", new Document() - .append("status", doc.get("status")) - .append("internal", doc.get("internal")) - )) - ); - }); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompleteFileStatusDataModel.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompleteFileStatusDataModel.java deleted file mode 100644 index b86ff580937..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompleteFileStatusDataModel.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog.issue_1849; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "complete_file_status_models", - description = "Complete File Status data models #1849", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20211126) -public class CompleteFileStatusDataModel extends MigrationTool { - - @Override - protected void run() throws Exception { - migrateCollection(MongoDBAdaptorFactory.FILE_COLLECTION, - new Document("status.id", new Document("$exists", false)), - Projections.include("_id", "status", "internal"), - (doc, bulk) -> { - CompleteStatusDataModelUtils.completeStatus(doc); - CompleteStatusDataModelUtils.completeInternalStatus(doc); - - bulk.add(new UpdateOneModel<>( - eq("_id", doc.get("_id")), - new Document("$set", new Document() - .append("status", doc.get("status")) - .append("internal", doc.get("internal")) - )) - ); - }); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompleteIndividualStatusDataModel.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompleteIndividualStatusDataModel.java deleted file mode 100644 index 32e94002908..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompleteIndividualStatusDataModel.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog.issue_1849; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "complete_individual_status_models", - description = "Complete Individual Status data models #1849", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20211126) -public class CompleteIndividualStatusDataModel extends MigrationTool { - - @Override - protected void run() throws Exception { - migrateCollection(MongoDBAdaptorFactory.INDIVIDUAL_COLLECTION, - new Document("status.id", new Document("$exists", false)), - Projections.include("_id", "status", "internal"), - (doc, bulk) -> { - CompleteStatusDataModelUtils.completeStatus(doc); - CompleteStatusDataModelUtils.completeInternalStatus(doc); - - bulk.add(new UpdateOneModel<>( - eq("_id", doc.get("_id")), - new Document("$set", new Document() - .append("status", doc.get("status")) - .append("internal", doc.get("internal")) - )) - ); - }); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompleteInterpretationStatusDataModel.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompleteInterpretationStatusDataModel.java deleted file mode 100644 index 7b77f24250d..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompleteInterpretationStatusDataModel.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog.issue_1849; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "complete_interpretation_status_models", - description = "Complete Interpretation Status data models #1849", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20211126) -public class CompleteInterpretationStatusDataModel extends MigrationTool { - - @Override - protected void run() throws Exception { - migrateCollection(MongoDBAdaptorFactory.INTERPRETATION_COLLECTION, - new Document("status.name", new Document("$exists", false)), - Projections.include("_id", "status", "internal"), - (doc, bulk) -> { - CompleteStatusDataModelUtils.completeStatus(doc); - CompleteStatusDataModelUtils.completeInternalStatus(doc); - - bulk.add(new UpdateOneModel<>( - eq("_id", doc.get("_id")), - new Document("$set", new Document() - .append("status", doc.get("status")) - .append("internal", doc.get("internal")) - )) - ); - }); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompleteJobStatusDataModel.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompleteJobStatusDataModel.java deleted file mode 100644 index bd91eee8011..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompleteJobStatusDataModel.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog.issue_1849; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "complete_job_status_models", - description = "Complete Job Status data models #1849", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20211126) -public class CompleteJobStatusDataModel extends MigrationTool { - - @Override - protected void run() throws Exception { - migrateCollection(MongoDBAdaptorFactory.JOB_COLLECTION, - new Document("internal.status.id", new Document("$exists", false)), - Projections.include("_id", "internal"), - (doc, bulk) -> { - CompleteStatusDataModelUtils.completeInternalStatus(doc); - - bulk.add(new UpdateOneModel<>( - eq("_id", doc.get("_id")), - new Document("$set", new Document() - .append("internal", doc.get("internal")) - )) - ); - }); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompletePanelStatusDataModel.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompletePanelStatusDataModel.java deleted file mode 100644 index ed9b5339f17..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompletePanelStatusDataModel.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog.issue_1849; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "complete_panel_status_models", - description = "Complete Panel Status data models #1849", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20211126) -public class CompletePanelStatusDataModel extends MigrationTool { - - @Override - protected void run() throws Exception { - migrateCollection(MongoDBAdaptorFactory.PANEL_COLLECTION, - new Document("status.id", new Document("$exists", false)), - Projections.include("_id", "status"), - (doc, bulk) -> { - CompleteStatusDataModelUtils.completeStatus(doc); - - bulk.add(new UpdateOneModel<>( - eq("_id", doc.get("_id")), - new Document("$set", new Document() - .append("status", doc.get("status")) - )) - ); - }); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompleteProjectStatusDataModel.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompleteProjectStatusDataModel.java deleted file mode 100644 index 2a409f57763..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompleteProjectStatusDataModel.java +++ /dev/null @@ -1,42 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog.issue_1849; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import java.util.List; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "complete_project_status_models", - description = "Complete Project Status data models #1849", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20211126) -public class CompleteProjectStatusDataModel extends MigrationTool { - - @Override - protected void run() throws Exception { - migrateCollection(MongoDBAdaptorFactory.USER_COLLECTION, - new Document() - .append("projects.id", new Document("$exists", true)) - .append("projects.internal.status.id", new Document("$exists", false)), - Projections.include("_id", "projects"), - (doc, bulk) -> { - List projects = doc.getList("projects", Document.class); - for (Document project : projects) { - CompleteStatusDataModelUtils.completeInternalStatus(project); - } - - bulk.add(new UpdateOneModel<>( - eq("_id", doc.get("_id")), - new Document("$set", new Document() - .append("projects", doc.get("projects")) - )) - ); - }); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompleteSampleStatusDataModel.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompleteSampleStatusDataModel.java deleted file mode 100644 index 320c427244f..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompleteSampleStatusDataModel.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog.issue_1849; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "complete_sample_status_models", - description = "Complete Sample Status data models #1849", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20211126) -public class CompleteSampleStatusDataModel extends MigrationTool { - - @Override - protected void run() throws Exception { - migrateCollection(MongoDBAdaptorFactory.SAMPLE_COLLECTION, - new Document("status.id", new Document("$exists", false)), - Projections.include("_id", "status", "internal"), - (doc, bulk) -> { - CompleteStatusDataModelUtils.completeStatus(doc); - CompleteStatusDataModelUtils.completeInternalStatus(doc); - - bulk.add(new UpdateOneModel<>( - eq("_id", doc.get("_id")), - new Document("$set", new Document() - .append("status", doc.get("status")) - .append("internal", doc.get("internal")) - )) - ); - }); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompleteStatusDataModelUtils.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompleteStatusDataModelUtils.java deleted file mode 100644 index d9e270c3d9f..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompleteStatusDataModelUtils.java +++ /dev/null @@ -1,47 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog.issue_1849; - -import org.apache.commons.lang3.StringUtils; -import org.bson.Document; -import org.opencb.opencga.core.common.GitRepositoryState; - -public class CompleteStatusDataModelUtils { - - public static void completeStatus(Document document) { - Document status = document.get("status", Document.class); - if (status != null) { - String id = status.getString("id"); - String name = status.getString("name"); - if (StringUtils.isEmpty(id)) { - status.put("id", StringUtils.isNotEmpty(name) ? name : ""); - } - status.put("name", StringUtils.isNotEmpty(name) ? name : ""); - } else { - document.put("status", new Document() - .append("id", "READY") - .append("name", "") - .append("description", "") - .append("date", "") - ); - } - } - - public static void completeInternalStatus(Document document) { - Document internal = document.get("internal", Document.class); - - if (internal == null) { - internal = new Document(); - document.put("internal", internal); - } - - completeStatus(internal); - Document status = internal.get("status", Document.class); - status.put("version", GitRepositoryState.getInstance().getBuildVersion()); - status.put("commit", GitRepositoryState.getInstance().getCommitId()); - - String id = status.getString("id"); - if (StringUtils.isEmpty(id)) { - status.put("id", "READY"); - } - } - -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompleteStudyStatusDataModel.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompleteStudyStatusDataModel.java deleted file mode 100644 index 9972b3c1264..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompleteStudyStatusDataModel.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog.issue_1849; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "complete_study_status_models", - description = "Complete Study Status data models #1849", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20211126) -public class CompleteStudyStatusDataModel extends MigrationTool { - - @Override - protected void run() throws Exception { - migrateCollection(MongoDBAdaptorFactory.STUDY_COLLECTION, - new Document("status.id", new Document("$exists", false)), - Projections.include("_id", "status", "internal"), - (doc, bulk) -> { - CompleteStatusDataModelUtils.completeStatus(doc); - CompleteStatusDataModelUtils.completeInternalStatus(doc); - - bulk.add(new UpdateOneModel<>( - eq("_id", doc.get("_id")), - new Document("$set", new Document() - .append("status", doc.get("status")) - .append("internal", doc.get("internal")) - )) - ); - }); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompleteUserStatusDataModel.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompleteUserStatusDataModel.java deleted file mode 100644 index fecc5d30c3d..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompleteUserStatusDataModel.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog.issue_1849; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "complete_user_status_models", - description = "Complete User Status data models #1849", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20211126) -public class CompleteUserStatusDataModel extends MigrationTool { - - @Override - protected void run() throws Exception { - migrateCollection(MongoDBAdaptorFactory.USER_COLLECTION, - new Document("internal.status.id", new Document("$exists", false)), - Projections.include("_id", "internal"), - (doc, bulk) -> { - CompleteStatusDataModelUtils.completeInternalStatus(doc); - - bulk.add(new UpdateOneModel<>( - eq("_id", doc.get("_id")), - new Document("$set", new Document() - .append("internal", doc.get("internal")) - )) - ); - }); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issues_1853_1855/IndividualOntologyTermMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issues_1853_1855/IndividualOntologyTermMigration.java deleted file mode 100644 index b505c4b7d9c..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issues_1853_1855/IndividualOntologyTermMigration.java +++ /dev/null @@ -1,56 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog.issues_1853_1855; - -import com.mongodb.client.MongoCollection; -import com.mongodb.client.model.IndexOptions; -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.biodata.models.core.OntologyTermAnnotation; -import org.opencb.biodata.models.core.SexOntologyTermAnnotation; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import java.util.HashMap; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "individual_ontology_term_migration_#1853", - description = "Change sex and ethnicity types for OntologyTermAnnotation #1855", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20211203) -public class IndividualOntologyTermMigration extends MigrationTool { - - @Override - protected void run() throws Exception { - migrateCollection(MongoDBAdaptorFactory.INDIVIDUAL_COLLECTION, - new Document("sex.id", new Document("$exists", false)), - Projections.include("_id", "sex", "ethnicity"), - (doc, bulk) -> { - String sex = doc.getString("sex"); - SexOntologyTermAnnotation sexAnnotation = new SexOntologyTermAnnotation(sex, "", "", "", "", new HashMap<>()); - Document sexDoc = convertToDocument(sexAnnotation); - - String ethnicity = doc.getString("ethnicity"); - OntologyTermAnnotation ethnicityAnnotation = new OntologyTermAnnotation(ethnicity, "", "", "", "", new HashMap<>()); - Document ethnicityDoc = convertToDocument(ethnicityAnnotation); - - bulk.add(new UpdateOneModel<>( - eq("_id", doc.get("_id")), - new Document("$set", new Document() - .append("sex", sexDoc) - .append("ethnicity", ethnicityDoc) - )) - ); - } - ); - - MongoCollection collection = getMongoCollection(MongoDBAdaptorFactory.INDIVIDUAL_COLLECTION); - dropIndex(collection, new Document().append("sex", 1).append("studyUid", 1)); - dropIndex(collection, new Document().append("ethnicity", 1).append("studyUid", 1)); - createIndex(collection, new Document().append("sex.id", 1).append("studyUid", 1), new IndexOptions().background(true)); - createIndex(collection, new Document().append("ethnicity.id", 1).append("studyUid", 1), new IndexOptions().background(true)); - } - -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issues_1853_1855/NewStudyDataModelFields.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issues_1853_1855/NewStudyDataModelFields.java deleted file mode 100644 index 926a5d14a77..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issues_1853_1855/NewStudyDataModelFields.java +++ /dev/null @@ -1,42 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog.issues_1853_1855; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; -import org.opencb.opencga.core.models.study.StudyType; - -import java.util.Collections; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "new_study_data_model_fields_#1853", - description = "New Study 'sources', 'type' and 'additionalInfo' fields #1853", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20211202) -public class NewStudyDataModelFields extends MigrationTool { - - @Override - protected void run() throws Exception { - migrateCollection(MongoDBAdaptorFactory.STUDY_COLLECTION, - new Document("type", new Document("$exists", false)), - Projections.include("_id"), - (doc, bulk) -> { - Document typeDoc = convertToDocument(StudyType.init()); - - bulk.add(new UpdateOneModel<>( - eq("_id", doc.get("_id")), - new Document("$set", new Document() - .append("type", typeDoc) - .append("sources", Collections.emptyList()) - .append("additionalInfo", Collections.emptyList()) - )) - ); - } - ); - } - -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issues_1853_1855/SampleProcessingAndCollectionChanges.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issues_1853_1855/SampleProcessingAndCollectionChanges.java deleted file mode 100644 index 58d986553f9..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issues_1853_1855/SampleProcessingAndCollectionChanges.java +++ /dev/null @@ -1,131 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog.issues_1853_1855; - -import com.mongodb.client.MongoCollection; -import com.mongodb.client.model.IndexOptions; -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.bson.Document; -import org.bson.conversions.Bson; -import org.opencb.biodata.models.core.OntologyTermAnnotation; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; -import org.opencb.opencga.core.models.common.ExternalSource; -import org.opencb.opencga.core.models.sample.SampleCollection; -import org.opencb.opencga.core.models.sample.SampleProcessing; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "sample_source_treatments_#1854", - description = "Sample source, treatments, processing and collection changes #1854", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20211201, patch = 2) -public class SampleProcessingAndCollectionChanges extends MigrationTool { - - @Override - protected void run() throws Exception { - migrateCollection(MongoDBAdaptorFactory.SAMPLE_COLLECTION, - new Document("collection.from", new Document("$exists", false)), - Projections.include("_id", "collection", "processing"), - (doc, bulk) -> { - Document processing = doc.get("processing", Document.class); - if (processing != null) { - String product = processing.getString("product"); - if (StringUtils.isNotEmpty(product)) { - OntologyTermAnnotation ontologyTermAnnotation = - new OntologyTermAnnotation(product, "", "", "", "", Collections.emptyMap()); - Document ontologyDoc = convertToDocument(ontologyTermAnnotation); - processing.put("product", Collections.singletonList(ontologyDoc)); - } else { - processing.put("product", Collections.emptyList()); - } - } else { - SampleProcessing sampleProcessing = SampleProcessing.init(); - processing = convertToDocument(sampleProcessing); - } - - Document collection = doc.get("collection", Document.class); - if (collection != null) { - List ontologyTermAnnotationList = new ArrayList<>(); - String tissue = collection.getString("tissue"); - processOntologyTermAnnotationField(tissue, "tissue", ontologyTermAnnotationList); - String organ = collection.getString("organ"); - processOntologyTermAnnotationField(organ, "organ", ontologyTermAnnotationList); - - List ontologyDocList = convertToDocument(ontologyTermAnnotationList); - collection.put("from", ontologyDocList); - collection.put("type", ""); - collection.remove("tissue"); - collection.remove("organ"); - } else { - SampleCollection sampleCollection = SampleCollection.init(); - collection = convertToDocument(sampleCollection); - } - - ExternalSource source = ExternalSource.init(); - Document sourceDoc = convertToDocument(source); - - bulk.add(new UpdateOneModel<>( - eq("_id", doc.get("_id")), - new Document("$set", new Document() - .append("source", sourceDoc) - .append("processing", processing) - .append("collection", collection) - .append("treatments", Collections.emptyList()) - )) - ); - } - ); - - MongoCollection collection = getMongoCollection(MongoDBAdaptorFactory.SAMPLE_COLLECTION); - dropIndex(collection, new Document().append("processing.product", 1).append("studyUid", 1)); - dropIndex(collection, new Document().append("collection.tissue", 1).append("studyUid", 1)); - dropIndex(collection, new Document().append("collection.organ", 1).append("studyUid", 1)); - collection.createIndex(new Document().append("processing.product.id", 1).append("studyUid", 1), new IndexOptions().background(true)); - collection.createIndex(new Document().append("collection.from.id", 1).append("studyUid", 1), new IndexOptions().background(true)); - collection.createIndex(new Document().append("collection.type", 1).append("studyUid", 1), new IndexOptions().background(true)); - - // Patch 2 - Remove array from processing.product - migrateCollection(MongoDBAdaptorFactory.SAMPLE_COLLECTION, - new Document("processing.product", new Document("$exists", true)), - Projections.include("_id", "processing"), - (doc, bulk) -> { - Document processing = doc.get("processing", Document.class); - if (processing != null) { - if (processing.get("product") instanceof Document) { - // Already migrated document - return; - } - List product = processing.getList("product", Document.class); - if (CollectionUtils.isEmpty(product)) { - processing.put("product", new Document()); - } else { - processing.put("product", product.get(0)); - } - } else { - SampleProcessing sampleProcessing = SampleProcessing.init(); - processing = convertToDocument(sampleProcessing); - } - - bulk.add(new UpdateOneModel<>( - eq("_id", doc.get("_id")), - new Document("$set", new Document("processing", processing)) - ) - ); - } - ); - } - - private void processOntologyTermAnnotationField(String field, String source, List ontologyTermAnnotationList) { - if (StringUtils.isNotEmpty(field)) { - ontologyTermAnnotationList.add(new OntologyTermAnnotation(field, "", "", source, "", Collections.emptyMap())); - } - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/renameVariableSetFieldFromVariable.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/renameVariableSetFieldFromVariable.java deleted file mode 100644 index 71c5f703b19..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/renameVariableSetFieldFromVariable.java +++ /dev/null @@ -1,61 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.catalog; - - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import java.util.List; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "rename_variableset_field", - description = "Rename Variable variableSet field to variables #1823", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20210920) -public class renameVariableSetFieldFromVariable extends MigrationTool { - - @Override - protected void run() throws Exception { - migrateCollection(MongoDBAdaptorFactory.STUDY_COLLECTION, - new Document("variableSets.variables.variableSet", new Document("$exists", true)), - Projections.include("_id", StudyDBAdaptor.QueryParams.VARIABLE_SET.key()), - (studyDoc, bulk) -> { - List variablesets = studyDoc.getList(StudyDBAdaptor.QueryParams.VARIABLE_SET.key(), Document.class); - if (variablesets != null) { - for (Document variableset : variablesets) { - List variables = variableset.getList("variables", Document.class); - if (variables != null) { - for (Document variable : variables) { - renameVariableSet(variable); - } - } - } - - bulk.add(new UpdateOneModel<>( - eq("_id", studyDoc.get("_id")), - new Document("$set", new Document(StudyDBAdaptor.QueryParams.VARIABLE_SET.key(), variablesets)) - ) - ); - } - } - ); - } - - public void renameVariableSet(Document variable) { - List variableList = variable.getList("variableSet", Document.class); - if (variableList != null) { - for (Document nestedVariable : variableList) { - renameVariableSet(nestedVariable); - } - } - variable.put("variables", variableList); - variable.remove("variableSet"); - } - -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/storage/SynchronizeCatalogStorage.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/storage/SynchronizeCatalogStorage.java deleted file mode 100644 index 9e99b3a50cf..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/storage/SynchronizeCatalogStorage.java +++ /dev/null @@ -1,27 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.storage; - -import org.opencb.opencga.analysis.variant.metadata.CatalogStorageMetadataSynchronizer; -import org.opencb.opencga.app.migrations.StorageMigrationTool; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; - -@Migration(id = "synchronize_catalog_storage_1850_1851", - description = "Synchronize catalog storage #1850 #1851", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.STORAGE, - date = 20220118) -public class SynchronizeCatalogStorage extends StorageMigrationTool { - - @Override - protected void run() throws Exception { - for (String project : getVariantStorageProjects()) { - VariantStorageMetadataManager mm = getVariantStorageEngineByProject(project).getMetadataManager(); - for (String study : mm.getStudies().keySet()) { - logger.info("Synchronize storage-catalog for study {}", study); - new CatalogStorageMetadataSynchronizer(catalogManager, mm) - .synchronizeCatalogStudyFromStorage(study, token); - } - } - } - -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/storage/UpdateSampleIndexStatus.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/storage/UpdateSampleIndexStatus.java deleted file mode 100644 index 1bc4b2406c9..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/storage/UpdateSampleIndexStatus.java +++ /dev/null @@ -1,133 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.storage; - -import org.opencb.commons.ProgressLogger; -import org.opencb.opencga.app.migrations.StorageMigrationTool; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; -import org.opencb.opencga.storage.core.metadata.models.SampleMetadata; -import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; -import org.opencb.opencga.storage.core.metadata.models.TaskMetadata; -import org.opencb.opencga.storage.core.variant.VariantStorageEngine; - -import java.util.Iterator; - -@Migration(id = "update_sample_index_status_1901", - description = " Improve sample-index multi-schema management. #1901", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.STORAGE, - date = 20220302) -public class UpdateSampleIndexStatus extends StorageMigrationTool { - - // Deprecated to avoid confusion with actual "SAMPLE_INDEX_STATUS" - protected static final String SAMPLE_INDEX_ANNOTATION_STATUS_OLD = "sampleIndex"; - - protected static final String SAMPLE_INDEX_STATUS = "sampleIndexGenotypes"; - protected static final String SAMPLE_INDEX_VERSION = "sampleIndexGenotypesVersion"; - protected static final String SAMPLE_INDEX_ANNOTATION_STATUS = "sampleIndexAnnotation"; - protected static final String SAMPLE_INDEX_ANNOTATION_VERSION = "sampleIndexAnnotationVersion"; - - private static final String FAMILY_INDEX_STATUS = "family_index"; - - private final String markKey = "migration_" + getAnnotation().id(); - private final int markValue = getAnnotation().patch(); - - @Override - protected void run() throws Exception { - - for (String project : getVariantStorageProjects()) { - VariantStorageEngine engine = getVariantStorageEngineByProject(project); - if (!engine.getStorageEngineId().equals("hadoop")) { - logger.info("Skip project '{}'. VariantStorageEngine not in hadoop."); - continue; - } - VariantStorageMetadataManager metadataManager = engine.getMetadataManager(); - for (String study : metadataManager.getStudyNames()) { - int studyId = metadataManager.getStudyId(study); - Iterator it = metadataManager.sampleMetadataIterator(studyId); - ProgressLogger progressLogger = new ProgressLogger("Update samples from study " + study); - progressLogger.setBatchSize(1000); - int read = 0; - int modified = 0; - while (it.hasNext()) { - SampleMetadata sampleMetadata = it.next(); - if (needsMigration(sampleMetadata)) { - metadataManager.updateSampleMetadata(studyId, sampleMetadata.getId(), - this::updateSampleMetadata); - modified++; - } - read++; - progressLogger.increment(1); - } - logger.info("Finish update samples from study {}. {}/{} samples updated", study, modified, read); - } - } - } - - protected void updateSampleMetadata(SampleMetadata sampleMetadata) { - if (needsMigration(sampleMetadata)) { - addVersionToSampleIndexStatus(sampleMetadata); - renameOldSampleIndexAnnotationStatus(sampleMetadata); - addVersionToSampleIndexAnnotationStatus(sampleMetadata); - addVersionToFamilyIndexStatus(sampleMetadata); -// removeOldSampleIndexStatus(sampleMetadata); - addUpdatedMark(sampleMetadata); - } - } - - protected boolean needsMigration(SampleMetadata sampleMetadata) { - return sampleMetadata.isIndexed() && sampleMetadata.getAttributes().getInt(markKey, -1) != markValue; - } - - protected void addVersionToSampleIndexStatus(SampleMetadata sampleMetadata) { - // This is a new status. In case of missing value (null), assume it's READY - TaskMetadata.Status status = sampleMetadata.getStatus(SAMPLE_INDEX_STATUS, TaskMetadata.Status.READY); - if (status != null) { - int version = sampleMetadata.getAttributes().getInt(SAMPLE_INDEX_VERSION, -1); - if (version == -1) { - version = StudyMetadata.DEFAULT_SAMPLE_INDEX_VERSION; - } - sampleMetadata.setSampleIndexStatus(status, version); - } - } - - protected void renameOldSampleIndexAnnotationStatus(SampleMetadata sampleMetadata) { - TaskMetadata.Status annotationStatus = sampleMetadata.getStatus(SAMPLE_INDEX_ANNOTATION_STATUS, null); - if (annotationStatus == null) { - TaskMetadata.Status annotationOldStatus = sampleMetadata.getStatus(SAMPLE_INDEX_ANNOTATION_STATUS_OLD, null); - if (annotationOldStatus != null) { - // Use old as new status - sampleMetadata.setStatus(SAMPLE_INDEX_ANNOTATION_STATUS, annotationOldStatus); - } - } - } - - protected void addVersionToSampleIndexAnnotationStatus(SampleMetadata sampleMetadata) { - TaskMetadata.Status annotationStatus = sampleMetadata.getStatus(SAMPLE_INDEX_ANNOTATION_STATUS, null); - if (annotationStatus != null) { - int version = sampleMetadata.getAttributes().getInt(SAMPLE_INDEX_ANNOTATION_VERSION, -1); - if (version == -1) { - version = StudyMetadata.DEFAULT_SAMPLE_INDEX_VERSION; - } - sampleMetadata.setSampleIndexAnnotationStatus(annotationStatus, version); - } - } - - private void addVersionToFamilyIndexStatus(SampleMetadata sampleMetadata) { - if (sampleMetadata.getStatus(FAMILY_INDEX_STATUS) == TaskMetadata.Status.READY) { - sampleMetadata.setFamilyIndexStatus(TaskMetadata.Status.READY, StudyMetadata.DEFAULT_SAMPLE_INDEX_VERSION); - } - } - - private void removeOldSampleIndexStatus(SampleMetadata sampleMetadata) { - sampleMetadata.getStatus().remove(SAMPLE_INDEX_STATUS); - sampleMetadata.getStatus().remove(SAMPLE_INDEX_ANNOTATION_STATUS_OLD); - sampleMetadata.getStatus().remove(SAMPLE_INDEX_ANNOTATION_STATUS); - sampleMetadata.getStatus().remove(FAMILY_INDEX_STATUS); - } - - - private void addUpdatedMark(SampleMetadata sampleMetadata) { - sampleMetadata.getAttributes().put(markKey, markValue); - } - -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_1/catalog/AddMissingBiotypes.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_1/catalog/AddMissingBiotypes.java deleted file mode 100644 index f907c064812..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_1/catalog/AddMissingBiotypes.java +++ /dev/null @@ -1,62 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_1.catalog; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.db.mongodb.converters.StudyConverter; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; -import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.models.study.Variable; -import org.opencb.opencga.core.models.study.VariableSet; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "add_missing_biotypes", - description = "Add missing biotypes, #TASK-625", version = "2.2.1", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20220412) -public class AddMissingBiotypes extends MigrationTool { - - @Override - protected void run() throws Exception { - StudyConverter converter = new StudyConverter(); - List newBiotypes = Arrays.asList("vault_RNA", "guide_RNA", "lncRNA", "IG_LV_gene", "Mt_tRNA_pseudogene", "artifact", - "disrupted_domain"); - migrateCollection(MongoDBAdaptorFactory.STUDY_COLLECTION, - new Document(), - Projections.include("_id", "variableSets"), - (doc, bulk) -> { - Study study = converter.convertToDataModelType(doc); - for (VariableSet variableSet : study.getVariableSets()) { - if ("opencga_sample_variant_stats".equals(variableSet.getId()) - || "opencga_cohort_variant_stats".equals(variableSet.getId())) { - for (Variable variable : variableSet.getVariables()) { - if ("biotypeCount".equals(variable.getId())) { - List allowedKeys = new ArrayList<>(variable.getAllowedKeys()); - for (String newBiotype : newBiotypes) { - if (!allowedKeys.contains(newBiotype)) { - allowedKeys.add(newBiotype); - variable.setAllowedKeys(allowedKeys); - } - } - } - } - } - } - - List variableSetDocumentList = convertToDocument(study.getVariableSets()); - bulk.add(new UpdateOneModel<>( - eq("_id", doc.get("_id")), - new Document("$set", new Document("variableSets", variableSetDocumentList))) - ); - } - ); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_1/catalog/InitSharedProjectField.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_1/catalog/InitSharedProjectField.java deleted file mode 100644 index d0c2e7ea105..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_1/catalog/InitSharedProjectField.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_1.catalog; - -import com.mongodb.client.MongoCollection; -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.Updates; -import org.bson.Document; -import org.bson.conversions.Bson; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import java.util.Collections; - -@Migration(id = "init_shared_project", - description = "Initialise sharedProjects #TASK-702", version = "2.2.1", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20220502) -public class InitSharedProjectField extends MigrationTool { - - @Override - protected void run() throws Exception { - MongoCollection collection = getMongoCollection(MongoDBAdaptorFactory.USER_COLLECTION); - Bson query = Filters.or( - Filters.exists("sharedProjects", false), - Filters.eq("sharedProjects", null) - ); - Bson update = Updates.set("sharedProjects", Collections.emptyList()); - - collection.updateMany(query, update); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_1/storage/SampleIndexConfigurationAddMissingStatus.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_1/storage/SampleIndexConfigurationAddMissingStatus.java deleted file mode 100644 index 95b90716b59..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_1/storage/SampleIndexConfigurationAddMissingStatus.java +++ /dev/null @@ -1,36 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_1.storage; - -import org.opencb.opencga.app.migrations.StorageMigrationTool; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; -import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; -import org.opencb.opencga.storage.core.variant.VariantStorageEngine; - -@Migration(id = "sample_index_configuration_add_missing_status", - description = "Sample index configuration add missing status #TASK-512", version = "2.2.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.STORAGE, - date = 20220328) -public class SampleIndexConfigurationAddMissingStatus extends StorageMigrationTool { - @Override - protected void run() throws Exception { - for (String project : getVariantStorageProjects()) { - VariantStorageEngine engine = getVariantStorageEngineByProject(project); - VariantStorageMetadataManager metadataManager = engine.getMetadataManager(); - for (Integer studyId : metadataManager.getStudyIds()) { - StudyMetadata study = metadataManager.getStudyMetadata(studyId); - if (study.getSampleIndexConfigurations() != null && - study.getSampleIndexConfigurations().stream().anyMatch(sc -> sc.getStatus() == null)) { - logger.info("Update study {} as it contains sample index configurations without Status", study.getName()); - metadataManager.updateStudyMetadata(studyId, sm -> { - for (StudyMetadata.SampleIndexConfigurationVersioned sc : sm.getSampleIndexConfigurations()) { - if (sc.getStatus() == null) { - sc.setStatus(StudyMetadata.SampleIndexConfigurationVersioned.Status.ACTIVE); - } - } - }); - } - } - } - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_3_0/catalog/AddLockedToInterpretation.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_3_0/catalog/AddLockedToInterpretation.java deleted file mode 100644 index 3a03510cb2c..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_3_0/catalog/AddLockedToInterpretation.java +++ /dev/null @@ -1,42 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_3_0.catalog; - -import com.mongodb.client.MongoCollection; -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.Updates; -import org.bson.Document; -import org.bson.conversions.Bson; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import java.util.Arrays; - -@Migration(id = "add_locked_to_interpretation_TASK-552", - description = "Add new 'locked' field to Interpretation #TASK-552", version = "2.3.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20220401) -public class AddLockedToInterpretation extends MigrationTool { - - @Override - protected void run() throws Exception { - logger.info("Creating new index in Interpretation collection"); - MongoCollection interpretationCollection = getMongoCollection(MongoDBAdaptorFactory.INTERPRETATION_COLLECTION); - Document lockedIndex = new Document() - .append("locked", 1) - .append("studyUid", 1); - createIndex(interpretationCollection, lockedIndex); - - logger.info("Propagating all ClinicalAnalysis 'locked' values to the related Interpretations"); - queryMongo(MongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, new Document(), - Projections.include(Arrays.asList("id", "studyUid", "locked")), (doc) -> { - Bson query = Filters.and( - Filters.eq("studyUid", doc.get("studyUid")), - Filters.eq("clinicalAnalysisId", doc.getString("id")) - ); - Bson update = Updates.set("locked", doc.getBoolean("locked")); - interpretationCollection.updateMany(query, update); - }); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_3_0/catalog/AddPanelInternalField.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_3_0/catalog/AddPanelInternalField.java deleted file mode 100644 index 8450bf77ca5..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_3_0/catalog/AddPanelInternalField.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_3_0.catalog; - -import com.mongodb.client.MongoCollection; -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.Updates; -import org.bson.Document; -import org.bson.conversions.Bson; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; -import org.opencb.opencga.core.models.panel.PanelInternal; - -@Migration(id = "add_panel_internal_field-TASK_734", - description = "Add 'internal' to Panel data model #TASK-734", version = "2.3.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20220509) -public class AddPanelInternalField extends MigrationTool { - - @Override - protected void run() throws Exception { - MongoCollection collection = getMongoCollection(MongoDBAdaptorFactory.PANEL_COLLECTION); - - PanelInternal internal = PanelInternal.init(); - internal.getStatus().setDescription("Internal status added during migration"); - Document internalDoc = convertToDocument(internal); - - Bson query = Filters.exists("internal", false); - Bson update = Updates.set("internal", internalDoc); - collection.updateMany(query, update); - - Document panelIndex = new Document() - .append("internal.status.name", 1) - .append("studyUid", 1); - createIndex(collection, panelIndex); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_3_0/catalog/AutoIncrementVersion.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_3_0/catalog/AutoIncrementVersion.java deleted file mode 100644 index f98ef6aec6a..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_3_0/catalog/AutoIncrementVersion.java +++ /dev/null @@ -1,60 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_3_0.catalog; - -import com.mongodb.client.MongoCollection; -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.UpdateOneModel; -import com.mongodb.client.model.UpdateOptions; -import com.mongodb.client.model.Updates; -import com.mongodb.client.result.DeleteResult; -import org.bson.Document; -import org.bson.conversions.Bson; -import org.opencb.commons.datastore.mongodb.GenericDocumentComplexConverter; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptor; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -@Migration(id = "autoincrement_version_TASK-323", - description = "Reorganise data in new collections (autoincrement version) #TASK-323", version = "2.3.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - offline = true, - date = 20220512) -public class AutoIncrementVersion extends MigrationTool { - - private void reorganiseData(String collection, String archiveCollection) { - logger.info("Copying data from '{}' to '{}' collection", collection, archiveCollection); - // Replicate all the data in the archive collection - migrateCollection(collection, archiveCollection, new Document(), new Document(), - ((document, bulk) -> { - Document replacedDotsDoc = GenericDocumentComplexConverter.replaceDots(document); - Bson query = Filters.eq("_id", replacedDotsDoc.get("_id")); - Bson update = Updates.setOnInsert(replacedDotsDoc); - bulk.add(new UpdateOneModel<>(query, update, new UpdateOptions().upsert(true))); - })); - - // Delete all documents that are not lastOfVersion from main collection - logger.info("Removing outdated data (_lastOfVersion = false) from '{}' collection", collection); - MongoCollection mongoCollection = getMongoCollection(collection); - DeleteResult deleteResult = mongoCollection.deleteMany(Filters.eq(MongoDBAdaptor.LAST_OF_VERSION, false)); - logger.info("{} documents removed from '{}' collection", deleteResult.getDeletedCount(), collection); - } - - @Override - protected void run() throws Exception { - // Step 1: Create all indexes - logger.info("Creating indexes in new archive collections..."); - catalogManager.installIndexes(token); - - // Step 2: Replicate data to archive collections and delete documents that are not the last of version from main collection - reorganiseData(MongoDBAdaptorFactory.SAMPLE_COLLECTION, MongoDBAdaptorFactory.SAMPLE_ARCHIVE_COLLECTION); - reorganiseData(MongoDBAdaptorFactory.INDIVIDUAL_COLLECTION, MongoDBAdaptorFactory.INDIVIDUAL_ARCHIVE_COLLECTION); - reorganiseData(MongoDBAdaptorFactory.FAMILY_COLLECTION, MongoDBAdaptorFactory.FAMILY_ARCHIVE_COLLECTION); - reorganiseData(MongoDBAdaptorFactory.PANEL_COLLECTION, MongoDBAdaptorFactory.PANEL_ARCHIVE_COLLECTION); - - // Reduce batch size for interpretations - setBatchSize(1); - reorganiseData(MongoDBAdaptorFactory.INTERPRETATION_COLLECTION, MongoDBAdaptorFactory.INTERPRETATION_ARCHIVE_COLLECTION); - } - -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_3_0/catalog/InitSampleIndexConfiguration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_3_0/catalog/InitSampleIndexConfiguration.java deleted file mode 100644 index c03d1bac3cb..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_3_0/catalog/InitSampleIndexConfiguration.java +++ /dev/null @@ -1,47 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_3_0.catalog; - -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.Projections; -import org.bson.conversions.Bson; -import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; -import org.opencb.opencga.app.migrations.StorageMigrationTool; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.core.config.storage.SampleIndexConfiguration; - -import java.util.LinkedList; -import java.util.List; - -@Migration(id = "init_sampleIndexConfiguration-TASK550", - description = "Initialise SampleIndexConfiguration in Study internal #TASK-550", version = "2.3.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20220502) -public class InitSampleIndexConfiguration extends StorageMigrationTool { - - @Override - protected void run() throws Exception { - - Bson query = Filters.or( - Filters.exists("internal.configuration.variantEngine.sampleIndex", false), - Filters.eq("internal.configuration.variantEngine.sampleIndex", null) - ); - Bson projection = Projections.include("fqn"); - List fqnList = new LinkedList<>(); - queryMongo(MongoDBAdaptorFactory.STUDY_COLLECTION, query, projection, (d) -> fqnList.add(d.getString("fqn"))); - - for (String study : fqnList) { - VariantStorageManager manager = getVariantStorageManager(); - SampleIndexConfiguration configuration; - if (manager.exists(study, token)) { - configuration = manager.getStudyMetadata(study, token).getSampleIndexConfigurationLatest(true).getConfiguration(); - } else { - configuration = SampleIndexConfiguration.defaultConfiguration(); - } - // add - catalogManager.getStudyManager() - .setVariantEngineConfigurationSampleIndex(study, configuration, token); - } - } - -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_3_0/catalog/MoveCellbaseFromProjectInternalToProject.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_3_0/catalog/MoveCellbaseFromProjectInternalToProject.java deleted file mode 100644 index 2eb34cb9009..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_3_0/catalog/MoveCellbaseFromProjectInternalToProject.java +++ /dev/null @@ -1,46 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_3_0.catalog; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import com.mongodb.client.model.Updates; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import java.util.List; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "move_cellbase_from_project_internal_to_project_TASK-354", - description = "Move cellbase from project.internal to project #TASK-354", version = "2.3.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20220422) - -public class MoveCellbaseFromProjectInternalToProject extends MigrationTool { - - @Override - protected void run() throws Exception { - migrateCollection(MongoDBAdaptorFactory.USER_COLLECTION, - new Document("projects.id", new Document("$exists", true)), - Projections.include("_id", "projects"), - (user, bulk) -> { - List projects = user.getList("projects", Document.class); - for (Document project : projects) { - Document internal = project.get("internal", Document.class); - if (internal != null) { - Document cellbase = internal.get("cellbase", Document.class); - project.put("cellbase", cellbase); - internal.remove("cellbase"); - } - } - - bulk.add(new UpdateOneModel<>( - eq("_id", user.get("_id")), - Updates.set("projects", projects))); - } - ); - } - -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_3_0/catalog/RenameDeletedCollections.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_3_0/catalog/RenameDeletedCollections.java deleted file mode 100644 index 9455bdb9883..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_3_0/catalog/RenameDeletedCollections.java +++ /dev/null @@ -1,43 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_3_0.catalog; - -import com.mongodb.MongoNamespace; -import com.mongodb.client.MongoCollection; -import org.bson.Document; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import java.util.HashMap; -import java.util.Map; - -import static org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory.*; - -@Migration(id = "rename_deleted_collections_TASK-359", - description = "Rename deleted collections #TASK-359", version = "2.3.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20220502) -public class RenameDeletedCollections extends MigrationTool { - - @Override - protected void run() throws Exception { - - Map collectionRename = new HashMap<>(); - collectionRename.put(OLD_DELETED_USER_COLLECTION, DELETED_USER_COLLECTION); - collectionRename.put(OLD_DELETED_STUDY_COLLECTION, DELETED_STUDY_COLLECTION); - collectionRename.put(OLD_DELETED_FILE_COLLECTION, DELETED_FILE_COLLECTION); - collectionRename.put(OLD_DELETED_JOB_COLLECTION, DELETED_JOB_COLLECTION); - collectionRename.put(OLD_DELETED_SAMPLE_COLLECTION, DELETED_SAMPLE_COLLECTION); - collectionRename.put(OLD_DELETED_INDIVIDUAL_COLLECTION, DELETED_INDIVIDUAL_COLLECTION); - collectionRename.put(OLD_DELETED_COHORT_COLLECTION, DELETED_COHORT_COLLECTION); - collectionRename.put(OLD_DELETED_FAMILY_COLLECTION, DELETED_FAMILY_COLLECTION); - collectionRename.put(OLD_DELETED_PANEL_COLLECTION, DELETED_PANEL_COLLECTION); - collectionRename.put(OLD_DELETED_CLINICAL_ANALYSIS_COLLECTION, DELETED_CLINICAL_ANALYSIS_COLLECTION); - collectionRename.put(OLD_DELETED_INTERPRETATION_COLLECTION, DELETED_INTERPRETATION_COLLECTION); - - for (Map.Entry entry : collectionRename.entrySet()) { - MongoCollection collection = getMongoCollection(entry.getKey()); - collection.renameCollection(new MongoNamespace(dbAdaptorFactory.getMongoDataStore().getDatabaseName(), entry.getValue())); - } - - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_3_0/catalog/RenameDeprecatedVariantStorageToolId.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_3_0/catalog/RenameDeprecatedVariantStorageToolId.java deleted file mode 100644 index e89a7b8969c..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_3_0/catalog/RenameDeprecatedVariantStorageToolId.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_3_0.catalog; - -import com.mongodb.client.MongoCollection; -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.Updates; -import org.bson.Document; -import org.bson.conversions.Bson; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -@Migration(id = "rename_variant_storage_tool_ids_TASK-705", - description = "Rename variant storage tool ids #TASK-705", version = "2.3.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20220506) -public class RenameDeprecatedVariantStorageToolId extends MigrationTool { - - @Override - protected void run() throws Exception { - renameToolId("variant-sample-index", " variant-secondary-sample-index"); - renameToolId("variant-secondary-index", "variant-secondary-annotation-index"); - } - - private void renameToolId(String oldToolId, String newToolId) { - MongoCollection collection = getMongoCollection(MongoDBAdaptorFactory.JOB_COLLECTION); - Bson query = Filters.eq("tool.id", oldToolId); - Bson update = Updates.set("tool.id", newToolId); - collection.updateMany(query, update); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_3_0/catalog/UpdateJWTSecretKey.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_3_0/catalog/UpdateJWTSecretKey.java deleted file mode 100644 index c8c3ffd644c..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_3_0/catalog/UpdateJWTSecretKey.java +++ /dev/null @@ -1,27 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_3_0.catalog; - -import com.mongodb.client.MongoCollection; -import com.mongodb.client.model.Updates; -import org.bson.Document; -import org.opencb.opencga.catalog.auth.authentication.JwtManager; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; -import org.opencb.opencga.core.common.PasswordUtils; - -@Migration(id = "update_jwt_secret_key_TASK-807", - description = "Update JWT secret key #TASK-807", version = "2.3.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20220513) -public class UpdateJWTSecretKey extends MigrationTool { - - @Override - protected void run() throws Exception { - MongoCollection collection = getMongoCollection(MongoDBAdaptorFactory.METADATA_COLLECTION); - collection.updateOne(new Document(), Updates.set("admin.secretKey", - PasswordUtils.getStrongRandomPassword(JwtManager.SECRET_KEY_MIN_LENGTH))); - logger.info("JWT secret key successfully changed"); - } - -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_3_0/storage/AddMissingColumnsToPhoenix.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_3_0/storage/AddMissingColumnsToPhoenix.java deleted file mode 100644 index 590b1ce895f..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_3_0/storage/AddMissingColumnsToPhoenix.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_3_0.storage; - -import org.opencb.opencga.app.migrations.StorageMigrationTool; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.storage.core.variant.VariantStorageEngine; - -@Migration(id="add_missing_columns_to_phoenix_TASK-789", description = "Add missing 1000G columns to phoenix #TASK-789", - version = "2.3.0", domain = Migration.MigrationDomain.STORAGE, date = 20220614 -) -public class AddMissingColumnsToPhoenix extends StorageMigrationTool { - - @Override - protected void run() throws Exception { - for (String project : getVariantStorageProjects()) { - VariantStorageEngine engine = getVariantStorageEngineByProject(project); - if (engine.getStorageEngineId().equals("hadoop")) { - logger.info("Adding missing columns (if any) for project " + project); - Class aClass = Class.forName("org.opencb.opencga.storage.hadoop.variant.migration.v2_3_0.AddMissingColumns"); - Runnable runnable = (Runnable) aClass - .getConstructor(Object.class) - .newInstance(engine); - runnable.run(); - } - } - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_3_1/catalog/MoveCellbaseFromProjectInternalToProjectBugFix.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_3_1/catalog/MoveCellbaseFromProjectInternalToProjectBugFix.java deleted file mode 100644 index e1f84b3a182..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_3_1/catalog/MoveCellbaseFromProjectInternalToProjectBugFix.java +++ /dev/null @@ -1,46 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_3_1.catalog; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import com.mongodb.client.model.Updates; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import java.util.List; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "bugfix_move_cellbase_from_project_internal_to_project_TASK-1100", - description = "Bugfix: Move cellbase from project.internal to project #TASK-1100", version = "2.3.1", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20220620) - -public class MoveCellbaseFromProjectInternalToProjectBugFix extends MigrationTool { - - @Override - protected void run() throws Exception { - migrateCollection(MongoDBAdaptorFactory.USER_COLLECTION, - new Document("projects.internal.cellbase", new Document("$exists", true)), - Projections.include("_id", "projects"), - (user, bulk) -> { - List projects = user.getList("projects", Document.class); - for (Document project : projects) { - Document internal = project.get("internal", Document.class); - if (internal != null) { - Document cellbase = internal.get("cellbase", Document.class); - project.put("cellbase", cellbase); - internal.remove("cellbase"); - } - } - - bulk.add(new UpdateOneModel<>( - eq("_id", user.get("_id")), - Updates.set("projects", projects))); - } - ); - } - -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_0/catalog/AddClinicalStatusType_607.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_0/catalog/AddClinicalStatusType_607.java deleted file mode 100644 index b3dcffc1eea..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_0/catalog/AddClinicalStatusType_607.java +++ /dev/null @@ -1,77 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_4_0.catalog; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; -import org.opencb.opencga.core.models.clinical.ClinicalAnalysis; -import org.opencb.opencga.core.models.clinical.ClinicalStatusValue; -import org.opencb.opencga.core.models.study.configuration.ClinicalAnalysisStudyConfiguration; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "add_clinical_status_type_TASK-607", - description = "Automatically close cases depending on the new clinical status type #TASK-607", version = "2.4.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20220610) -public class AddClinicalStatusType_607 extends MigrationTool { - - private void addStatusType(Document status, Map defaultStatusTypeMap) { - for (Map.Entry entry : status.entrySet()) { - List statusValueList = (List) entry.getValue(); - for (Document statusValue : statusValueList) { - String id = statusValue.getString("id"); - statusValue.put("type", defaultStatusTypeMap.getOrDefault(id, "UNKNOWN")); - } - } - } - - @Override - protected void run() throws Exception { - ClinicalAnalysisStudyConfiguration clinicalConfiguration = ClinicalAnalysisStudyConfiguration.defaultConfiguration(); - Map caseStatusTypeMap = new HashMap<>(); - Map interpretationStatusTypeMap = new HashMap<>(); - - // Default statuses and types are the same for every ClinicalAnalysis.Type, so we can take this shortcut - List statuses = clinicalConfiguration.getStatus().get(ClinicalAnalysis.Type.FAMILY); - for (ClinicalStatusValue clinicalStatusValue : statuses) { - caseStatusTypeMap.put(clinicalStatusValue.getId(), clinicalStatusValue.getType().name()); - } - statuses = clinicalConfiguration.getInterpretation().getStatus().get(ClinicalAnalysis.Type.FAMILY); - for (ClinicalStatusValue status : statuses) { - interpretationStatusTypeMap.put(status.getId(), status.getType().name()); - } - - migrateCollection(MongoDBAdaptorFactory.STUDY_COLLECTION, - new Document("internal.configuration.clinical.status.FAMILY.type", new Document("$exists", false)), - Projections.include("internal.configuration.clinical"), - (doc, bulk) -> { - Document internal = doc.get("internal", Document.class); - Document configuration = internal.get("configuration", Document.class); - Document clinicalConfig = configuration.get("clinical", Document.class); - - // ClinicalAnalysis status type - Document status = clinicalConfig.get("status", Document.class); - addStatusType(status, caseStatusTypeMap); - - Document interpretation = clinicalConfig.get("interpretation", Document.class); - status = interpretation.get("status", Document.class); - addStatusType(status, interpretationStatusTypeMap); - - bulk.add(new UpdateOneModel<>( - eq("_id", doc.get("_id")), - new Document("$set", new Document("internal.configuration.clinical", clinicalConfig)) - ) - ); - } - ); - } - -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_0/catalog/AddPrivateDueDate_803.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_0/catalog/AddPrivateDueDate_803.java deleted file mode 100644 index 1fc9886f043..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_0/catalog/AddPrivateDueDate_803.java +++ /dev/null @@ -1,51 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_4_0.catalog; - -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.apache.commons.lang3.StringUtils; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; -import org.opencb.opencga.core.common.TimeUtils; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "add_private_dueDate_TASK-803", - description = "Add private _dueDate to ClinicalAnalysis #TASK-803", version = "2.4.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20220530) -public class AddPrivateDueDate_803 extends MigrationTool { - - @Override - protected void run() throws Exception { - createIndex(MongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, new Document() - .append("_dueDate", 1) - .append("studyUid", 1) - ); - - migrateCollection(MongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, - Filters.exists("_dueDate", false), Projections.include("dueDate"), (doc, bulk) -> { - Document update = new Document(); - String dueDate = doc.getString("dueDate"); - if (StringUtils.isEmpty(dueDate)) { - dueDate = TimeUtils.getTime(TimeUtils.add1MonthtoDate(TimeUtils.getDate())); - update.put("dueDate", dueDate); - } - update.put("_dueDate", TimeUtils.toDate(dueDate)); - - bulk.add(new UpdateOneModel<>( - eq("_id", doc.get("_id")), - new Document("$set", update) - ) - ); - }); - - dropIndex(MongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, new Document() - .append("dueDate", 1) - .append("studyUid", 1)); - } - -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_0/catalog/IndexPanelSource.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_0/catalog/IndexPanelSource.java deleted file mode 100644 index eb07b135cd4..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_0/catalog/IndexPanelSource.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_4_0.catalog; - -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import java.util.Arrays; - -@Migration(id = "index_panel_source_TASK-473", - description = "Index panel source #TASK-473", version = "2.4.0", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20220530) -public class IndexPanelSource extends MigrationTool { - - @Override - protected void run() throws Exception { - createIndexes(Arrays.asList( - MongoDBAdaptorFactory.PANEL_COLLECTION, - MongoDBAdaptorFactory.PANEL_ARCHIVE_COLLECTION - ), - Arrays.asList( - new Document() - .append("source.id", 1) - .append("studyUid", 1), - new Document() - .append("source.name", 1) - .append("studyUid", 1) - ) - ); - - } - -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_10/catalog/AddMissingEndDateOnFinishedJobs.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_10/catalog/AddMissingEndDateOnFinishedJobs.java deleted file mode 100644 index d2ddd32af9b..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_10/catalog/AddMissingEndDateOnFinishedJobs.java +++ /dev/null @@ -1,69 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_4_10.catalog; - -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; -import org.opencb.opencga.core.common.TimeUtils; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "add_missing_endDate_on_finished_jobs" , - description = "Add missing end date on finished jobs", - version = "2.4.10", - domain = Migration.MigrationDomain.CATALOG, - language = Migration.MigrationLanguage.JAVA, - date = 20221026 -) -public class AddMissingEndDateOnFinishedJobs extends MigrationTool { - - @Override - protected void run() throws Exception { - migrateCollection(Arrays.asList(MongoDBAdaptorFactory.JOB_COLLECTION, MongoDBAdaptorFactory.DELETED_JOB_COLLECTION), - Filters.and(Arrays.asList( - Filters.in("internal.status.id", Arrays.asList("DONE", "ERROR", "ABORTED", "UNKNOWN", "READY")), - Filters.or(Arrays.asList( - Filters.exists("execution.end", false), - Filters.eq("execution.end", null)) - ) - )), - Projections.include(Arrays.asList("execution", "_modificationDate")), - (document, bulk) -> { - Document execution = document.get("execution", Document.class); - if (execution != null) { - // Add end date - Object modificationDate = document.get("_modificationDate"); - if (modificationDate == null) { - modificationDate = TimeUtils.getDate(); - } - execution.put("end", modificationDate); - - // Add event - List events = execution.getList("events", Document.class); - if (events == null) { - events = new ArrayList<>(); - } else { - events = new ArrayList<>(events); - } - events.add(new Document() - .append("type", "WARNING") - .append("id", "missing-execution-end-date") - .append("message", "Missing execution field 'end'. End date set as part of migration fix.")); - execution.put("events", events); - - bulk.add(new UpdateOneModel<>( - eq("_id", document.get("_id")), - new Document("$set", new Document("execution", execution))) - ); - } - }); - } - -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_11/catalog/SignatureFittingsMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_11/catalog/SignatureFittingsMigration.java deleted file mode 100644 index 0f3401e0755..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_11/catalog/SignatureFittingsMigration.java +++ /dev/null @@ -1,61 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_4_11.catalog; - -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.apache.commons.collections4.CollectionUtils; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "signature_fittings" , - description = "Replace fitting for fittings in Signature", - version = "2.4.11", - domain = Migration.MigrationDomain.CATALOG, - language = Migration.MigrationLanguage.JAVA, - date = 20221109 -) -public class SignatureFittingsMigration extends MigrationTool { - @Override - protected void run() throws Exception { - migrateCollection(Arrays.asList(MongoDBAdaptorFactory.SAMPLE_COLLECTION, MongoDBAdaptorFactory.SAMPLE_ARCHIVE_COLLECTION, - MongoDBAdaptorFactory.DELETED_SAMPLE_COLLECTION), - Filters.exists("qualityControl.variant.signatures.fitting"), - Projections.include(Collections.singletonList("qualityControl.variant.signatures")), - (document, bulk) -> { - Document qc = document.get("qualityControl", Document.class); - if (qc == null) { - return; - } - Document variant = qc.get("variant", Document.class); - if (variant == null) { - return; - } - List signatures = variant.getList("signatures", Document.class); - if (CollectionUtils.isNotEmpty(signatures)) { - for (Document signature : signatures) { - if (signature != null) { - Document fitting = signature.get("fitting", Document.class); - if (fitting != null) { - fitting.put("id", "fitting-1"); - signature.put("fittings", Collections.singletonList(fitting)); - signature.remove("fitting"); - } - } - } - - bulk.add(new UpdateOneModel<>( - eq("_id", document.get("_id")), - new Document("$set", new Document("qualityControl.variant.signatures", signatures))) - ); - } - }); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_2/catalog/AddClinicalDiscussion.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_2/catalog/AddClinicalDiscussion.java deleted file mode 100644 index 240c7dd86fe..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_2/catalog/AddClinicalDiscussion.java +++ /dev/null @@ -1,131 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_4_2.catalog; - -import com.mongodb.client.MongoCollection; -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.bson.Document; -import org.opencb.biodata.models.clinical.ClinicalDiscussion; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; -import org.opencb.opencga.core.common.TimeUtils; - -import java.util.Arrays; -import java.util.List; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "add_clinical_discussion_TASK-1472", - description = "Add ClinicalDiscussion #TASK-1472", version = "2.4.2", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20220727) -public class AddClinicalDiscussion extends MigrationTool { - - @Override - protected void run() throws Exception { - // Replace discussion from ClinicalAnalysis report.discussion - migrateCollection( - Arrays.asList(MongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, MongoDBAdaptorFactory.DELETED_CLINICAL_ANALYSIS_COLLECTION), - new Document("report.discussion.author", new Document("$exists", false)), - Projections.include("report", "analyst"), - ((document, bulk) -> { - Document report = document.get("report", Document.class); - if (report != null) { - String author = extractAuthor(document); - Document discussionDoc = migrateDiscussion(report, author);; - bulk.add(new UpdateOneModel<>( - eq("_id", document.get("_id")), - new Document("$set", new Document("report.discussion", discussionDoc)) - ) - ); - } - }) - ); - - // Replace discussion from ClinicalVariants from Interpretations primaryFindings.discussion, secondaryFindings.discussion - for (String collectionString : Arrays.asList(MongoDBAdaptorFactory.INTERPRETATION_COLLECTION, - MongoDBAdaptorFactory.INTERPRETATION_ARCHIVE_COLLECTION, MongoDBAdaptorFactory.DELETED_INTERPRETATION_COLLECTION)) { - logger.info("Migrating documents from collection '{}'", collectionString); - MongoCollection collection = getMongoCollection(collectionString); - queryMongo(collectionString, - new Document("$or", Arrays.asList( - new Document("primaryFindings.discussion.author", new Document("$exists", false)), - new Document("secondaryFindings.discussion.author", new Document("$exists", false)) - )), - Projections.include("primaryFindings", "secondaryFindings", "analyst", "studyUid", "id"), - (document) -> { - String author = extractAuthor(document); - List primaryFindings = document.getList("primaryFindings", Document.class); - List secondaryFindings = document.getList("secondaryFindings", Document.class); - if (CollectionUtils.isNotEmpty(primaryFindings)) { - for (Document primaryFinding : primaryFindings) { - migrateClinicalVariant(primaryFinding, author); - } - } - if (CollectionUtils.isNotEmpty(secondaryFindings)) { - for (Document secondaryFinding : secondaryFindings) { - migrateClinicalVariant(secondaryFinding, author); - } - } - - try { - collection.updateOne(eq("_id", document.get("_id")), - new Document("$set", new Document() - .append("primaryFindings", primaryFindings) - .append("secondaryFindings", secondaryFindings))); - } catch (Exception e) { - logger.warn("Could not add ClinicalDiscussion to Interpretation '{}' from study uid {}. Error message: {}", - document.getString("id"), document.get("studyUid", Number.class), e.getMessage()); - } - }); - } - } - - private void migrateClinicalVariant(Document finding, String author) { - // Migrate discussion in root - migrateDiscussion(finding, author); - - // Iterate over evidences - List evidences = finding.getList("evidences", Document.class); - if (CollectionUtils.isNotEmpty(evidences)) { - for (Document evidence : evidences) { - Document review = evidence.get("review", Document.class); - if (review != null) { - migrateDiscussion(review, author); - } - } - } - } - - private Document migrateDiscussion(Document document, String author) { - Object discussion = document.get("discussion"); - if (discussion == null || discussion instanceof String) { - if (discussion != null) { - ClinicalDiscussion cDiscussion = new ClinicalDiscussion(author, TimeUtils.getTime(), String.valueOf(discussion)); - discussion = convertToDocument(cDiscussion); - } else { - discussion = new Document(); - } - } - // Replace discussion field - document.put("discussion", discussion); - - return (Document) discussion; - } - - private String extractAuthor(Document document) { - String author = ""; - Document analyst = document.get("analyst", Document.class); - if (analyst != null) { - String id = analyst.getString("id"); - if (StringUtils.isNotEmpty(id)) { - author = id; - } - } - return author; - } - -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_2/catalog/RecoverProbandSamplesInCases.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_2/catalog/RecoverProbandSamplesInCases.java deleted file mode 100644 index 602c57f90c7..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_2/catalog/RecoverProbandSamplesInCases.java +++ /dev/null @@ -1,242 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_4_2.catalog; - -import com.mongodb.client.MongoCollection; -import com.mongodb.client.MongoCursor; -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.Sorts; -import org.apache.commons.collections4.CollectionUtils; -import org.bson.Document; -import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.commons.datastore.core.Query; -import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.catalog.db.api.ClinicalAnalysisDBAdaptor; -import org.opencb.opencga.catalog.db.api.IndividualDBAdaptor; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.db.mongodb.converters.ClinicalAnalysisConverter; -import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; -import org.opencb.opencga.core.models.clinical.ClinicalAnalysis; -import org.opencb.opencga.core.models.common.Enums; -import org.opencb.opencga.core.models.individual.Individual; -import org.opencb.opencga.core.models.sample.Sample; - -import java.util.*; - -@Migration(id = "recover_proband_samples_in_cases_TASK-1470", - description = "Recover lost samples in clinical collection #TASK-1470", version = "2.4.2", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20220725, - patch = 2) -public class RecoverProbandSamplesInCases extends MigrationTool { - - @Override - protected void run() throws Exception { - MongoCollection audit = getMongoCollection(MongoDBAdaptorFactory.AUDIT_COLLECTION); - ClinicalAnalysisConverter converter = new ClinicalAnalysisConverter(); - - queryMongo(MongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, - new Document("$or", Arrays.asList( - new Document("proband.samples", null), - new Document("proband.samples", Collections.emptyList()) - )), - Projections.include("uid", "id", "uuid", "studyUid", "proband", "type"), (doc) -> { - ClinicalAnalysis currentCase = converter.convertToDataModelType(doc); - logger.info("Trying to recover Clinical Analysis [id: {}, uid: {}, uuid: {}]", currentCase.getId(), - currentCase.getUid(), currentCase.getUuid()); - - if (currentCase.getProband() == null) { - logger.warn("Proband not defined in ClinicalAnalysis '{}'. Skipping...", currentCase.getId()); - return; - } - - ClinicalAnalysis clinicalAnalysis = findUpdatedCaseInAudit(currentCase.getUuid(), currentCase.getProband().getId(), - audit, converter); - if (clinicalAnalysis == null) { - clinicalAnalysis = findCreatedCaseInAudit(currentCase.getUuid(), currentCase.getProband().getId(), audit, - converter); - } - if (clinicalAnalysis == null) { - recoverWithoutAudit(currentCase); - } else { - recoverFromAudit(currentCase, clinicalAnalysis); - } - }); - } - - private void recoverWithoutAudit(ClinicalAnalysis currentCase) { - ClinicalAnalysis.Type type = currentCase.getType(); - - // The proband from the audit matches the one used in the case currently :) - Individual proband; - try { - Query query = new Query() - .append(IndividualDBAdaptor.QueryParams.UID.key(), currentCase.getProband().getUid()) - .append(IndividualDBAdaptor.QueryParams.VERSION.key(), currentCase.getProband().getVersion()); - proband = dbAdaptorFactory.getCatalogIndividualDBAdaptor().get(query, - new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(IndividualDBAdaptor.QueryParams.ID.key(), - IndividualDBAdaptor.QueryParams.VERSION.key(), IndividualDBAdaptor.QueryParams.SAMPLES.key(), - IndividualDBAdaptor.QueryParams.STUDY_UID.key()))) - .first(); - } catch (CatalogException e) { - throw new RuntimeException(e); - } - - List sampleList = new ArrayList<>(); - if (CollectionUtils.isNotEmpty(proband.getSamples())) { - if (type == ClinicalAnalysis.Type.CANCER) { - if (proband.getSamples().size() == 2) { - sampleList = proband.getSamples(); - } else if (proband.getSamples().size() > 2){ - sampleList = proband.getSamples().subList(0, 2); - logger.warn("Proband '{}' from study uid {} has more than 2 samples. Considering the first 2.", proband.getId(), - proband.getStudyUid()); - } else { - sampleList = proband.getSamples(); - logger.warn("Proband '{}' from study uid {} has {} samples. Considering those.", proband.getId(), proband.getStudyUid(), - proband.getSamples().size()); - } - } else { - if (proband.getSamples().size() == 1) { - sampleList = proband.getSamples(); - } else { - sampleList = proband.getSamples().subList(0, 1); - logger.warn("Proband '{}' from study uid {} has more than 1 sample. Considering the first sample.", proband.getId(), - proband.getStudyUid()); - } - } - } else { - logger.warn("Could not find samples for proband '{}' of study uid {}. Setting an empty list.", proband.getId(), - proband.getStudyUid()); - } - - proband.setSamples(sampleList); - ObjectMap params = new ObjectMap(ClinicalAnalysisDBAdaptor.QueryParams.PROBAND.key(), proband); - try { - dbAdaptorFactory.getClinicalAnalysisDBAdaptor().update(currentCase.getUid(), params, null, QueryOptions.empty()); - } catch (CatalogException e) { - throw new RuntimeException(e); - } - } - - private void recoverFromAudit(ClinicalAnalysis currentCase, ClinicalAnalysis clinicalAnalysis) { - if (clinicalAnalysis.getProband() == null) { - throw new RuntimeException("Proband from ClinicalAnalysis '" + currentCase.getId() - + "' could not be found in the audit collection."); - } - if (CollectionUtils.isEmpty(clinicalAnalysis.getProband().getSamples())) { - throw new RuntimeException("List of samples from proband of ClinicalAnalysis '" + currentCase.getId() - + "' could not be found in the audit collection."); - } - - // The proband from the audit matches the one used in the case currently :) - Individual proband; - try { - Query query = new Query() - .append(IndividualDBAdaptor.QueryParams.UID.key(), currentCase.getProband().getUid()) - .append(IndividualDBAdaptor.QueryParams.VERSION.key(), currentCase.getProband().getVersion()); - proband = dbAdaptorFactory.getCatalogIndividualDBAdaptor().get(query, - new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(IndividualDBAdaptor.QueryParams.ID.key(), - IndividualDBAdaptor.QueryParams.VERSION.key(), IndividualDBAdaptor.QueryParams.SAMPLES.key()))) - .first(); - } catch (CatalogException e) { - throw new RuntimeException(e); - } - List sampleList = new ArrayList<>(clinicalAnalysis.getProband().getSamples().size()); - - Map sampleMap = new HashMap<>(); - for (Sample sample : proband.getSamples()) { - sampleMap.put(sample.getId(), sample); - } - for (Sample sample : clinicalAnalysis.getProband().getSamples()) { - if (sampleMap.containsKey(sample.getId())) { - sampleList.add(sample); - } else { - logger.warn("Sample '{}' is no longer associated to the individual '{}'", sample.getId(), proband.getId()); - } - } - - if (sampleList.isEmpty()) { - logger.warn("The list of samples associated to the proband '{}' is empty after processing", proband.getId()); - } - proband.setSamples(sampleList); - - ObjectMap params = new ObjectMap(ClinicalAnalysisDBAdaptor.QueryParams.PROBAND.key(), proband); - try { - dbAdaptorFactory.getClinicalAnalysisDBAdaptor().update(currentCase.getUid(), params, null, QueryOptions.empty()); - } catch (CatalogException e) { - throw new RuntimeException(e); - } - } - - private ClinicalAnalysis findCreatedCaseInAudit(String uuid, String probandId, MongoCollection audit, - ClinicalAnalysisConverter converter) { - // Fetch case from Audit collection - try (MongoCursor iterator = audit.find( - Filters.and( - Filters.eq("resource", Enums.Resource.CLINICAL_ANALYSIS.name()), - Filters.eq("resourceUuid", uuid), - Filters.eq("action", Enums.Action.CREATE.name()) - ) - ).iterator()) { - if (iterator.hasNext()) { - Document auditDoc = iterator.next(); - Document auditParams = auditDoc.get("params", Document.class); - if (auditParams != null) { - Document caseDoc = auditParams.get("clinicalAnalysis", Document.class); - if (caseDoc != null) { - Document caseProbandOnly = new Document("proband", caseDoc.get("proband")); - converter.validateProbandToUpdate(caseProbandOnly); - ClinicalAnalysis clinicalAnalysis = converter.convertToDataModelType(caseProbandOnly); - if (!probandId.equals(clinicalAnalysis.getProband().getId())) { - logger.error("Proband '{}' from case '{}' does not match the proband '{}' used on create.", probandId, uuid, - clinicalAnalysis.getProband().getId()); - } else { - logger.debug("Found case in Audit create"); - return clinicalAnalysis; - } - } - } - } - } - - return null; - } - - private ClinicalAnalysis findUpdatedCaseInAudit(String uuid, String probandId, MongoCollection audit, - ClinicalAnalysisConverter converter) { - // Fetch case from Audit collection - try (MongoCursor iterator = audit.find( - Filters.and( - Filters.eq("resource", Enums.Resource.CLINICAL_ANALYSIS.name()), - Filters.eq("resourceUuid", uuid), - Filters.eq("action", Enums.Action.UPDATE.name()) - ) - ).sort(Sorts.descending("date")).iterator()) { - while (iterator.hasNext()) { - Document auditDoc = iterator.next(); - Document auditParams = auditDoc.get("params", Document.class); - if (auditParams != null) { - Document caseDoc = auditParams.get("updateParams", Document.class); - if (caseDoc != null) { - Document caseProbandOnly = new Document("proband", caseDoc.get("proband")); - converter.validateProbandToUpdate(caseProbandOnly); - ClinicalAnalysis clinicalAnalysis = converter.convertToDataModelType(caseProbandOnly); - if (clinicalAnalysis != null && clinicalAnalysis.getProband() != null - && probandId.equals(clinicalAnalysis.getProband().getId()) - && CollectionUtils.isNotEmpty(clinicalAnalysis.getProband().getSamples())) { - logger.debug("Found case in Audit update"); - return clinicalAnalysis; - } - } - } - } - } - - return null; - } - - -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_3/catalog/MigrateToClinicalAcmg.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_3/catalog/MigrateToClinicalAcmg.java deleted file mode 100644 index 9f4f62c4272..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_3/catalog/MigrateToClinicalAcmg.java +++ /dev/null @@ -1,125 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_4_3.catalog; - -import com.mongodb.client.MongoCollection; -import com.mongodb.client.model.Projections; -import org.apache.commons.lang3.StringUtils; -import org.bson.Document; -import org.opencb.biodata.models.clinical.ClinicalAcmg; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "migrate_to_clinical_acmg_TASK-1194", - description = "Migrate to ClinicalAcmg #TASK-1194", version = "2.4.3", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20220809) -public class MigrateToClinicalAcmg extends MigrationTool { - - private void migrateAcmg(Document document, String date, String author) { - if (document == null) { - return; - } - - List acmg; - try { - acmg = document.getList("acmg", String.class); - } catch (ClassCastException e) { - // We try to cast it to Document to validate it is already a list of ClinicalAcmg - document.getList("acmg", Document.class); - logger.warn("Skipping acmg list. It seems it was already migrated: {}", e.getMessage()); - return; - } - - if (acmg != null) { - List acmgList = new ArrayList<>(acmg.size()); - for (String acmgValue : acmg) { - ClinicalAcmg clinicalAcmg = new ClinicalAcmg(acmgValue, "", - "Values author and date automatically filled by migration script", author, date); - Document acmgDoc = convertToDocument(clinicalAcmg); - acmgList.add(acmgDoc); - } - document.put("acmg", acmgList); - } - } - - private void processFindings(Document interpretation, String field, Document update, String date, String author) { - List findings = interpretation.getList(field, Document.class); - if (findings == null) { - return; - } - - for (Document finding : findings) { - List evidences = finding.getList("evidences", Document.class); - if (evidences != null) { - for (Document evidence : evidences) { - Document classification = evidence.get("classification", Document.class); - migrateAcmg(classification, date, author); - - Document review = evidence.get("review", Document.class); - migrateAcmg(review, date, author); - } - } - } - - update.put(field, findings); - } - - private String extractAuthor(Document document) { - String author = ""; - Document analyst = document.get("analyst", Document.class); - if (analyst != null) { - String id = analyst.getString("id"); - if (StringUtils.isNotEmpty(id)) { - author = id; - } - } - return author; - } - - @Override - protected void run() throws Exception { - // Replace acmg for ClinicalAcmg - /* - findings.evidences. - -classification.acmg -review.acmg - */ - - for (String collectionString : Arrays.asList(MongoDBAdaptorFactory.INTERPRETATION_COLLECTION, - MongoDBAdaptorFactory.INTERPRETATION_ARCHIVE_COLLECTION, MongoDBAdaptorFactory.DELETED_INTERPRETATION_COLLECTION)) { - logger.info("Migrating documents from collection '{}'", collectionString); - MongoCollection collection = getMongoCollection(collectionString); - queryMongo(collectionString, - new Document(), - Projections.include("primaryFindings", "secondaryFindings", "modificationDate", "analyst", "studyUid", "id"), - (document) -> { - String date = document.getString("modificationDate"); - String author = extractAuthor(document); - - Document update = new Document(); - processFindings(document, "primaryFindings", update, date, author); - processFindings(document, "secondaryFindings", update, date, author); - - if (!update.isEmpty()) { - try { - collection.updateOne(eq("_id", document.get("_id")), - new Document("$set", update)); - } catch (Exception e) { - logger.warn("Could not replace Acmg for ClinicalAcmg to Interpretation '{}' from study uid {}. " - + "Error message: {}", document.getString("id"), document.get("studyUid", Number.class), - e.getMessage()); - } - } - }); - } - } - -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_3/catalog/ShortenIndexedFieldVariables.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_3/catalog/ShortenIndexedFieldVariables.java deleted file mode 100644 index b5fe47ebb30..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_3/catalog/ShortenIndexedFieldVariables.java +++ /dev/null @@ -1,67 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_4_3.catalog; - -import com.mongodb.client.MongoCollection; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import java.util.Arrays; -import java.util.List; - -@Migration(id = "shorten_indexed_field_variables-1386", - description = "Shorten indexed field variables #TASK-1386", version = "2.4.3", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20220809) -public class ShortenIndexedFieldVariables extends MigrationTool { - - @Override - protected void run() throws Exception { - List collections = Arrays.asList( - // Samples - MongoDBAdaptorFactory.SAMPLE_COLLECTION, MongoDBAdaptorFactory.SAMPLE_ARCHIVE_COLLECTION, - MongoDBAdaptorFactory.DELETED_SAMPLE_COLLECTION, - // Individuals - MongoDBAdaptorFactory.INDIVIDUAL_COLLECTION, MongoDBAdaptorFactory.INDIVIDUAL_ARCHIVE_COLLECTION, - MongoDBAdaptorFactory.DELETED_INDIVIDUAL_COLLECTION, - // Files - MongoDBAdaptorFactory.FILE_COLLECTION, MongoDBAdaptorFactory.DELETED_FILE_COLLECTION, - // Families - MongoDBAdaptorFactory.FAMILY_COLLECTION, MongoDBAdaptorFactory.FAMILY_ARCHIVE_COLLECTION, - MongoDBAdaptorFactory.DELETED_FAMILY_COLLECTION, - // Cohorts - MongoDBAdaptorFactory.COHORT_COLLECTION, MongoDBAdaptorFactory.DELETED_COHORT_COLLECTION); - for (String collection : collections) { - MongoCollection mongoCollection = getMongoCollection(collection); - mongoCollection.updateMany(new Document(), new Document("$rename", new Document() - .append("customAnnotationSets", "_as") - .append("customInternalAnnotationSets", "_ias") - )); - } - - List indexedCollections = Arrays.asList( - // Samples - MongoDBAdaptorFactory.SAMPLE_COLLECTION, MongoDBAdaptorFactory.SAMPLE_ARCHIVE_COLLECTION, - // Individuals - MongoDBAdaptorFactory.INDIVIDUAL_COLLECTION, MongoDBAdaptorFactory.INDIVIDUAL_ARCHIVE_COLLECTION, - // Files - MongoDBAdaptorFactory.FILE_COLLECTION, - // Families - MongoDBAdaptorFactory.FAMILY_COLLECTION, MongoDBAdaptorFactory.FAMILY_ARCHIVE_COLLECTION, - // Cohorts - MongoDBAdaptorFactory.COHORT_COLLECTION - ); - // Drop old indexes - dropIndex(indexedCollections, new Document("customAnnotationSets.as", 1).append("studyUid", 1)); - dropIndex(indexedCollections, new Document("customAnnotationSets.vs", 1).append("studyUid", 1)); - dropIndex(indexedCollections, new Document("customAnnotationSets.id", 1).append("customAnnotationSets.value", 1).append("studyUid", 1)); - - dropIndex(indexedCollections, new Document("customInternalAnnotationSets.as", 1).append("studyUid", 1)); - dropIndex(indexedCollections, new Document("customInternalAnnotationSets.vs", 1).append("studyUid", 1)); - dropIndex(indexedCollections, new Document("customInternalAnnotationSets.id", 1).append("customInternalAnnotationSets.value", 1).append("studyUid", 1)); - - // Build new indexes - catalogManager.installIndexes(token); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_4/catalog/AvoidCaseStatusIdNullMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_4/catalog/AvoidCaseStatusIdNullMigration.java deleted file mode 100644 index 8193b898cce..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_4/catalog/AvoidCaseStatusIdNullMigration.java +++ /dev/null @@ -1,38 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_4_4.catalog; - -import com.mongodb.client.MongoCollection; -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.Updates; -import com.mongodb.client.result.UpdateResult; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import java.util.Arrays; -import java.util.List; - -@Migration(id = "avoidCaseStatusIdNull-1938", - description = "Avoid nullable status id in Clinical Analysis #TASK-1938", version = "2.4.4", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20220914) -public class AvoidCaseStatusIdNullMigration extends MigrationTool { - - @Override - protected void run() throws Exception { - List> collections = Arrays.asList( - getMongoCollection(MongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION), - getMongoCollection(MongoDBAdaptorFactory.DELETED_CLINICAL_ANALYSIS_COLLECTION) - ); - - for (MongoCollection collection : collections) { - UpdateResult result = collection.updateMany( - Filters.or(Arrays.asList(Filters.exists("status.id", false), Filters.eq("status.id", null))), - Updates.set("status.id", "") - ); - logger.info("Updated {} documents from '{}' collection", result.getModifiedCount(), collection.getNamespace().getCollectionName()); - } - } - -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_4/catalog/RemoveBiotypeAllowedKeysMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_4/catalog/RemoveBiotypeAllowedKeysMigration.java deleted file mode 100644 index d99e6ec532e..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_4/catalog/RemoveBiotypeAllowedKeysMigration.java +++ /dev/null @@ -1,52 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_4_4.catalog; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.db.mongodb.converters.StudyConverter; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; -import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.models.study.Variable; -import org.opencb.opencga.core.models.study.VariableSet; - -import java.util.List; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "remove_biotype_allowed_keys_TASK-1849", - description = "Remove biotype and consequence type allowed keys from variable sets #TASK-1849", version = "2.4.4", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.CATALOG, - date = 20220905) -public class RemoveBiotypeAllowedKeysMigration extends MigrationTool { - - @Override - protected void run() throws Exception { - StudyConverter converter = new StudyConverter(); - migrateCollection(MongoDBAdaptorFactory.STUDY_COLLECTION, - new Document(), - Projections.include("_id", "variableSets"), - (doc, bulk) -> { - Study study = converter.convertToDataModelType(doc); - for (VariableSet variableSet : study.getVariableSets()) { - if ("opencga_sample_variant_stats".equals(variableSet.getId()) - || "opencga_cohort_variant_stats".equals(variableSet.getId())) { - for (Variable variable : variableSet.getVariables()) { - if ("biotypeCount".equals(variable.getId()) || "consequenceTypeCount".equals(variable.getId())) { - variable.setAllowedKeys(null); - } - } - } - } - - List variableSetDocumentList = convertToDocument(study.getVariableSets()); - bulk.add(new UpdateOneModel<>( - eq("_id", doc.get("_id")), - new Document("$set", new Document("variableSets", variableSetDocumentList))) - ); - } - ); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_4/storage/InvalidateVariantArchivesMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_4/storage/InvalidateVariantArchivesMigration.java deleted file mode 100644 index 23a62a070a1..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_4/storage/InvalidateVariantArchivesMigration.java +++ /dev/null @@ -1,34 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_4_4.storage; - -import org.opencb.opencga.app.migrations.StorageMigrationTool; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; -import org.opencb.opencga.storage.core.variant.VariantStorageEngine; - -@Migration(id = "invalidate_variant_archives_migration-633", - description = "Invalidate variant archives from variant storage hadoop #TASK-633", version = "2.4.4", - language = Migration.MigrationLanguage.JAVA, - domain = Migration.MigrationDomain.STORAGE, - date = 20220825) -public class InvalidateVariantArchivesMigration extends StorageMigrationTool { - @Override - protected void run() throws Exception { - - for (String project : getVariantStorageProjects()) { - VariantStorageEngine engine = getVariantStorageEngineByProject(project); - if (engine.getStorageEngineId().equals("hadoop")) { - VariantStorageMetadataManager mm = engine.getMetadataManager(); - if (mm.exists()) { - for (Integer studyId : mm.getStudyIds()) { - for (Integer fileId : mm.getIndexedFiles(studyId)) { - mm.updateFileMetadata(studyId, fileId, fileMetadata -> { - fileMetadata.getAttributes().put("TASK-633", "AFFECTED"); - }); - } - } - } - } - } - - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_6_0/storage/AddCellbaseDataRelease.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_6_0/storage/AddCellbaseDataRelease.java deleted file mode 100644 index db4de48aa2b..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_6_0/storage/AddCellbaseDataRelease.java +++ /dev/null @@ -1,66 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_6_0.storage; - -import org.apache.solr.common.StringUtils; -import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; -import org.opencb.opencga.app.migrations.StorageMigrationTool; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.core.config.storage.CellBaseConfiguration; -import org.opencb.opencga.core.models.project.Project; -import org.opencb.opencga.storage.core.utils.CellBaseUtils; - -@Migration(id = "add_cellbase_data_release" , - description = "Add default cellbase data release if missing", - version = "2.6.0", - domain = Migration.MigrationDomain.STORAGE, - language = Migration.MigrationLanguage.JAVA, - patch = 2, - date = 20230104 -) -public class AddCellbaseDataRelease extends StorageMigrationTool { - - @Override - protected void run() throws Exception { - VariantStorageManager variantStorageManager = getVariantStorageManager(); - for (String projectFqn : getVariantStorageProjects()) { - Project project = catalogManager.getProjectManager().get(projectFqn, QueryOptions.empty(), token).first(); - CellBaseConfiguration cellbase = project.getCellbase(); - CellBaseUtils cellBaseUtils = getVariantStorageEngineByProject(projectFqn).getCellBaseUtils(); - boolean updateCellbase = false; - if (cellbase == null) { - cellbase = new CellBaseConfiguration(cellBaseUtils.getURL(), cellBaseUtils.getVersion()); - updateCellbase = true; - } - - if (StringUtils.isEmpty(cellbase.getDataRelease())) { - if (cellBaseUtils.getDataRelease() != null) { - cellbase.setDataRelease(cellBaseUtils.getDataRelease()); - updateCellbase = true; - } else { - String serverVersion; - try { - serverVersion = cellBaseUtils.getVersionFromServer(); - } catch (Exception e) { - if (cellBaseUtils.getVersion().equals("v4")) { - logger.debug(e.getMessage(), e); - logger.warn(e.getMessage(), e); - logger.info("DataRelease not supported on version '" + cellBaseUtils.getVersion() + "' . Leaving empty"); - continue; - } - throw e; - } - if (cellBaseUtils.supportsDataRelease()) { - cellbase.setDataRelease("1"); - updateCellbase = true; - } else { - logger.info("DataRelease not supported on version '" + serverVersion + "' . Leaving empty"); - } - } - } - if (updateCellbase) { - logger.info("Update cellbase info for project '{}' with '{}'", projectFqn, cellBaseUtils); - variantStorageManager.setCellbaseConfiguration(projectFqn, cellbase, false, null, token); - } - } - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_7_0/catalog/CancerRoleAndModeOfInheritanceMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_7_0/catalog/CancerRoleAndModeOfInheritanceMigration.java deleted file mode 100644 index 11b3be47fcf..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_7_0/catalog/CancerRoleAndModeOfInheritanceMigration.java +++ /dev/null @@ -1,88 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_7_0.catalog; - - -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "panel_roles_and_modesOfInheritance" , - description = "Change panel cancer.role and modeOfInheritance for cancer.roles[] and modesOfInheritance[]", - version = "2.7.0", - domain = Migration.MigrationDomain.CATALOG, - language = Migration.MigrationLanguage.JAVA, - date = 20230117 -) -public class CancerRoleAndModeOfInheritanceMigration extends MigrationTool { - - private List replaceFields(Document doc, String key) { - List list = doc.getList(key, Document.class); - if (CollectionUtils.isNotEmpty(list)) { - for (Document common : list) { - String modeOfInheritance = common.getString("modeOfInheritance"); - common.put("modesOfInheritance", - StringUtils.isNotEmpty(modeOfInheritance) - ? Collections.singletonList(modeOfInheritance) - : Collections.emptyList() - ); - common.remove("modeOfInheritance"); - - Document cancer = common.get("cancer", Document.class); - if (cancer != null) { - String role = cancer.getString("role"); - cancer.put("roles", - StringUtils.isNotEmpty(role) - ? Collections.singletonList(role) - : Collections.emptyList()); - cancer.remove("role"); - } - } - } - return list; - } - - - @Override - protected void run() throws Exception { - migrateCollection(Arrays.asList(MongoDBAdaptorFactory.PANEL_COLLECTION, MongoDBAdaptorFactory.PANEL_ARCHIVE_COLLECTION, - MongoDBAdaptorFactory.DELETED_PANEL_COLLECTION), - Filters.or( - Filters.exists("genes.modeOfInheritance"), - Filters.exists("genes.cancer.role"), - Filters.exists("variants.modeOfInheritance"), - Filters.exists("variants.cancer.role"), - Filters.exists("strs.modeOfInheritance"), - Filters.exists("strs.cancer.role"), - Filters.exists("regions.modeOfInheritance"), - Filters.exists("regions.cancer.role") - ), - Projections.include("genes", "variants", "strs", "regions"), - (document, bulk) -> { - List genes = replaceFields(document, "genes"); - List variants = replaceFields(document, "variants"); - List strs = replaceFields(document, "strs"); - List regions = replaceFields(document, "regions"); - - bulk.add(new UpdateOneModel<>( - eq("_id", document.get("_id")), - new Document("$set", new Document() - .append("genes", genes) - .append("variants", variants) - .append("strs", strs) - .append("regions", regions) - )) - ); - }); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_8_0/catalog/CalculatePedigreeGraphMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_8_0/catalog/CalculatePedigreeGraphMigration.java deleted file mode 100644 index 6ff2cc5bff5..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_8_0/catalog/CalculatePedigreeGraphMigration.java +++ /dev/null @@ -1,104 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_8_0.catalog; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang.StringUtils; -import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.commons.datastore.core.Query; -import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.analysis.family.PedigreeGraphInitAnalysis; -import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationRun; -import org.opencb.opencga.catalog.migration.MigrationTool; -import org.opencb.opencga.catalog.utils.PedigreeGraphUtils; -import org.opencb.opencga.core.api.ParamConstants; -import org.opencb.opencga.core.models.common.Enums; -import org.opencb.opencga.core.models.family.Family; -import org.opencb.opencga.core.models.job.Job; -import org.opencb.opencga.core.models.job.JobReferenceParam; -import org.opencb.opencga.core.models.project.Project; -import org.opencb.opencga.core.models.study.Study; - -import java.util.*; - -@Migration(id = "calculate_pedigree_graph" , - description = "Calculate Pedigree Graph for all the families", - version = "2.8.0", - domain = Migration.MigrationDomain.CATALOG, - language = Migration.MigrationLanguage.JAVA, - date = 20230313 -) -public class CalculatePedigreeGraphMigration extends MigrationTool { - - @Override - protected void run() throws Exception { - MigrationRun migrationRun = getMigrationRun(); - - // Map study studyFqn -> job - Map jobs = new HashMap<>(); - for (JobReferenceParam jobReference : migrationRun.getJobs()) { - Job job = catalogManager.getJobManager().get(jobReference.getStudyId(), jobReference.getId(), new QueryOptions(), token) - .first(); - logger.info("Reading already executed job '{}' for study '{}' with status '{}'", - job.getId(), - job.getStudy().getId(), - job.getInternal().getStatus().getId()); - jobs.put(job.getStudy().getId(), job); - } - - Set studies = new LinkedHashSet<>(getStudies()); - logger.info("Study IDs (num. total = {}) to initialize pedigree graphs: {}", studies.size(), StringUtils.join(studies, ", ")); - - // Ensure that studies with already executed jobs are included in the migration run - getMigrationRun().getJobs().forEach(j -> studies.add(j.getStudyId())); - - logger.info("Study IDs (num. total = {}) after adding studies from migration jobs: {}", studies.size(), - StringUtils.join(studies, ", ")); - - for (String study : studies) { - Job job = jobs.get(study); - if (job != null) { - String status = job.getInternal().getStatus().getId(); - if (status.equals(Enums.ExecutionStatus.DONE)) { - // Skip this study. Already migrated - logger.info("Study {} already migrated", study); - continue; - } else if (status.equals(Enums.ExecutionStatus.ERROR) || status.equals(Enums.ExecutionStatus.ABORTED)) { - logger.info("Retry migration job for study {}", study); - } else { - logger.info("Job {} for migrating study {} in status {}. Wait for completion", job.getId(), study, status); - continue; - } - getMigrationRun().removeJob(job); - } - - logger.info("Adding new job to migrate/initialize pedigree graph for study {}", study); - ObjectMap params = new ObjectMap() - .append(ParamConstants.STUDY_PARAM, study); - Job newJob = catalogManager.getJobManager().submit(study, PedigreeGraphInitAnalysis.ID, Enums.Priority.MEDIUM, - params, null, null, null, new ArrayList<>(), token).first(); - getMigrationRun().addJob(newJob); - } - } - - public List getStudies() throws CatalogException { - Set studies = new LinkedHashSet<>(); - QueryOptions projectOptions = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList("id", "studies")); - QueryOptions familyOptions = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList("id", "members", "pedigreeGraph")); - for (Project project : catalogManager.getProjectManager().search(new Query(), projectOptions, token).getResults()) { - if (CollectionUtils.isNotEmpty(project.getStudies())) { - for (Study study : project.getStudies()) { - String id = study.getFqn(); - for (Family family : catalogManager.getFamilyManager().search(id, new Query(), familyOptions, token).getResults()) { - if (PedigreeGraphUtils.hasMinTwoGenerations(family) - && (family.getPedigreeGraph() == null || StringUtils.isEmpty(family.getPedigreeGraph().getBase64()))) { - studies.add(id); - break; - } - } - } - } - } - return new ArrayList<>(studies); - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_8_6/storage/MarkUnknownLargestVariantLength.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_8_6/storage/MarkUnknownLargestVariantLength.java deleted file mode 100644 index 593eb6a15a0..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_8_6/storage/MarkUnknownLargestVariantLength.java +++ /dev/null @@ -1,43 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_8_6.storage; - -import org.opencb.opencga.app.migrations.StorageMigrationTool; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.storage.core.metadata.models.SampleMetadata; -import org.opencb.opencga.storage.core.variant.VariantStorageEngine; -import org.opencb.opencga.storage.hadoop.variant.index.sample.SampleIndexSchema; - -import java.util.Map; - -@Migration(id = "mark_unknown_largest_variant_length" , - description = "Mark as unknown largest variant length", - version = "2.8.6", - domain = Migration.MigrationDomain.STORAGE, - language = Migration.MigrationLanguage.JAVA, - patch = 1, - date = 20230927 -) -public class MarkUnknownLargestVariantLength extends StorageMigrationTool { - - @Override - protected void run() throws Exception { - for (String project : getVariantStorageProjects()) { - try (VariantStorageEngine engine = getVariantStorageEngineByProject(project)) { - for (Map.Entry entry : engine.getMetadataManager().getStudies().entrySet()) { - String studyName = entry.getKey(); - Integer studyId = entry.getValue(); - logger.info("Process study '" + studyName + "' (" + studyId + ")"); - // Check for indexed samples with undefined largest variant length - for (SampleMetadata sampleMetadata : engine.getMetadataManager().sampleMetadataIterable(studyId)) { - if (sampleMetadata.isIndexed()) { - if (!sampleMetadata.getAttributes().containsKey(SampleIndexSchema.LARGEST_VARIANT_LENGTH)) { - logger.info("Mark unknown largest variant length for sample '" + sampleMetadata.getName() + "'"); - engine.getMetadataManager().updateSampleMetadata(studyId, sampleMetadata.getId(), - sm -> sm.getAttributes().put(SampleIndexSchema.UNKNOWN_LARGEST_VARIANT_LENGTH, true)); - } - } - } - } - } - } - } -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_9_0/catalog/RenameCellBaseToken2ApiKey.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_9_0/catalog/RenameCellBaseToken2ApiKey.java deleted file mode 100644 index 72fe8e6499e..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_9_0/catalog/RenameCellBaseToken2ApiKey.java +++ /dev/null @@ -1,50 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_9_0.catalog; - -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import com.mongodb.client.model.Updates; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; - -import java.util.List; - -import static com.mongodb.client.model.Filters.eq; - -@Migration(id = "rename_cellbase_token_2_api_key" , - description = "Rename CellBase Token to ApiKey", - version = "2.9.0", - domain = Migration.MigrationDomain.CATALOG, - language = Migration.MigrationLanguage.JAVA, - date = 20230829 -) -public class RenameCellBaseToken2ApiKey extends MigrationTool { - - @Override - protected void run() throws Exception { - - migrateCollection(MongoDBAdaptorFactory.USER_COLLECTION, - new Document("projects.id", new Document("$exists", true)), - Projections.include("_id", "projects"), - (userDocument, bulk) -> { - List projects = userDocument.getList("projects", Document.class); - for (int i = 0; i < projects.size(); i++) { - Document project = projects.get(i); - Document cellbase = project.get("cellbase", Document.class); - if (cellbase != null) { - String token = cellbase.getString("token"); - if (token != null) { - bulk.add(new UpdateOneModel<>( - eq("_id", userDocument.get("_id")), - Updates.combine( - Updates.set("projects." + i + ".cellbase.apiKey", token), - Updates.unset("projects." + i + ".cellbase.token") - ))); - } - } - } - }); - } - -} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_0_0/OrganizationMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_0_0/OrganizationMigration.java new file mode 100644 index 00000000000..9b226c0ea1a --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_0_0/OrganizationMigration.java @@ -0,0 +1,19 @@ +package org.opencb.opencga.app.migrations.v3.v3_0_0; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; +import org.opencb.opencga.core.config.Configuration; + +@Migration(id = "add_organizations", description = "Add new Organization layer #TASK-4389", version = "3.0.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20231212, + deprecatedSince = "3.1.0") +public class OrganizationMigration extends MigrationTool { + + public OrganizationMigration(Configuration configuration, String adminPassword, String userId) { + } + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_1_0/AuthOriginSimplificationMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_1_0/AuthOriginSimplificationMigration.java new file mode 100644 index 00000000000..e1b331fa987 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_1_0/AuthOriginSimplificationMigration.java @@ -0,0 +1,64 @@ +package org.opencb.opencga.app.migrations.v3.v3_1_0; + +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; +import com.mongodb.client.model.UpdateOneModel; +import com.mongodb.client.model.Updates; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.opencb.opencga.catalog.db.api.OrganizationDBAdaptor; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; +import org.opencb.opencga.core.common.PasswordUtils; + +import java.util.List; + +@Migration(id = "hide_secret_key", description = "Hide secret key #TASK-5923", version = "3.1.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240410) +public class AuthOriginSimplificationMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + Bson query = Filters.exists("configuration.token", false); + Bson projection = Projections.include("_id", "configuration"); + migrateCollection(OrganizationMongoDBAdaptorFactory.ORGANIZATION_COLLECTION, query, projection, (orgDocument, bulk) -> { + Document configuration = orgDocument.get(OrganizationDBAdaptor.QueryParams.CONFIGURATION.key(), Document.class); + if (configuration != null) { + String secretKey = null; + List authenticationOrigins = configuration.getList("authenticationOrigins", Document.class); + if (CollectionUtils.isNotEmpty(authenticationOrigins)) { + for (Document authenticationOrigin : authenticationOrigins) { + if (authenticationOrigin.getString("type").equals("OPENCGA")) { + secretKey = authenticationOrigin.getString("secretKey"); + if (authenticationOrigin.getString("id").equals("internal")) { + authenticationOrigin.put("id", "OPENCGA"); + } + } + authenticationOrigin.remove("secretKey"); + authenticationOrigin.remove("algorithm"); + authenticationOrigin.remove("expiration"); + } + } + secretKey = StringUtils.isNotEmpty(secretKey) ? secretKey : PasswordUtils.getStrongRandomPassword(32); + Document token = new Document() + .append("secretKey", secretKey) + .append("algorithm", "HS256") + .append("expiration", 3600L); + configuration.put("token", token); + bulk.add(new UpdateOneModel<>( + Filters.eq("_id", orgDocument.get("_id")), + Updates.set("configuration", configuration))); + } + }); + + // Change authOrigin id from all "internal" users + getMongoCollection(OrganizationMongoDBAdaptorFactory.USER_COLLECTION) + .updateMany( + new Document("account.authentication.id", "internal"), + Updates.set("account.authentication.id", "OPENCGA")); + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_1_0/NoteMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_1_0/NoteMigration.java new file mode 100644 index 00000000000..c3e49a2d85e --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_1_0/NoteMigration.java @@ -0,0 +1,51 @@ +package org.opencb.opencga.app.migrations.v3.v3_1_0; + +import com.mongodb.MongoNamespace; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.RenameCollectionOptions; +import com.mongodb.client.model.Updates; +import com.mongodb.client.result.UpdateResult; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.opencb.commons.datastore.mongodb.MongoDataStore; +import org.opencb.opencga.catalog.db.api.NoteDBAdaptor; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; +import org.opencb.opencga.core.models.notes.Note; + +@Migration(id = "migrate_notes", description = "Migrate notes #TASK-5836", version = "3.1.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240315) +public class NoteMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + // First migrate to add the new values + MongoCollection collection = getMongoCollection(organizationId, "notes"); + Bson query = Filters.exists(NoteDBAdaptor.QueryParams.STUDY_UID.key(), false); + Bson update = Updates.combine( + Updates.set(NoteDBAdaptor.QueryParams.STUDY_UID.key(), -1L), + Updates.set(NoteDBAdaptor.QueryParams.SCOPE.key(), Note.Scope.ORGANIZATION.name()), + Updates.set(NoteDBAdaptor.QueryParams.STUDY.key(), ""), + Updates.set(NoteDBAdaptor.QueryParams.VISIBILITY.key(), Note.Visibility.PRIVATE.name()) + ); + UpdateResult updateResult = collection.updateMany(query, update); + if (updateResult.getModifiedCount() == 0) { + logger.info("Note data model could not be updated. Notes found in organization '{}': {}", organizationId, updateResult.getMatchedCount()); + } + + // Rename Note collection + RenameCollectionOptions options = new RenameCollectionOptions().dropTarget(true); + // Rename collection + String databaseName = dbAdaptorFactory.getMongoDataStore(organizationId).getDatabaseName(); + logger.info("Renaming notes collection for organization '{}' -> Database: '{}'", organizationId, databaseName); + MongoDataStore mongoDataStore = dbAdaptorFactory.getMongoDataStore(organizationId); + mongoDataStore.getDb().getCollection("notes").renameCollection(new MongoNamespace(databaseName, OrganizationMongoDBAdaptorFactory.NOTE_COLLECTION), options); + mongoDataStore.getDb().getCollection("notes_archive").renameCollection(new MongoNamespace(databaseName, OrganizationMongoDBAdaptorFactory.NOTE_ARCHIVE_COLLECTION), options); + mongoDataStore.getDb().getCollection("notes_deleted").renameCollection(new MongoNamespace(databaseName, OrganizationMongoDBAdaptorFactory.DELETED_NOTE_COLLECTION), options); + + catalogManager.installIndexes(organizationId, token); + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_1_0/UserBanMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_1_0/UserBanMigration.java new file mode 100644 index 00000000000..b7d79b73914 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_1_0/UserBanMigration.java @@ -0,0 +1,41 @@ +package org.opencb.opencga.app.migrations.v3.v3_1_0; + +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; +import com.mongodb.client.model.UpdateOneModel; +import com.mongodb.client.model.Updates; +import org.bson.Document; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; +import org.opencb.opencga.catalog.utils.Constants; +import org.opencb.opencga.core.common.TimeUtils; + +@Migration(id = "addFailedLoginAttemptsMigration", description = "Add failedAttempts to User #TASK-6013", version = "3.2.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240419) +public class UserBanMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + String lastModified = TimeUtils.getTime(); + migrateCollection(OrganizationMongoDBAdaptorFactory.USER_COLLECTION, + Filters.exists("internal.registrationDate", false), + Projections.include("_id", "internal", "account"), + (document, bulk) -> { + Document internal = document.get("internal", Document.class); + Document account = document.get("account", Document.class); + internal.put("failedAttempts", 0); + internal.put("registrationDate", account.get("creationDate")); + internal.put("lastModified", lastModified); + account.put("expirationDate", Constants.DEFAULT_USER_EXPIRATION_DATE); + + bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), + Updates.combine( + Updates.set("internal", internal), + Updates.set("account", account)) + ) + ); + }); + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/AddNewJobFieldsMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/AddNewJobFieldsMigration.java new file mode 100644 index 00000000000..8e196973bba --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/AddNewJobFieldsMigration.java @@ -0,0 +1,34 @@ +package org.opencb.opencga.app.migrations.v3.v3_2_0; + +import com.mongodb.client.MongoCollection; +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Updates; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +import java.util.Arrays; + +@Migration(id = "add_jobParentId_scheduledStartTime", + description = "Add 'jobParentId' and 'scheduledStartTime' to existing jobs #TASK-6171 #TASK-6089", version = "3.2.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240506) +public class AddNewJobFieldsMigration extends MigrationTool { + + + @Override + protected void run() throws Exception { + for (String jobCollection : Arrays.asList(OrganizationMongoDBAdaptorFactory.JOB_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_JOB_COLLECTION)) { + MongoCollection mongoCollection = getMongoCollection(jobCollection); + Bson query = Filters.exists("parentId", false); + Bson update = Updates.combine( + Updates.set("parentId", ""), + Updates.set("scheduledStartTime", "") + ); + mongoCollection.updateMany(query, update); + } + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddInterpretationName.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddInterpretationName.java new file mode 100644 index 00000000000..372130c0a5e --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddInterpretationName.java @@ -0,0 +1,35 @@ +package org.opencb.opencga.app.migrations.v3.v3_2_0.TASK_5964; + +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; +import com.mongodb.client.model.UpdateOneModel; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +import java.util.Arrays; + +@Migration(id = "add_interpretation_name", description = "Add Interpretation name #TASK-5964", version = "3.2.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240610) +public class AddInterpretationName extends MigrationTool { + + @Override + protected void run() throws Exception { + // Add new Interpretation name field + Bson nameDoesNotExistQuery = Filters.exists("name", false); + Bson projection = Projections.include("id"); + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.INTERPRETATION_COLLECTION, + OrganizationMongoDBAdaptorFactory.INTERPRETATION_ARCHIVE_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_INTERPRETATION_COLLECTION)) { + migrateCollection(collection, nameDoesNotExistQuery, projection, + (document, bulk) -> { + Document updateDocument = new Document() + .append("name", document.get("id")); + bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), new Document("$set", updateDocument))); + }); + } + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddNewClinicalStatusValues.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddNewClinicalStatusValues.java new file mode 100644 index 00000000000..775cf2c64d7 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddNewClinicalStatusValues.java @@ -0,0 +1,112 @@ +package org.opencb.opencga.app.migrations.v3.v3_2_0.TASK_5964; + +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; +import com.mongodb.client.model.UpdateOneModel; +import org.apache.commons.lang3.StringUtils; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptor; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; +import org.opencb.opencga.core.common.GitRepositoryState; +import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.models.clinical.ClinicalStatus; +import org.opencb.opencga.core.models.clinical.ClinicalStatusValue; +import org.opencb.opencga.core.models.study.configuration.ClinicalAnalysisStudyConfiguration; + +import java.util.Arrays; +import java.util.List; + +@Migration(id = "add_new_clinical_status", + description = "Add new ClinicalStatus to ClinicalAnalysis and Interpretation, #TASK-5964", + version = "3.2.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240611) +public class AddNewClinicalStatusValues extends MigrationTool { + + @Override + protected void run() throws Exception { + ClinicalAnalysisStudyConfiguration defaultConfiguration = ClinicalAnalysisStudyConfiguration.defaultConfiguration(); + + Bson query = Filters.exists("status.author", false); + Bson projection = Projections.include("status", "id"); + + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, + OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_ARCHIVE_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_CLINICAL_ANALYSIS_COLLECTION)) { + migrateCollection(collection, query, projection, + (document, bulk) -> { + MongoDBAdaptor.UpdateDocument updateDocument = new MongoDBAdaptor.UpdateDocument(); + String clinicalId = document.getString("id"); + Document status = document.get("status", Document.class); + + Document newStatus = fillStatusValues("Clinical Analysis", clinicalId, status, defaultConfiguration.getStatus()); + updateDocument.getSet().put("status", newStatus); + + Document effectiveUpdate = updateDocument.toFinalUpdateDocument(); + + logger.debug("Updating clinical analysis '{}': {}", document.get("id"), effectiveUpdate.toBsonDocument()); + bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), effectiveUpdate)); + }); + } + + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.INTERPRETATION_COLLECTION, + OrganizationMongoDBAdaptorFactory.INTERPRETATION_ARCHIVE_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_INTERPRETATION_COLLECTION)) { + migrateCollection(collection, query, projection, + (document, bulk) -> { + MongoDBAdaptor.UpdateDocument updateDocument = new MongoDBAdaptor.UpdateDocument(); + String interpretationId = document.getString("id"); + Document status = document.get("status", Document.class); + + Document newStatus = fillStatusValues("Interpretation", interpretationId, status, + defaultConfiguration.getInterpretation().getStatus()); + updateDocument.getSet().put("status", newStatus); + + Document effectiveUpdate = updateDocument.toFinalUpdateDocument(); + + logger.debug("Updating interpretation '{}': {}", document.get("id"), effectiveUpdate.toBsonDocument()); + bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), effectiveUpdate)); + }); + } + } + + private Document fillStatusValues(String entity, String id, Document status, List statusValueList) { + ClinicalStatus clinicalStatus = new ClinicalStatus(); + String clinicalStatusId = status != null ? status.getString("id") : null; + if (status == null || StringUtils.isEmpty(clinicalStatusId)) { + logger.warn("Status is empty or does not contain 'id' field. Setting default status value for {} '{}'", entity, id); + for (ClinicalStatusValue clinicalStatusValue : statusValueList) { + if (clinicalStatusValue.getType().equals(ClinicalStatusValue.ClinicalStatusType.NOT_STARTED)) { + clinicalStatus.setId(clinicalStatusValue.getId()); + clinicalStatus.setDescription(clinicalStatusValue.getDescription()); + clinicalStatus.setType(clinicalStatusValue.getType()); + } + } + } else { + for (ClinicalStatusValue clinicalStatusValue : statusValueList) { + if (clinicalStatusValue.getId().equals(clinicalStatusId)) { + clinicalStatus.setId(clinicalStatusValue.getId()); + clinicalStatus.setDescription(clinicalStatusValue.getDescription()); + clinicalStatus.setType(clinicalStatusValue.getType()); + break; + } + } + if (clinicalStatus.getType() == null) { + logger.warn("Status '{}' not found in the list of available status values. Keeping original status for {} '{}'", + clinicalStatusId, entity, id); + clinicalStatus.setId(clinicalStatusId); + clinicalStatus.setDescription(status.getString("description")); + } + } + clinicalStatus.setDate(TimeUtils.getTime()); + clinicalStatus.setVersion(GitRepositoryState.getInstance().getBuildVersion()); + clinicalStatus.setCommit(GitRepositoryState.getInstance().getCommitId()); + clinicalStatus.setAuthor("opencga"); + + Document document = convertToDocument(clinicalStatus); + return document; + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddVersionToClinicalAnalysisMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddVersionToClinicalAnalysisMigration.java new file mode 100644 index 00000000000..302637cf968 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddVersionToClinicalAnalysisMigration.java @@ -0,0 +1,62 @@ +package org.opencb.opencga.app.migrations.v3.v3_2_0.TASK_5964; + +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; +import com.mongodb.client.model.UpdateOneModel; +import org.apache.commons.collections4.CollectionUtils; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +import java.util.Collections; +import java.util.List; + +@Migration(id = "add_version_to_clinicalAnalysis", description = "Add version to Clinical Analysis #TASK-5964", version = "3.2.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240527) +public class AddVersionToClinicalAnalysisMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + // Add missing mandatory "versioning" fields to Clinical Analysis + logger.info("Adding missing 'versioning' fields to Clinical Analysis..."); + Bson versionDoesNotExistQuery = Filters.exists("version", false); + Bson projection = Projections.include("release", "interpretation", "secondaryInterpretations"); + migrateCollection(OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, versionDoesNotExistQuery, projection, + (document, bulk) -> { + int version = 1; + Document interpretation = document.get("interpretation", Document.class); + if (interpretation != null) { + int interpretationVersion = interpretation.get("version", Number.class).intValue(); + version = Math.max(version, interpretationVersion + 1); + } + List secondaryInterpretations = document.getList("secondaryInterpretations", Document.class); + if (CollectionUtils.isNotEmpty(secondaryInterpretations)) { + for (Document secondaryInterpretation : secondaryInterpretations) { + int secondaryInterpretationVersion = secondaryInterpretation.get("version", Number.class).intValue(); + version = Math.max(version, secondaryInterpretationVersion + 1); + } + } + int release = document.get("release", Number.class).intValue(); + Document updateDocument = new Document() + .append("version", version) + .append("_releaseFromVersion", Collections.singletonList(release)) + .append("_lastOfVersion", true) + .append("_lastOfRelease", true); + bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), new Document("$set", updateDocument))); + }); + + // Recalculate indexes + logger.info("Installing new indexes..."); + catalogManager.installIndexes(organizationId, token); + + // Copy all Clinical Analyses to the archive collection + logger.info("Copying all the data from the main collection to the archive one..."); + copyData(new Document(), OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, + OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_ARCHIVE_COLLECTION); + + + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/RemoveStatusNameMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/RemoveStatusNameMigration.java new file mode 100644 index 00000000000..80692ae910c --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/RemoveStatusNameMigration.java @@ -0,0 +1,114 @@ +package org.opencb.opencga.app.migrations.v3.v3_2_0.TASK_5964; + +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Updates; +import org.bson.conversions.Bson; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +@Migration(id = "remove_status_name", description = "Remove status name #TASK-5964", version = "3.2.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240612) +public class RemoveStatusNameMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + // Remove from user + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.USER_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_USER_COLLECTION)) { + removeStatusField(collection, Collections.singletonList("internal.status.name")); + } + + // Remove from project + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.PROJECT_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_PROJECT_COLLECTION)) { + removeStatusField(collection, Collections.singletonList("internal.status.name")); + } + + // Remove from study + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.STUDY_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_STUDY_COLLECTION)) { + List statusList = Arrays.asList("status.name", "internal.status.name"); + removeStatusField(collection, statusList); + } + + // Remove from sample + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.SAMPLE_COLLECTION, + OrganizationMongoDBAdaptorFactory.SAMPLE_ARCHIVE_COLLECTION, OrganizationMongoDBAdaptorFactory.DELETED_SAMPLE_COLLECTION)) { + List statusList = Arrays.asList("status.name", "internal.status.name", "internal.variant.index.status.name", + "internal.variant.sampleGenotypeIndex.status.name", "internal.variant.sampleGenotypeIndex.familyStatus.name", + "internal.variant.secondarySampleIndex.status.name", "internal.variant.secondarySampleIndex.familyStatus.name", + "internal.variant.annotationIndex.status.name", "internal.variant.secondaryAnnotationIndex.status.name"); + removeStatusField(collection, statusList); + } + + // Remove from file + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.FILE_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_FILE_COLLECTION)) { + List statusList = Arrays.asList("status.name", "internal.status.name", "internal.variant.index.status.name", + "internal.variant.annotationIndex.status.name", "internal.variant.secondaryAnnotationIndex.status.name", + "internal.alignment.index.status.name"); + removeStatusField(collection, statusList); + } + + // Remove from individual + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.INDIVIDUAL_COLLECTION, + OrganizationMongoDBAdaptorFactory.INDIVIDUAL_ARCHIVE_COLLECTION, OrganizationMongoDBAdaptorFactory.DELETED_INDIVIDUAL_COLLECTION)) { + List statusList = Arrays.asList("status.name", "internal.status.name"); + removeStatusField(collection, statusList); + } + + // Remove from family + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.FAMILY_COLLECTION, + OrganizationMongoDBAdaptorFactory.FAMILY_ARCHIVE_COLLECTION, OrganizationMongoDBAdaptorFactory.DELETED_FAMILY_COLLECTION)) { + List statusList = Arrays.asList("status.name", "internal.status.name"); + removeStatusField(collection, statusList); + } + + // Remove from cohort + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.COHORT_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_COHORT_COLLECTION)) { + List statusList = Arrays.asList("status.name", "internal.status.name"); + removeStatusField(collection, statusList); + } + + // Remove from panel + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.PANEL_COLLECTION, + OrganizationMongoDBAdaptorFactory.PANEL_ARCHIVE_COLLECTION, OrganizationMongoDBAdaptorFactory.DELETED_PANEL_COLLECTION)) { + List statusList = Arrays.asList("status.name", "internal.status.name"); + removeStatusField(collection, statusList); + } + + // Remove from job + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.JOB_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_JOB_COLLECTION)) { + removeStatusField(collection, Collections.singletonList("internal.status.name")); + } + + // Remove from clinical + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, + OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_ARCHIVE_COLLECTION, OrganizationMongoDBAdaptorFactory.DELETED_CLINICAL_ANALYSIS_COLLECTION)) { + removeStatusField(collection, Collections.singletonList("internal.status.name")); + } + + // Remove from interpretation + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.INTERPRETATION_COLLECTION, + OrganizationMongoDBAdaptorFactory.INTERPRETATION_ARCHIVE_COLLECTION, OrganizationMongoDBAdaptorFactory.DELETED_INTERPRETATION_COLLECTION)) { + removeStatusField(collection, Collections.singletonList("internal.status.name")); + } + } + + private void removeStatusField(String collection, List statusList) throws CatalogDBException { + logger.info("Removing status.name fields from collection {}...", collection); + List exists = statusList.stream().map(Filters::exists).collect(Collectors.toList()); + List unsetList = statusList.stream().map(Updates::unset).collect(Collectors.toList()); + getMongoCollection(collection).updateMany(Filters.or(exists), Updates.combine(unsetList)); + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/RenamePanelLockMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/RenamePanelLockMigration.java new file mode 100644 index 00000000000..32c10c3fab7 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/RenamePanelLockMigration.java @@ -0,0 +1,26 @@ +package org.opencb.opencga.app.migrations.v3.v3_2_0.TASK_5964; + +import org.bson.Document; +import org.bson.conversions.Bson; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +import java.util.Arrays; + +@Migration(id = "rename_panel_lock_from_clinical__task_5964", description = "Rename 'panelLock' to 'panelLocked' #TASK-5964", + version = "3.2.0", language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240710) +public class RenamePanelLockMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + Bson query = new Document(); + + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, + OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_ARCHIVE_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_CLINICAL_ANALYSIS_COLLECTION)) { + getMongoCollection(collection).updateMany(query, new Document("$rename", new Document("panelLock", "panelLocked"))); + } + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/UpdateClinicalStudyConfiguration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/UpdateClinicalStudyConfiguration.java new file mode 100644 index 00000000000..35960978d4c --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/UpdateClinicalStudyConfiguration.java @@ -0,0 +1,68 @@ +package org.opencb.opencga.app.migrations.v3.v3_2_0.TASK_5964; + +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; +import com.mongodb.client.model.UpdateOneModel; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptor; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationRuntimeException; +import org.opencb.opencga.catalog.migration.MigrationTool; +import org.opencb.opencga.core.models.study.configuration.ClinicalAnalysisStudyConfiguration; + +import java.util.Arrays; +import java.util.List; + +@Migration(id = "update_clinical_study_configuration", + description = "Setting new default Clinical Study Configuration status values, #TASK-5964", + version = "3.2.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240610) +public class UpdateClinicalStudyConfiguration extends MigrationTool { + + @Override + protected void run() throws Exception { + ClinicalAnalysisStudyConfiguration clinicalAnalysisStudyConfiguration = ClinicalAnalysisStudyConfiguration.defaultConfiguration(); + Document clinicalConfigurationDocument = convertToDocument(clinicalAnalysisStudyConfiguration); + + Bson query = new Document(); + Bson projection = Projections.include("internal.configuration.clinical", "fqn"); + + for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.STUDY_COLLECTION, OrganizationMongoDBAdaptorFactory.DELETED_STUDY_COLLECTION)) { + migrateCollection(collection, query, projection, + (document, bulk) -> { + Document internal = document.get("internal", Document.class); + if (internal == null) { + throw new MigrationRuntimeException("'internal' field not found in study '" + document.get("fqn") + "'"); + } + Document configuration = internal.get("configuration", Document.class); + if (configuration == null) { + throw new MigrationRuntimeException("'internal.configuration' field not found in study '" + document.get("fqn") + "'"); + } + Document clinicalDocument = configuration.get("clinical", Document.class); + + MongoDBAdaptor.UpdateDocument updateDocument = new MongoDBAdaptor.UpdateDocument(); + if (clinicalDocument == null) { + logger.warn("Found empty 'internal.configuration.clinical' field in study '{}'. Creating a new one...", document.get("fqn")); + updateDocument.getSet().put("internal.configuration.clinical", clinicalConfigurationDocument); + } else { + Object statusObject = clinicalDocument.get("status"); + if (statusObject instanceof List) { + // The study seems to be already migrated. Skipping... + logger.warn("Study '{}' seems to be already migrated. Skipping...", document.get("fqn")); + return; + } + // Study needs to be migrated + logger.info("Migrating study '{}'", document.get("fqn")); + updateDocument.getSet().put("internal.configuration.clinical.status", clinicalConfigurationDocument.get("status")); + updateDocument.getSet().put("internal.configuration.clinical.flags", clinicalConfigurationDocument.get("flags")); + updateDocument.getSet().put("internal.configuration.clinical.interpretation.status", clinicalConfigurationDocument.get("interpretation", Document.class).get("status")); + } + logger.debug("Updating study '{}': {}", document.get("fqn"), updateDocument.toFinalUpdateDocument()); + bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), updateDocument.toFinalUpdateDocument())); + }); + } + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/AddPasswordHistoryMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/AddPasswordHistoryMigration.java new file mode 100644 index 00000000000..f016e564c77 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/AddPasswordHistoryMigration.java @@ -0,0 +1,45 @@ +package org.opencb.opencga.app.migrations.v3.v3_2_1; + +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; +import com.mongodb.client.model.UpdateOneModel; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptor; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +import java.util.Arrays; +import java.util.Collections; + +import static org.opencb.opencga.catalog.db.mongodb.UserMongoDBAdaptor.*; + +@Migration(id = "add_archivePasswords_array", + description = "Add password history #6494", version = "3.2.1", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240723) +public class AddPasswordHistoryMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + Bson query = Filters.exists(PRIVATE_PASSWORD_ARCHIVE, false); + Bson projection = Projections.include(PRIVATE_PASSWORD); + migrateCollection(Arrays.asList(OrganizationMongoDBAdaptorFactory.USER_COLLECTION, OrganizationMongoDBAdaptorFactory.DELETED_USER_COLLECTION), + query, projection, (document, bulk) -> { + String currentPassword = document.getString("_password"); + + Document passwordDoc = new Document() + .append(HASH, currentPassword) + .append(SALT, ""); + Document privatePassword = new Document(); + privatePassword.put(CURRENT, passwordDoc); + privatePassword.put(ARCHIVE, Collections.singletonList(passwordDoc)); + + MongoDBAdaptor.UpdateDocument updateDocument = new MongoDBAdaptor.UpdateDocument(); + updateDocument.getSet().put(PRIVATE_PASSWORD, privatePassword); + + bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), updateDocument.toFinalUpdateDocument())); + }); + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/MoveUserAccountToInternalMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/MoveUserAccountToInternalMigration.java new file mode 100644 index 00000000000..68f2cc00e7c --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/MoveUserAccountToInternalMigration.java @@ -0,0 +1,50 @@ +package org.opencb.opencga.app.migrations.v3.v3_2_1; + +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; +import com.mongodb.client.model.UpdateOneModel; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptor; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +import java.util.Arrays; + +@Migration(id = "move_user_account_to_internal", + description = "Move account to internal.account #6494", version = "3.2.1", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240723) +public class MoveUserAccountToInternalMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + Bson query = Filters.exists("account", true); + Bson projection = Projections.include("internal", "account"); + migrateCollection(Arrays.asList(OrganizationMongoDBAdaptorFactory.USER_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_USER_COLLECTION), + query, projection, (document, bulk) -> { + MongoDBAdaptor.UpdateDocument updateDocument = new MongoDBAdaptor.UpdateDocument(); + + Document account = document.get("account", Document.class); + Document internal = document.get("internal", Document.class); + internal.put("account", account); + + updateDocument.getSet().put("modificationDate", internal.get("lastModified")); + updateDocument.getSet().put("creationDate", account.get("creationDate")); + account.remove("creationDate"); + + Document password = new Document() + .append("expirationDate", null) + .append("lastModified", internal.get("lastModified")); + account.put("password", password); + account.put("failedAttempts", internal.get("failedAttempts")); + internal.remove("failedAttempts"); + + updateDocument.getSet().put("internal", internal); + updateDocument.getUnset().add("account"); + + bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), updateDocument.toFinalUpdateDocument())); + }); + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/VariantSetupMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/VariantSetupMigration.java new file mode 100644 index 00000000000..9d02073d099 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_2_0/VariantSetupMigration.java @@ -0,0 +1,47 @@ +package org.opencb.opencga.app.migrations.v3_2_0; + +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; +import org.opencb.opencga.app.migrations.StorageMigrationTool; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.models.study.VariantSetupResult; +import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; + +import java.util.LinkedHashSet; + +@Migration(id = "variant_setup", description = "Add a dummy variant setup for studies with data", version = "3.2.0", + domain = Migration.MigrationDomain.STORAGE, date = 20240516) +public class VariantSetupMigration extends StorageMigrationTool { + + @Override + protected void run() throws Exception { + VariantStorageManager variantStorageManager = getVariantStorageManager(); + for (String study : getVariantStorageStudies()) { + logger.info("--- Checking study '{}'", study); + if (variantStorageManager.hasVariantSetup(study, token)) { + logger.info("Study '{}' already has a variant setup", study); + continue; + } + + String projectFqn = catalogManager.getStudyManager().getProjectFqn(study); + VariantStorageMetadataManager metadataManager = getVariantStorageEngineByProject(projectFqn).getMetadataManager(); + int studyId = metadataManager.getStudyId(study); + LinkedHashSet indexedFiles = metadataManager.getIndexedFiles(studyId); + if (indexedFiles.isEmpty()) { + logger.info("Study '{}' does not have any indexed files. Skipping variant setup", study); + continue; + } + logger.info("Study '{}' doesn't have a variant setup, but it has {} indexed files. Creating a dummy variant setup", + study, indexedFiles.size()); + logger.info("Creating a dummy variant setup for study '{}'", study); + VariantSetupResult dummy = new VariantSetupResult(); + dummy.setDate(TimeUtils.getTime()); + dummy.setUserId(catalogManager.getUserManager().getUserId(token)); + dummy.setParams(new ObjectMap("executed_from_migration", getId())); + dummy.setStatus(VariantSetupResult.Status.READY); + dummy.setOptions(new ObjectMap()); + catalogManager.getStudyManager().setVariantEngineSetupOptions(study, dummy, token); + } + } +} diff --git a/opencga-app/src/main/resources/cli-usage.yml b/opencga-app/src/main/resources/cli-usage.yml index a3451aea6a7..6a712b72a63 100644 --- a/opencga-app/src/main/resources/cli-usage.yml +++ b/opencga-app/src/main/resources/cli-usage.yml @@ -2,6 +2,7 @@ categories: - name: catalog description: "Catalog commands:" options: + - organizations - users - projects - studies diff --git a/opencga-app/src/test/java/org/opencb/opencga/app/cli/analysis/InternalMainTest.java b/opencga-app/src/test/java/org/opencb/opencga/app/cli/analysis/InternalMainTest.java index e8548e128b2..c963f46e106 100644 --- a/opencga-app/src/test/java/org/opencb/opencga/app/cli/analysis/InternalMainTest.java +++ b/opencga-app/src/test/java/org/opencb/opencga/app/cli/analysis/InternalMainTest.java @@ -46,8 +46,6 @@ import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.sample.SampleReferenceParam; import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.models.user.Account; -import org.opencb.opencga.core.models.user.User; import org.opencb.opencga.core.testclassification.duration.ShortTests; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; @@ -83,6 +81,7 @@ public class InternalMainTest { private CatalogManager catalogManager; + private final String organizationId = "test"; private final String userId = "user"; private final String dbNameVariants = "opencga_variants_test"; private final String dbNameAlignments = "opencga_alignments_test"; @@ -102,9 +101,9 @@ public void setUp() throws Exception { opencga.clearStorageDB(STORAGE_ENGINE, dbNameVariants); opencga.clearStorageDB(STORAGE_ENGINE, dbNameAlignments); - User user = catalogManager.getUserManager().create(userId, "User", "user@email.org", "user", "ACME", null, Account.AccountType.FULL, opencga.getAdminToken()).first(); + catalogManager.getUserManager().create(userId, "User", "user@email.org", "user", organizationId, null, opencga.getAdminToken()); - sessionId = catalogManager.getUserManager().login(userId, "user").getToken(); + sessionId = catalogManager.getUserManager().login(organizationId, userId, "user").getToken(); projectId = catalogManager.getProjectManager().create("p1", "p1", "Project 1", "Homo sapiens", null, "GRCh38", new QueryOptions(), sessionId).first().getId(); diff --git a/opencga-app/src/test/java/org/opencb/opencga/app/migrations/MigrationsTest.java b/opencga-app/src/test/java/org/opencb/opencga/app/migrations/MigrationsTest.java new file mode 100644 index 00000000000..6d72b9149f5 --- /dev/null +++ b/opencga-app/src/test/java/org/opencb/opencga/app/migrations/MigrationsTest.java @@ -0,0 +1,88 @@ +package org.opencb.opencga.app.migrations; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.analysis.variant.OpenCGATestExternalResource; +import org.opencb.opencga.app.migrations.v3_2_0.VariantSetupMigration; +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; +import org.opencb.opencga.core.models.study.Study; +import org.opencb.opencga.core.testclassification.duration.LongTests; +import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; +import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; + +import java.net.URL; +import java.util.Arrays; +import java.util.Collections; + +@Category(LongTests.class) +public class MigrationsTest { + + public OpenCGATestExternalResource opencga; + + @Test + public void testVariantSetupMigration() throws Exception { + setup("v3.0.0", false); + String studyName = "test@1000G:phase1"; + + VariantStorageMetadataManager metadataManager = opencga.getVariantStorageEngineByProject("test@1000G").getMetadataManager(); + StudyMetadata studyMetadata = metadataManager.createStudy(studyName); + int fileId = metadataManager.registerFile(studyMetadata.getId(), "folder/file.vcf", Arrays.asList("s1", "s2")); + metadataManager.addIndexedFiles(studyMetadata.getId(), Collections.singletonList(fileId)); + + for (Study study : opencga.getCatalogManager().getStudyManager().searchInOrganization("test", new Query(), new QueryOptions(), opencga.getAdminToken()).getResults()) { + Assert.assertNull(study.getInternal().getConfiguration().getVariantEngine().getSetup()); + } + + runMigration(VariantSetupMigration.class); + + for (Study study : opencga.getCatalogManager().getStudyManager().searchInOrganization("test", new Query(), new QueryOptions(), opencga.getAdminToken()).getResults()) { + if (study.getFqn().equals(studyName)) { + Assert.assertNotNull(study.getInternal().getConfiguration().getVariantEngine().getSetup()); + } else { + Assert.assertNull(study.getInternal().getConfiguration().getVariantEngine().getSetup()); + } + } + } + + @After + public void tearDown() throws Exception { + if (opencga != null) { + opencga.after(); + opencga = null; + } + } + + protected void testMigration(Class migration, String dataset) throws Exception { + setup(dataset); + runMigration(migration); + } + + private void setup(String dataset) throws Exception { + setup(dataset, false); + } + + private void setup(String dataset, boolean storageHadoop) throws Exception { + if (opencga != null) { + opencga.after(); + opencga = null; + } + opencga = new OpenCGATestExternalResource(storageHadoop); + opencga.before(); + URL resource = getClass().getResource("/datasets/opencga/" + dataset + "/"); + opencga.restore(resource); + } + + private void runMigration(Class migration) throws CatalogException { + Migration annotation = migration.getAnnotation(Migration.class); + opencga.getCatalogManager().getMigrationManager() + .runManualMigration(annotation.version(), annotation.id(), opencga.getOpencgaHome(), new ObjectMap(), opencga.getAdminToken()); + } + +} \ No newline at end of file diff --git a/opencga-app/src/test/java/org/opencb/opencga/app/migrations/v2_2_0/storage/UpdateSampleIndexStatusTest.java b/opencga-app/src/test/java/org/opencb/opencga/app/migrations/v2_2_0/storage/UpdateSampleIndexStatusTest.java deleted file mode 100644 index e488896da6e..00000000000 --- a/opencga-app/src/test/java/org/opencb/opencga/app/migrations/v2_2_0/storage/UpdateSampleIndexStatusTest.java +++ /dev/null @@ -1,102 +0,0 @@ -package org.opencb.opencga.app.migrations.v2_2_0.storage; - -import org.junit.Before; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.opencb.opencga.core.testclassification.duration.ShortTests; -import org.opencb.opencga.storage.core.metadata.models.SampleMetadata; -import org.opencb.opencga.storage.core.metadata.models.TaskMetadata; - -import static org.junit.Assert.*; - -@Category(ShortTests.class) -public class UpdateSampleIndexStatusTest { - - private UpdateSampleIndexStatus migrationTool; - private SampleMetadata sm; - - @Before - public void setUp() throws Exception { - migrationTool = new UpdateSampleIndexStatus(); - } - - @Test - public void testUpdateUnindexed() { - sm = new SampleMetadata(1, 1, "1"); - assertFalse(migrationTool.needsMigration(sm)); - migrationTool.updateSampleMetadata(sm); - - check(null, null); - } - - @Test - public void testUpdateIndexedWithoutSampleIndex() { - sm = new SampleMetadata(1, 1, "1") - .setIndexStatus(TaskMetadata.Status.READY) - .setStatus(UpdateSampleIndexStatus.SAMPLE_INDEX_STATUS, TaskMetadata.Status.NONE); - assertTrue(migrationTool.needsMigration(sm)); - migrationTool.updateSampleMetadata(sm); - assertFalse(migrationTool.needsMigration(sm)); - - check(null, null); - } - - @Test - public void testUpdateUnversioned() { - sm = new SampleMetadata(1, 1, "1") - .setIndexStatus(TaskMetadata.Status.READY) - .setStatus(UpdateSampleIndexStatus.SAMPLE_INDEX_ANNOTATION_STATUS_OLD, TaskMetadata.Status.READY); - assertTrue(migrationTool.needsMigration(sm)); - migrationTool.updateSampleMetadata(sm); - assertFalse(migrationTool.needsMigration(sm)); - - check(1, 1); - } - - @Test - public void testUpdateVersionedPartial() { - sm = new SampleMetadata(1, 1, "1") - .setIndexStatus(TaskMetadata.Status.READY) - .setStatus(UpdateSampleIndexStatus.SAMPLE_INDEX_ANNOTATION_STATUS_OLD, TaskMetadata.Status.READY) - .putAttribute(UpdateSampleIndexStatus.SAMPLE_INDEX_VERSION, 2); - assertTrue(migrationTool.needsMigration(sm)); - migrationTool.updateSampleMetadata(sm); - assertFalse(migrationTool.needsMigration(sm)); - - check(2, 1); - } - - @Test - public void testUpdateVersioned() { - sm = new SampleMetadata(1, 1, "1") - .setIndexStatus(TaskMetadata.Status.READY) - .setStatus(UpdateSampleIndexStatus.SAMPLE_INDEX_ANNOTATION_STATUS_OLD, TaskMetadata.Status.READY) - .putAttribute(UpdateSampleIndexStatus.SAMPLE_INDEX_VERSION, 3) - .putAttribute(UpdateSampleIndexStatus.SAMPLE_INDEX_ANNOTATION_VERSION, 2); - assertTrue(migrationTool.needsMigration(sm)); - migrationTool.updateSampleMetadata(sm); - assertFalse(migrationTool.needsMigration(sm)); - - check(3, 2); - - } - - private void check(Integer sampleIndexVersion, Integer sampleIndexAnnotationVersion) { - System.out.println("sm = " + sm.toString()); - assertEquals(sampleIndexVersion, sm.getSampleIndexVersion()); - assertEquals(sampleIndexAnnotationVersion, sm.getSampleIndexAnnotationVersion()); - - if (sampleIndexVersion != null) { - for (int i = 0; i < sampleIndexVersion - 1; i++) { - assertEquals(TaskMetadata.Status.NONE, sm.getSampleIndexStatus(i)); - } - assertEquals(TaskMetadata.Status.READY, sm.getSampleIndexStatus(sampleIndexVersion)); - } - if (sampleIndexAnnotationVersion != null) { - for (int i = 0; i < sampleIndexAnnotationVersion - 1; i++) { - assertEquals(TaskMetadata.Status.NONE, sm.getSampleIndexAnnotationStatus(i)); - } - assertEquals(TaskMetadata.Status.READY, sm.getSampleIndexAnnotationStatus(sampleIndexAnnotationVersion)); - } - } -} \ No newline at end of file diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/audit.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/audit.json.gz new file mode 100644 index 00000000000..2b44b35ae32 Binary files /dev/null and b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/audit.json.gz differ diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/file.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/file.json.gz new file mode 100644 index 00000000000..cb73bece530 Binary files /dev/null and b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/file.json.gz differ diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/migration.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/migration.json.gz new file mode 100644 index 00000000000..58bb941a512 Binary files /dev/null and b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/migration.json.gz differ diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/note.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/note.json.gz new file mode 100644 index 00000000000..7500b56e11e Binary files /dev/null and b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/note.json.gz differ diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/organization.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/organization.json.gz new file mode 100644 index 00000000000..57cafc8910a Binary files /dev/null and b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/organization.json.gz differ diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/project.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/project.json.gz new file mode 100644 index 00000000000..ef669b9c164 Binary files /dev/null and b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/project.json.gz differ diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/study.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/study.json.gz new file mode 100644 index 00000000000..e5aad572d01 Binary files /dev/null and b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/study.json.gz differ diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/user.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/user.json.gz new file mode 100644 index 00000000000..3425717560d Binary files /dev/null and b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/opencga/user.json.gz differ diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/audit.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/audit.json.gz new file mode 100644 index 00000000000..9e783d0c02b Binary files /dev/null and b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/audit.json.gz differ diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/cohort.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/cohort.json.gz new file mode 100644 index 00000000000..fff41de3126 Binary files /dev/null and b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/cohort.json.gz differ diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/file.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/file.json.gz new file mode 100644 index 00000000000..72820a3d31c Binary files /dev/null and b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/file.json.gz differ diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/individual.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/individual.json.gz new file mode 100644 index 00000000000..ed972760e0b Binary files /dev/null and b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/individual.json.gz differ diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/migration.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/migration.json.gz new file mode 100644 index 00000000000..432b4097641 Binary files /dev/null and b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/migration.json.gz differ diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/organization.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/organization.json.gz new file mode 100644 index 00000000000..9290eaeeff1 Binary files /dev/null and b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/organization.json.gz differ diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/project.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/project.json.gz new file mode 100644 index 00000000000..41e2ae77fa8 Binary files /dev/null and b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/project.json.gz differ diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/sample.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/sample.json.gz new file mode 100644 index 00000000000..f8a2ba4f5fe Binary files /dev/null and b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/sample.json.gz differ diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/study.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/study.json.gz new file mode 100644 index 00000000000..0ea0384acdc Binary files /dev/null and b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/study.json.gz differ diff --git a/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/user.json.gz b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/user.json.gz new file mode 100644 index 00000000000..aefbad4adcc Binary files /dev/null and b/opencga-app/src/test/resources/datasets/opencga/v3.0.0/mongodb/test/user.json.gz differ diff --git a/opencga-catalog/pom.xml b/opencga-catalog/pom.xml index 4cbfa454138..c55f55fadd5 100644 --- a/opencga-catalog/pom.xml +++ b/opencga-catalog/pom.xml @@ -47,10 +47,6 @@ org.opencb.commons commons-datastore-mongodb - - org.opencb.commons - commons-datastore-solr - org.opencb.biodata biodata-formats @@ -65,6 +61,11 @@ + + org.mockito + mockito-core + test + org.apache.commons commons-collections4 @@ -114,10 +115,6 @@ jjwt-impl runtime - - org.apache.solr - solr-solrj - com.microsoft.azure adal4j @@ -192,11 +189,6 @@ com.github.samtools htsjdk - - org.apache.solr - solr-core - test - org.hamcrest hamcrest-core @@ -213,28 +205,11 @@ com.microsoft.azure azure-client-runtime - - com.microsoft.azure - azure - - - io.fabric8 - kubernetes-client - - - com.microsoft.azure - azure-batch - io.jsonwebtoken jjwt-jackson runtime - - org.apache.solr - solr-test-framework - test - diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/AuthenticationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/AuthenticationManager.java index 569ee7de57a..d93cf48c4a7 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/AuthenticationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/AuthenticationManager.java @@ -20,6 +20,7 @@ import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.user.AuthenticationResponse; import org.opencb.opencga.core.models.user.User; import org.opencb.opencga.core.response.OpenCGAResult; @@ -27,6 +28,7 @@ import org.slf4j.LoggerFactory; import javax.crypto.spec.SecretKeySpec; +import java.io.Closeable; import java.security.Key; import java.util.Collections; import java.util.Date; @@ -36,7 +38,7 @@ /** * @author Jacobo Coll <jacobo167@gmail.com> */ -public abstract class AuthenticationManager { +public abstract class AuthenticationManager implements Closeable { protected JwtManager jwtManager; @@ -61,13 +63,15 @@ Key converStringToKeyObject(String keyString, String jcaAlgorithm) { /** * Authenticate the user against the Authentication server. * - * @param userId User to authenticate + * @param organizationId Organization id. + * @param userId User to authenticate * @param password Password. * @return AuthenticationResponse object. * @throws CatalogAuthenticationException CatalogAuthenticationException if any of the credentials are wrong or the access is denied - * for any other reason. + * for any other reason. */ - public abstract AuthenticationResponse authenticate(String userId, String password) throws CatalogAuthenticationException; + public abstract AuthenticationResponse authenticate(String organizationId, String userId, String password) + throws CatalogAuthenticationException; /** * Authenticate the user against the Authentication server. @@ -77,7 +81,14 @@ Key converStringToKeyObject(String keyString, String jcaAlgorithm) { * @throws CatalogAuthenticationException CatalogAuthenticationException if any of the credentials are wrong or the access is denied * for any other reason. */ - public abstract AuthenticationResponse refreshToken(String refreshToken) throws CatalogAuthenticationException; + public AuthenticationResponse refreshToken(String refreshToken) throws CatalogAuthenticationException { + JwtPayload payload = getPayload(refreshToken); + if (!ParamConstants.ANONYMOUS_USER_ID.equals(payload.getUserId())) { + return new AuthenticationResponse(createToken(payload.getOrganization(), payload.getUserId())); + } else { + throw new CatalogAuthenticationException("Cannot refresh token for '" + ParamConstants.ANONYMOUS_USER_ID + "'."); + } + } /** * Obtains the userId corresponding to the token. @@ -88,12 +99,27 @@ Key converStringToKeyObject(String keyString, String jcaAlgorithm) { */ public String getUserId(String token) throws CatalogAuthenticationException { if (StringUtils.isEmpty(token) || "null".equalsIgnoreCase(token)) { - return ParamConstants.ANONYMOUS_USER_ID; + throw new CatalogAuthenticationException("Token is null or empty."); } return jwtManager.getUser(token); } + /** + * Obtains the userId corresponding to the token. + * + * @param token token that have been assigned to a user. + * @return the user id corresponding to the token given. + * @throws CatalogAuthenticationException when the token does not correspond to any user or the token has expired. + */ + public JwtPayload getPayload(String token) throws CatalogAuthenticationException { + if (StringUtils.isEmpty(token) || "null".equalsIgnoreCase(token)) { + throw new CatalogAuthenticationException("Token is null or empty."); + } + + return jwtManager.getPayload(token); + } + public abstract List getUsersFromRemoteGroup(String group) throws CatalogException; public abstract List getRemoteUserInformation(List userStringList) throws CatalogException; @@ -103,93 +129,92 @@ public String getUserId(String token) throws CatalogAuthenticationException { /** * Change users password. Could throw "UnsupportedOperationException" depending if the implementation supports password changes. * - * @param userId UserId - * @param oldPassword Old password - * @param newPassword New password + * @param organizationId Organization id. + * @param userId UserId + * @param oldPassword Old password + * @param newPassword New password * @throws CatalogException CatalogException */ - public abstract void changePassword(String userId, String oldPassword, String newPassword) throws CatalogException; + public abstract void changePassword(String organizationId, String userId, String oldPassword, String newPassword) + throws CatalogException; /** * Reset the user password. Sets an automatically generated password and sends an email to the user. * Throws "UnsupportedOperationException" is the implementation does not support this operation. * - * @param userId UserId + * @param organizationId Organization id. + * @param userId UserId * @return OpenCGAResult OpenCGAResult * @throws CatalogException CatalogException */ - public abstract OpenCGAResult resetPassword(String userId) throws CatalogException; + public abstract OpenCGAResult resetPassword(String organizationId, String userId) throws CatalogException; /** * Set a password to a user without a password. * Throws "UnsupportedOperationException" is the implementation does not support this operation. * - * @param userId UserId without password - * @param newPassword New password + * @param organizationId Organization id. + * @param userId UserId without password + * @param newPassword New password * @throws CatalogException CatalogException */ - public abstract void newPassword(String userId, String newPassword) throws CatalogException; + public abstract void newPassword(String organizationId, String userId, String newPassword) throws CatalogException; /** * Create a token for the user with default expiration time. * - * @param userId user. + * @param organizationId Organization id. + * @param userId user. * @return A token. */ - public String createToken(String userId) { - return createToken(userId, Collections.emptyMap(), expiration); + public String createToken(String organizationId, String userId) { + return createToken(organizationId, userId, Collections.emptyMap(), expiration); } /** * Create a token for the user with default expiration time. * - * @param userId user. - * @param expiration expiration time. + * @param organizationId Organization id. + * @param userId user. + * @param expiration expiration time. * @return A token. */ - public String createToken(String userId, long expiration) { - return createToken(userId, Collections.emptyMap(), expiration); + public String createToken(String organizationId, String userId, long expiration) { + return createToken(organizationId, userId, Collections.emptyMap(), expiration); } /** * Create a token for the user with default expiration time. * - * @param userId user. - * @param claims claims. + * @param organizationId Organization id. + * @param userId user. + * @param claims claims. * @return A token. */ - public String createToken(String userId, Map claims) { - return createToken(userId, claims, expiration); + public String createToken(String organizationId, String userId, Map claims) { + return createToken(organizationId, userId, claims, expiration); } /** * Create a token for the user. * - * @param userId user. - * @param claims claims. - * @param expiration Expiration time in seconds. + * @param organizationId Organization id. + * @param userId user. + * @param claims claims. + * @param expiration Expiration time in seconds. * @return A token. */ - public abstract String createToken(String userId, Map claims, long expiration); - - /** - * Create a token for the user with no expiration time. - * - * @param userId user. - * @return A token. - */ - public String createNonExpiringToken(String userId) { - return createNonExpiringToken(userId, Collections.emptyMap()); - } + public abstract String createToken(String organizationId, String userId, Map claims, long expiration); /** * Create a token for the user with no expiration time. * - * @param userId user. - * @param claims claims. + * @param organizationId Organization id. + * @param userId user. + * @param claims claims. * @return A token. */ - public abstract String createNonExpiringToken(String userId, Map claims); + public abstract String createNonExpiringToken(String organizationId, String userId, Map claims); public Date getExpirationDate(String token) throws CatalogAuthenticationException { return jwtManager.getExpiration(token); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/AzureADAuthenticationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/AzureADAuthenticationManager.java index 06854118d47..6a3a0480c28 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/AzureADAuthenticationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/AzureADAuthenticationManager.java @@ -39,7 +39,9 @@ import org.opencb.opencga.catalog.auth.authentication.azure.AuthenticationProvider; import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.AuthenticationOrigin; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.user.*; import org.opencb.opencga.core.response.OpenCGAResult; @@ -145,6 +147,15 @@ public AzureADAuthenticationManager(AuthenticationOrigin authenticationOrigin) t Configurator.setLevel("com.microsoft.aad.adal4j.AuthenticationAuthority", Level.WARN); } + public static void validateAuthenticationOriginConfiguration(AuthenticationOrigin authenticationOrigin) throws CatalogException { + if (authenticationOrigin.getType() != AuthenticationOrigin.AuthenticationType.AzureAD) { + throw new CatalogException("Unknown authentication type. Expected type '" + AuthenticationOrigin.AuthenticationType.AzureAD + + "' but received '" + authenticationOrigin.getType() + "'."); + } + AzureADAuthenticationManager azureADAuthenticationManager = new AzureADAuthenticationManager(authenticationOrigin); + azureADAuthenticationManager.close(); + } + private OIDCProviderMetadata getProviderMetadata(String host) throws IOException, ParseException { URL providerConfigurationURL = new URL(host); InputStream stream = providerConfigurationURL.openStream(); @@ -222,7 +233,8 @@ private PublicKey getPublicKey(String token) throws CatalogAuthenticationExcepti @Override - public AuthenticationResponse authenticate(String userId, String password) throws CatalogAuthenticationException { + public AuthenticationResponse authenticate(String organizationId, String userId, String password) + throws CatalogAuthenticationException { AuthenticationContext context; AuthenticationResult result; try { @@ -371,10 +383,10 @@ private List extractUserInformation(List(), attributes); + Account account = new Account().setAuthentication(new Account.AuthenticationOrigin(originId, false)); + User user = new User(id, name, mail, "", TimeUtils.getTime(), TimeUtils.getTime(), + new UserInternal(new UserStatus(), account), new UserQuota(-1, -1, -1, -1), + Collections.emptyMap(), new LinkedList<>(), attributes); userList.add(user); } @@ -388,30 +400,39 @@ public String getUserId(String token) throws CatalogAuthenticationException { } @Override - public void changePassword(String userId, String oldPassword, String newPassword) throws CatalogException { + public JwtPayload getPayload(String token) throws CatalogAuthenticationException { + return jwtManager.getPayload(token, getPublicKey(token)); + } + + @Override + public void changePassword(String organizationId, String userId, String oldPassword, String newPassword) throws CatalogException { throw new UnsupportedOperationException("Please, contact the Azure administrator to change the password."); } @Override - public OpenCGAResult resetPassword(String userId) throws CatalogException { + public OpenCGAResult resetPassword(String organizationId, String userId) throws CatalogException { throw new UnsupportedOperationException("Please, contact the Azure administrator to reset the password."); } @Override - public void newPassword(String userId, String newPassword) throws CatalogException { + public void newPassword(String organizationId, String userId, String newPassword) throws CatalogException { throw new UnsupportedOperationException("Please, contact the Azure administrator to renew the password."); } @Override - public String createToken(String userId, Map claims, long expiration) { + public String createToken(String organizationId, String userId, Map claims, long expiration) { // Tokens are generated by Azure via authorization code or user-password throw new UnsupportedOperationException("Tokens are generated by Azure via authorization code or user-password"); } @Override - public String createNonExpiringToken(String userId, Map claims) { + public String createNonExpiringToken(String organizationId, String userId, Map claims) { // Tokens are generated by Azure via authorization code or user-password throw new UnsupportedOperationException("Tokens are generated by Azure via authorization code or user-password"); } + @Override + public void close() { + THREAD_POOL.shutdown(); + } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/CatalogAuthenticationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/CatalogAuthenticationManager.java index 88b807edd3e..61f9fef8079 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/CatalogAuthenticationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/CatalogAuthenticationManager.java @@ -17,7 +17,6 @@ package org.opencb.opencga.catalog.auth.authentication; import io.jsonwebtoken.SignatureAlgorithm; -import org.apache.commons.lang3.RandomStringUtils; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.catalog.db.DBAdaptorFactory; import org.opencb.opencga.catalog.db.api.UserDBAdaptor; @@ -26,6 +25,8 @@ import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.common.MailUtils; +import org.opencb.opencga.core.common.PasswordUtils; +import org.opencb.opencga.core.config.AuthenticationOrigin; import org.opencb.opencga.core.config.Email; import org.opencb.opencga.core.models.user.AuthenticationResponse; import org.opencb.opencga.core.models.user.User; @@ -41,40 +42,47 @@ */ public class CatalogAuthenticationManager extends AuthenticationManager { + // TODO: Remove INTERNAL field and its usages after several releases (TASK-5923) + @Deprecated public static final String INTERNAL = "internal"; + public static final String OPENCGA = "OPENCGA"; private final Email emailConfig; - private final UserDBAdaptor userDBAdaptor; + private final DBAdaptorFactory dbAdaptorFactory; - public CatalogAuthenticationManager(DBAdaptorFactory dbAdaptorFactory, Email emailConfig, String secretKeyString, long expiration) { + public CatalogAuthenticationManager(DBAdaptorFactory dbAdaptorFactory, Email emailConfig, String algorithm, String secretKeyString, + long expiration) { super(expiration); this.emailConfig = emailConfig; - this.userDBAdaptor = dbAdaptorFactory.getCatalogUserDBAdaptor(); + this.dbAdaptorFactory = dbAdaptorFactory; - Key secretKey = this.converStringToKeyObject(secretKeyString, SignatureAlgorithm.HS256.getJcaName()); - this.jwtManager = new JwtManager(SignatureAlgorithm.HS256.getValue(), secretKey); + SignatureAlgorithm signatureAlgorithm = SignatureAlgorithm.valueOf(algorithm); + Key secretKey = this.converStringToKeyObject(secretKeyString, signatureAlgorithm.getJcaName()); + this.jwtManager = new JwtManager(signatureAlgorithm.getValue(), secretKey); this.logger = LoggerFactory.getLogger(CatalogAuthenticationManager.class); } - @Override - public AuthenticationResponse authenticate(String userId, String password) throws CatalogAuthenticationException { - try { - userDBAdaptor.authenticate(userId, password); - return new AuthenticationResponse(createToken(userId)); - } catch (CatalogDBException e) { - throw new CatalogAuthenticationException("Could not validate '" + userId + "' password\n" + e.getMessage(), e); + public static void validateAuthenticationOriginConfiguration(AuthenticationOrigin authenticationOrigin) throws CatalogException { + if (!OPENCGA.equals(authenticationOrigin.getId())) { + throw new CatalogException("Unknown authentication origin. Expected origin id '" + OPENCGA + "' but received '" + + authenticationOrigin.getId() + "'."); + } + if (authenticationOrigin.getType() != AuthenticationOrigin.AuthenticationType.OPENCGA) { + throw new CatalogException("Unknown authentication type. Expected type '" + AuthenticationOrigin.AuthenticationType.OPENCGA + + "' but received '" + authenticationOrigin.getType() + "'."); } } @Override - public AuthenticationResponse refreshToken(String refreshToken) throws CatalogAuthenticationException { - String userId = getUserId(refreshToken); - if (!"*".equals(userId)) { - return new AuthenticationResponse(createToken(userId)); - } else { - throw new CatalogAuthenticationException("Cannot refresh token for '*'"); + public AuthenticationResponse authenticate(String organizationId, String userId, String password) + throws CatalogAuthenticationException { + try { + dbAdaptorFactory.getCatalogUserDBAdaptor(organizationId).authenticate(userId, password); + return new AuthenticationResponse(createToken(organizationId, userId)); + } catch (CatalogDBException e) { + throw new CatalogAuthenticationException("Could not validate '" + userId + "' password\n" + e.getMessage(), e); } } @@ -94,33 +102,34 @@ public List getRemoteGroups(String token) throws CatalogException { } @Override - public void changePassword(String userId, String oldPassword, String newPassword) throws CatalogException { - userDBAdaptor.changePassword(userId, oldPassword, newPassword); + public void changePassword(String organizationId, String userId, String oldPassword, String newPassword) throws CatalogException { + dbAdaptorFactory.getCatalogUserDBAdaptor(organizationId).changePassword(userId, oldPassword, newPassword); } @Override - public void newPassword(String userId, String newPassword) throws CatalogException { - userDBAdaptor.changePassword(userId, "", newPassword); + public void newPassword(String organizationId, String userId, String newPassword) throws CatalogException { + dbAdaptorFactory.getCatalogUserDBAdaptor(organizationId).changePassword(userId, "", newPassword); } @Override - public String createToken(String userId, Map claims, long expiration) { - return jwtManager.createJWTToken(userId, claims, expiration); + public String createToken(String organizationId, String userId, Map claims, long expiration) { + return jwtManager.createJWTToken(organizationId, AuthenticationOrigin.AuthenticationType.OPENCGA, userId, claims, expiration); } @Override - public String createNonExpiringToken(String userId, Map claims) { - return jwtManager.createJWTToken(userId, claims, 0L); + public String createNonExpiringToken(String organizationId, String userId, Map claims) { + return jwtManager.createJWTToken(organizationId, AuthenticationOrigin.AuthenticationType.OPENCGA, userId, claims, 0L); } @Override - public OpenCGAResult resetPassword(String userId) throws CatalogException { + public OpenCGAResult resetPassword(String organizationId, String userId) throws CatalogException { ParamUtils.checkParameter(userId, "userId"); OpenCGAResult result = null; - String newPassword = RandomStringUtils.randomAlphanumeric(12); + String newPassword = PasswordUtils.getStrongRandomPassword(); OpenCGAResult user = - userDBAdaptor.get(userId, new QueryOptions(QueryOptions.INCLUDE, UserDBAdaptor.QueryParams.EMAIL.key())); + dbAdaptorFactory.getCatalogUserDBAdaptor(organizationId) + .get(userId, new QueryOptions(QueryOptions.INCLUDE, UserDBAdaptor.QueryParams.EMAIL.key())); if (user == null || user.getNumResults() != 1) { throw new CatalogException("Could not retrieve the user e-mail."); @@ -134,11 +143,21 @@ public OpenCGAResult resetPassword(String userId) throws CatalogException { String mailPort = this.emailConfig.getPort(); try { MailUtils.sendResetPasswordMail(email, newPassword, mailUser, mailPassword, mailHost, mailPort, userId); - result = userDBAdaptor.resetPassword(userId, email, newPassword); + result = dbAdaptorFactory.getCatalogUserDBAdaptor(organizationId).resetPassword(userId, email, newPassword); } catch (Exception e) { throw new CatalogException("Email could not be sent.", e); } return result; } + + public static AuthenticationOrigin createOpencgaAuthenticationOrigin() { + return new AuthenticationOrigin() + .setId(CatalogAuthenticationManager.OPENCGA) + .setType(AuthenticationOrigin.AuthenticationType.OPENCGA); + } + + @Override + public void close() { + } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/JwtManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/JwtManager.java index 15034dc866a..fd0462e3a98 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/JwtManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/JwtManager.java @@ -18,6 +18,8 @@ import io.jsonwebtoken.*; import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; +import org.opencb.opencga.core.config.AuthenticationOrigin; +import org.opencb.opencga.core.models.JwtPayload; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,6 +30,8 @@ import java.util.List; import java.util.Map; +import static org.opencb.opencga.core.models.JwtPayload.AUTH_ORIGIN; + public class JwtManager { private SignatureAlgorithm algorithm; @@ -83,19 +87,21 @@ public JwtManager setPublicKey(Key publicKey) { return this; } - public String createJWTToken(String userId, long expiration) { - return createJWTToken(userId, Collections.emptyMap(), expiration); - } - - public String createJWTToken(String userId, Map claims, long expiration) { + public String createJWTToken(String organizationId, AuthenticationOrigin.AuthenticationType type, String userId, + Map claims, long expiration) { long currentTime = System.currentTimeMillis(); JwtBuilder jwtBuilder = Jwts.builder(); if (claims != null && !claims.isEmpty()) { jwtBuilder.setClaims(claims); } + if (type != null) { + jwtBuilder.addClaims(Collections.singletonMap(AUTH_ORIGIN, type)); + } + jwtBuilder.setSubject(userId) - .setAudience("OpenCGA users") + .setAudience(organizationId) + .setIssuer("OpenCGA") .setIssuedAt(new Date(currentTime)) .signWith(privateKey, algorithm); @@ -115,6 +121,27 @@ public void validateToken(String token, Key publicKey) throws CatalogAuthenticat parseClaims(token, publicKey); } + public JwtPayload getPayload(String token) throws CatalogAuthenticationException { + Claims body = parseClaims(token, publicKey).getBody(); + return new JwtPayload(body.getSubject(), body.getAudience(), getAuthOrigin(body), body.getIssuer(), body.getIssuedAt(), + body.getExpiration(), token); + } + + public JwtPayload getPayload(String token, Key publicKey) throws CatalogAuthenticationException { + Claims body = parseClaims(token, publicKey).getBody(); + return new JwtPayload(body.getSubject(), body.getAudience(), getAuthOrigin(body), body.getIssuer(), body.getIssuedAt(), + body.getExpiration(), token); + } + + private AuthenticationOrigin.AuthenticationType getAuthOrigin(Claims claims) { + String o = claims.get(AUTH_ORIGIN, String.class); + if (o != null) { + return AuthenticationOrigin.AuthenticationType.valueOf(o); + } else { + return null; + } + } + public String getAudience(String token) throws CatalogAuthenticationException { return getAudience(token, this.publicKey); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/LDAPAuthenticationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/LDAPAuthenticationManager.java index 5d74c8c1f1b..f79eb85b19d 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/LDAPAuthenticationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/LDAPAuthenticationManager.java @@ -24,8 +24,10 @@ import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.AuthenticationOrigin; +import org.opencb.opencga.core.models.organizations.TokenConfiguration; import org.opencb.opencga.core.models.user.*; import org.opencb.opencga.core.response.OpenCGAResult; import org.slf4j.LoggerFactory; @@ -66,7 +68,7 @@ public class LDAPAuthenticationManager extends AuthenticationManager { private String host; private boolean ldaps; - public LDAPAuthenticationManager(AuthenticationOrigin authenticationOrigin, String secretKeyString, long expiration) { + public LDAPAuthenticationManager(AuthenticationOrigin authenticationOrigin, String algorithm, String secretKeyString, long expiration) { super(expiration); this.logger = LoggerFactory.getLogger(LDAPAuthenticationManager.class); this.host = authenticationOrigin.getHost(); @@ -107,8 +109,9 @@ public LDAPAuthenticationManager(AuthenticationOrigin authenticationOrigin, Stri logger.info("Init LDAP AuthenticationManager. Host: {}, env:{}", host, envToStringRedacted(getDefaultEnv())); - Key secretKey = this.converStringToKeyObject(secretKeyString, SignatureAlgorithm.HS256.getJcaName()); - this.jwtManager = new JwtManager(SignatureAlgorithm.HS256.getValue(), secretKey); + SignatureAlgorithm signatureAlgorithm = SignatureAlgorithm.valueOf(algorithm); + Key secretKey = this.converStringToKeyObject(secretKeyString, signatureAlgorithm.getJcaName()); + this.jwtManager = new JwtManager(signatureAlgorithm.getValue(), secretKey); } protected static String envToStringRedacted(Hashtable env) { @@ -123,8 +126,27 @@ protected static String envToStringRedacted(Hashtable env) { return string; } + public static void validateAuthenticationOriginConfiguration(AuthenticationOrigin authenticationOrigin) throws CatalogException { + if (authenticationOrigin.getType() != AuthenticationType.LDAP) { + throw new CatalogException("Unknown authentication type. Expected type '" + AuthenticationType.LDAP + "' but received '" + + authenticationOrigin.getType() + "'."); + } + ParamUtils.checkParameter(authenticationOrigin.getHost(), AuthenticationType.LDAP + " host."); + + TokenConfiguration defaultTokenConfig = TokenConfiguration.init(); + LDAPAuthenticationManager ldapAuthenticationManager = new LDAPAuthenticationManager(authenticationOrigin, + defaultTokenConfig.getAlgorithm(), defaultTokenConfig.getSecretKey(), defaultTokenConfig.getExpiration()); + DirContext dirContext = ldapAuthenticationManager.getDirContext(ldapAuthenticationManager.getDefaultEnv(), 1); + if (dirContext == null) { + throw new CatalogException("LDAP: Could not connect to the LDAP server using the provided configuration."); + } + ldapAuthenticationManager.closeDirContextAndSuppress(dirContext, new Exception()); + ldapAuthenticationManager.close(); + } + @Override - public AuthenticationResponse authenticate(String userId, String password) throws CatalogAuthenticationException { + public AuthenticationResponse authenticate(String organizationId, String userId, String password) + throws CatalogAuthenticationException { Map claims = new HashMap<>(); List userInfoFromLDAP = getUserInfoFromLDAP(Arrays.asList(userId), usersSearch); @@ -143,17 +165,7 @@ public AuthenticationResponse authenticate(String userId, String password) throw throw wrapException(e); } - return new AuthenticationResponse(createToken(userId, claims)); - } - - @Override - public AuthenticationResponse refreshToken(String refreshToken) throws CatalogAuthenticationException { - String userId = getUserId(refreshToken); - if (!"*".equals(userId)) { - return new AuthenticationResponse(createToken(userId)); - } else { - throw new CatalogAuthenticationException("Cannot refresh token for '*'"); - } + return new AuthenticationResponse(createToken(organizationId, userId, claims)); } @Override @@ -186,9 +198,11 @@ public List getRemoteUserInformation(List userStringList) throws C Map attributes = new HashMap<>(); attributes.put("LDAP_RDN", rdn); - User user = new User(uid, displayName, mail, usersSearch, new Account().setType(Account.AccountType.GUEST) - .setAuthentication(new Account.AuthenticationOrigin(originId, false)), new UserInternal(new UserStatus()), - new UserQuota(-1, -1, -1, -1), new ArrayList<>(), new ArrayList<>(), new HashMap<>(), new LinkedList<>(), attributes); + Account account = new Account() + .setAuthentication(new Account.AuthenticationOrigin(originId, false)); + User user = new User(uid, displayName, mail, usersSearch, TimeUtils.getTime(), TimeUtils.getTime(), + new UserInternal(new UserStatus(), account), + new UserQuota(-1, -1, -1, -1), new HashMap<>(), new LinkedList<>(), attributes); userList.add(user); } @@ -205,28 +219,28 @@ public List getRemoteGroups(String token) throws CatalogException { } @Override - public void changePassword(String userId, String oldPassword, String newPassword) throws CatalogException { + public void changePassword(String organizationId, String userId, String oldPassword, String newPassword) throws CatalogException { throw new UnsupportedOperationException("Please, contact the LDAP administrator to change the password."); } @Override - public OpenCGAResult resetPassword(String userId) throws CatalogException { + public OpenCGAResult resetPassword(String organizationId, String userId) throws CatalogException { throw new UnsupportedOperationException("Please, contact the LDAP administrator to reset the password."); } @Override - public void newPassword(String userId, String newPassword) throws CatalogException { + public void newPassword(String organizationId, String userId, String newPassword) throws CatalogException { throw new UnsupportedOperationException("Please, contact the LDAP administrator to renew the password."); } @Override - public String createToken(String userId, Map claims, long expiration) { - return jwtManager.createJWTToken(userId, claims, expiration); + public String createToken(String organizationId, String userId, Map claims, long expiration) { + return jwtManager.createJWTToken(organizationId, AuthenticationType.LDAP, userId, claims, expiration); } @Override - public String createNonExpiringToken(String userId, Map claims) { - return jwtManager.createJWTToken(userId, claims, 0L); + public String createNonExpiringToken(String organizationId, String userId, Map claims) { + return jwtManager.createJWTToken(organizationId, AuthenticationType.LDAP, userId, claims, 0L); } /* Private methods */ @@ -511,4 +525,9 @@ private String takeString(ObjectMap objectMap, String key, String defaultValue) objectMap.remove(key); return value; } + + @Override + public void close() { + executorService.shutdown(); + } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/SSOAuthenticationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/SSOAuthenticationManager.java new file mode 100644 index 00000000000..e676029e413 --- /dev/null +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/SSOAuthenticationManager.java @@ -0,0 +1,87 @@ +package org.opencb.opencga.catalog.auth.authentication; + +import io.jsonwebtoken.SignatureAlgorithm; +import org.apache.commons.lang3.NotImplementedException; +import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.core.config.AuthenticationOrigin; +import org.opencb.opencga.core.models.user.AuthenticationResponse; +import org.opencb.opencga.core.models.user.User; +import org.opencb.opencga.core.response.OpenCGAResult; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.security.Key; +import java.util.List; +import java.util.Map; + +public class SSOAuthenticationManager extends AuthenticationManager { + + public SSOAuthenticationManager(String algorithm, String secretKeyString, long expiration) { + super(expiration); + + SignatureAlgorithm signatureAlgorithm = SignatureAlgorithm.valueOf(algorithm); + Key secretKey = this.converStringToKeyObject(secretKeyString, signatureAlgorithm.getJcaName()); + this.jwtManager = new JwtManager(signatureAlgorithm.getValue(), secretKey); + + this.logger = LoggerFactory.getLogger(SSOAuthenticationManager.class); + } + + public static void validateAuthenticationOriginConfiguration(AuthenticationOrigin authenticationOrigin) throws CatalogException { + if (authenticationOrigin.getType() != AuthenticationOrigin.AuthenticationType.SSO) { + throw new CatalogException("Unknown authentication type. Expected type '" + AuthenticationOrigin.AuthenticationType.SSO + + "' but received '" + authenticationOrigin.getType() + "'."); + } + } + + @Override + public AuthenticationResponse authenticate(String organizationId, String userId, String password) + throws CatalogAuthenticationException { + throw new NotImplementedException("Authentication should be done through SSO"); + } + + @Override + public List getUsersFromRemoteGroup(String group) throws CatalogException { + throw new NotImplementedException("Operation not implemented"); + } + + @Override + public List getRemoteUserInformation(List userStringList) throws CatalogException { + throw new NotImplementedException("Operation not implemented"); + } + + @Override + public List getRemoteGroups(String token) throws CatalogException { + throw new NotImplementedException("Operation not implemented"); + } + + @Override + public void changePassword(String organizationId, String userId, String oldPassword, String newPassword) throws CatalogException { + throw new NotImplementedException("Change password should be done through SSO"); + } + + @Override + public OpenCGAResult resetPassword(String organizationId, String userId) throws CatalogException { + throw new NotImplementedException("Reset password should be done through SSO"); + } + + @Override + public void newPassword(String organizationId, String userId, String newPassword) throws CatalogException { + throw new NotImplementedException("Setting a new password should be done through SSO"); + } + + @Override + public String createToken(String organizationId, String userId, Map claims, long expiration) { + return jwtManager.createJWTToken(organizationId, AuthenticationOrigin.AuthenticationType.SSO, userId, claims, expiration); + } + + @Override + public String createNonExpiringToken(String organizationId, String userId, Map claims) { + return jwtManager.createJWTToken(organizationId, AuthenticationOrigin.AuthenticationType.SSO, userId, claims, 0L); + } + + @Override + public void close() throws IOException { + + } +} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/azure/AuthenticationFactory.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/azure/AuthenticationFactory.java new file mode 100644 index 00000000000..7172401fda7 --- /dev/null +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/azure/AuthenticationFactory.java @@ -0,0 +1,186 @@ +package org.opencb.opencga.catalog.auth.authentication.azure; + +import org.apache.commons.collections4.CollectionUtils; +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.catalog.auth.authentication.*; +import org.opencb.opencga.catalog.db.DBAdaptorFactory; +import org.opencb.opencga.catalog.db.api.OrganizationDBAdaptor; +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.managers.OrganizationManager; +import org.opencb.opencga.catalog.utils.ParamUtils; +import org.opencb.opencga.core.config.AuthenticationOrigin; +import org.opencb.opencga.core.config.Configuration; +import org.opencb.opencga.core.models.organizations.Organization; +import org.opencb.opencga.core.models.user.AuthenticationResponse; +import org.opencb.opencga.core.models.user.User; +import org.opencb.opencga.core.response.OpenCGAResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public final class AuthenticationFactory { + + // Map of organizationId -> authenticationOriginId -> AuthenticationManager + private final Map> authenticationManagerMap; + private final Logger logger = LoggerFactory.getLogger(AuthenticationFactory.class); + private final DBAdaptorFactory catalogDBAdaptorFactory; + private final Configuration configuration; + + public AuthenticationFactory(DBAdaptorFactory catalogDBAdaptorFactory, Configuration configuration) { + this.catalogDBAdaptorFactory = catalogDBAdaptorFactory; + this.configuration = configuration; + authenticationManagerMap = new ConcurrentHashMap<>(); + } + + public void configureOrganizationAuthenticationManager(Organization organization) throws CatalogException { + Map tmpAuthenticationManagerMap = new HashMap<>(); + + long expiration = organization.getConfiguration().getToken().getExpiration(); + String algorithm = organization.getConfiguration().getToken().getAlgorithm(); + String secretKey = organization.getConfiguration().getToken().getSecretKey(); + + if (organization.getConfiguration() != null + && CollectionUtils.isNotEmpty(organization.getConfiguration().getAuthenticationOrigins())) { + for (AuthenticationOrigin authOrigin : organization.getConfiguration().getAuthenticationOrigins()) { + if (authOrigin.getId() != null) { + switch (authOrigin.getType()) { + case LDAP: + tmpAuthenticationManagerMap.put(authOrigin.getId(), + new LDAPAuthenticationManager(authOrigin, algorithm, secretKey, expiration)); + break; + case AzureAD: + tmpAuthenticationManagerMap.put(authOrigin.getId(), new AzureADAuthenticationManager(authOrigin)); + break; + case OPENCGA: + CatalogAuthenticationManager catalogAuthenticationManager = + new CatalogAuthenticationManager(catalogDBAdaptorFactory, configuration.getEmail(), algorithm, + secretKey, expiration); + tmpAuthenticationManagerMap.put(CatalogAuthenticationManager.INTERNAL, catalogAuthenticationManager); + tmpAuthenticationManagerMap.put(CatalogAuthenticationManager.OPENCGA, catalogAuthenticationManager); + break; + case SSO: + tmpAuthenticationManagerMap.put(authOrigin.getId(), new SSOAuthenticationManager(algorithm, secretKey, + expiration)); + break; + default: + logger.warn("Unexpected authentication origin type '{}' for id '{}' found in organization '{}'. " + + "Authentication origin will be ignored.", authOrigin.getType(), organization.getId(), + authOrigin.getId()); + break; + } + } + } + } + if (tmpAuthenticationManagerMap.isEmpty()) { + throw new CatalogException("No authentication origin found for organization '" + organization.getId() + "'"); + } + if (authenticationManagerMap.containsKey(organization.getId())) { + for (AuthenticationManager authenticationManager : authenticationManagerMap.get(organization.getId()).values()) { + try { + logger.info("Closing previous authentication manager for organization '{}'", organization.getId()); + authenticationManager.close(); + } catch (IOException e) { + throw new CatalogException("Unable to close previous authentication manager for organization '" + organization.getId() + + "'.", e); + } + } + logger.info("Reloading new set of AuthenticationManagers for organization '{}'", organization.getId()); + } + authenticationManagerMap.put(organization.getId(), tmpAuthenticationManagerMap); + } + + public String createToken(String organizationId, String authOriginId, String userId) throws CatalogException { + return getOrganizationAuthenticationManager(organizationId, authOriginId).createToken(organizationId, userId); + } + + public void validateToken(String organizationId, String authOriginId, String token) throws CatalogException { + getOrganizationAuthenticationManager(organizationId, authOriginId).getUserId(token); + } + + public AuthenticationResponse authenticate(String organizationId, String authenticationOriginId, String userId, String password) + throws CatalogException { + AuthenticationManager organizationAuthenticationManager = getOrganizationAuthenticationManager(organizationId, + authenticationOriginId); + return organizationAuthenticationManager.authenticate(organizationId, userId, password); + } + + public void changePassword(String organizationId, String authOriginId, String userId, String oldPassword, String newPassword) + throws CatalogException { + getOrganizationAuthenticationManager(organizationId, authOriginId).changePassword(organizationId, userId, oldPassword, newPassword); + } + + public OpenCGAResult resetPassword(String organizationId, String authOriginId, String userId) throws CatalogException { + return getOrganizationAuthenticationManager(organizationId, authOriginId).resetPassword(organizationId, userId); + } + + public String getUserId(String organizationId, String authOriginId, String token) throws CatalogException { + return getOrganizationAuthenticationManager(organizationId, authOriginId).getUserId(token); + } + + public List getRemoteUserInformation(String organizationId, String authOriginId, List userStringList) + throws CatalogException { + return getOrganizationAuthenticationManager(organizationId, authOriginId).getRemoteUserInformation(userStringList); + } + + public List getUsersFromRemoteGroup(String organizationId, String authOriginId, String group) throws CatalogException { + return getOrganizationAuthenticationManager(organizationId, authOriginId).getUsersFromRemoteGroup(group); + } + + public List getRemoteGroups(String organizationId, String authOriginId, String token) throws CatalogException { + return getOrganizationAuthenticationManager(organizationId, authOriginId).getRemoteGroups(token); + } + + public Map getOrganizationAuthenticationManagers(String organizationId) throws CatalogException { + if (!authenticationManagerMap.containsKey(organizationId)) { + // Check if the organization exists (it must have been created on a different instance) + for (String id : catalogDBAdaptorFactory.getOrganizationIds()) { + if (id.equals(organizationId)) { + QueryOptions options = new QueryOptions(OrganizationManager.INCLUDE_ORGANIZATION_CONFIGURATION); + options.put(OrganizationDBAdaptor.IS_ORGANIZATION_ADMIN_OPTION, true); + Organization organization = catalogDBAdaptorFactory.getCatalogOrganizationDBAdaptor(organizationId).get(options) + .first(); + configureOrganizationAuthenticationManager(organization); + return authenticationManagerMap.get(organizationId); + } + } + throw new CatalogException("Organization '" + organizationId + "' not found."); + } + return authenticationManagerMap.get(organizationId); + } + + public AuthenticationManager getOrganizationAuthenticationManager(String organizationId, String authOriginId) + throws CatalogException { + Map organizationAuthenticationManagers = getOrganizationAuthenticationManagers(organizationId); + if (!organizationAuthenticationManagers.containsKey(authOriginId)) { + throw new CatalogException("Authentication origin '" + authOriginId + "' for organization '" + organizationId + "' not found."); + } + return organizationAuthenticationManagers.get(authOriginId); + } + + public void validateAuthenticationOrigin(AuthenticationOrigin authenticationOrigin) throws CatalogException { + ParamUtils.checkParameter(authenticationOrigin.getId(), "authentication origin id"); + ParamUtils.checkObj(authenticationOrigin.getType(), "authentication origin type"); + switch (authenticationOrigin.getType()) { + case OPENCGA: + CatalogAuthenticationManager.validateAuthenticationOriginConfiguration(authenticationOrigin); + break; + case LDAP: + LDAPAuthenticationManager.validateAuthenticationOriginConfiguration(authenticationOrigin); + break; + case AzureAD: + AzureADAuthenticationManager.validateAuthenticationOriginConfiguration(authenticationOrigin); + break; + case SSO: + SSOAuthenticationManager.validateAuthenticationOriginConfiguration(authenticationOrigin); + break; + default: + throw new CatalogException("Unknown authentication origin type '" + authenticationOrigin.getType() + "'"); + } + } + +} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/AuthorizationDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/AuthorizationDBAdaptor.java index 58d9cd54ec5..09c142e68e9 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/AuthorizationDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/AuthorizationDBAdaptor.java @@ -16,10 +16,9 @@ package org.opencb.opencga.catalog.auth.authorization; -import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.exceptions.CatalogParameterException; +import org.opencb.opencga.core.models.Acl; import org.opencb.opencga.core.models.AclEntryList; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.study.PermissionRule; @@ -37,75 +36,77 @@ public interface AuthorizationDBAdaptor { /** * Retrieve the list of Acls for the list of members in the resource given. * + * @param Permissions enum. * @param resourceId id of the study, file, sample... where the Acl will be looked for. - * @param members members for whom the Acls will be obtained. + * @param members members for whom the Acls will be obtained. * @param userGroups Map of user id - list of groups to retrieve permissions from those groups as well. - * @param entry Entity for which the ACLs will be retrieved. - * @param clazz Class of type T. + * @param entry Entity for which the ACLs will be retrieved. + * @param clazz Class of type T. * @return the list of Acls defined for the members. - * @param Permissions enum. - * @throws CatalogException CatalogException. + * @throws CatalogException CatalogException. */ - > OpenCGAResult> get(long resourceId, List members, Map> userGroups, - Enums.Resource entry, Class clazz) throws CatalogException; + > OpenCGAResult> get(long resourceId, List members, + Map> userGroups, Enums.Resource entry, Class clazz) + throws CatalogException; /** * Retrieve the list of Acls for the list of members in the resources given. * + * @param Permissions enum. * @param resourceIds ids of the study, file, sample... where the Acl will be looked for. - * @param members members for whom the Acls will be obtained. - * @param userGroups Map of user id - list of groups to retrieve permissions from those groups as well. - * @param entry Entity for which the ACLs will be retrieved. - * @param clazz Class of type T. + * @param members members for whom the Acls will be obtained. + * @param userGroups Map of user id - list of groups to retrieve permissions from those groups as well. + * @param entry Entity for which the ACLs will be retrieved. + * @param clazz Class of type T. * @return the list of Acls defined for the members. - * @param Permissions enum. - * @throws CatalogException CatalogException. + * @throws CatalogException CatalogException. */ > OpenCGAResult> get(List resourceIds, List members, Map> userGroups, Enums.Resource entry, Class clazz) throws CatalogException; + List effectivePermissions(long studyUid, List resourceIdList, Enums.Resource entry) throws CatalogDBException; + /** * Remove all the Acls defined for the member in the resource for the study. * * @param studyId study id where the Acls will be removed from. - * @param member member from whom the Acls will be removed. - * @param entry Entity for which the ACLs will be retrieved. + * @param member member from whom the Acls will be removed. + * @param entry Entity for which the ACLs will be retrieved. * @return OpenCGAResult object. - * @throws CatalogException CatalogException. + * @throws CatalogException CatalogException. */ OpenCGAResult removeFromStudy(long studyId, String member, Enums.Resource entry) throws CatalogException; OpenCGAResult setToMembers(long studyId, List members, List aclParams) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + throws CatalogException; // Special method only to set acls in study - OpenCGAResult setToMembers(List studyIds, List members, List permissions) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult setToMembers(List studyIds, List members, List permissions) throws CatalogException; OpenCGAResult addToMembers(long studyId, List members, List aclParams) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + throws CatalogException; // Special method only to add acls in study - OpenCGAResult addToMembers(List studyIds, List members, List permissions) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult addToMembers(List studyIds, List members, List permissions) throws CatalogException; - OpenCGAResult removeFromMembers(List members, List aclParams) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult removeFromMembers(List members, List aclParams) throws CatalogException; - OpenCGAResult resetMembersFromAllEntries(long studyId, List members) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult resetMembersFromAllEntries(long studyId, List members) throws CatalogException; OpenCGAResult setAcls(List resourceIds, AclEntryList aclEntryList, Enums.Resource resource) throws CatalogDBException; - OpenCGAResult applyPermissionRules(long studyId, PermissionRule permissionRule, Enums.Entity entry) throws CatalogException; + OpenCGAResult applyPermissionRules(long studyId, PermissionRule permissionRule, Enums.Entity entry) + throws CatalogException; OpenCGAResult removePermissionRuleAndRemovePermissions(Study study, String permissionRuleId, Enums.Entity entry) throws CatalogException; - OpenCGAResult removePermissionRuleAndRestorePermissions(Study study, String permissionRuleToDeleteId, Enums.Entity entity) + OpenCGAResult removePermissionRuleAndRestorePermissions(Study study, String permissionRuleToDeleteId, + Enums.Entity entity) throws CatalogException; - OpenCGAResult removePermissionRule(long studyId, String permissionRuleToDelete, Enums.Entity entry) throws CatalogException; + OpenCGAResult removePermissionRule(long studyId, String permissionRuleToDelete, Enums.Entity entry) + throws CatalogException; } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/AuthorizationDBAdaptorFactory.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/AuthorizationDBAdaptorFactory.java new file mode 100644 index 00000000000..af03f4dd18a --- /dev/null +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/AuthorizationDBAdaptorFactory.java @@ -0,0 +1,9 @@ +package org.opencb.opencga.catalog.auth.authorization; + +import org.opencb.opencga.catalog.exceptions.CatalogDBException; + +public interface AuthorizationDBAdaptorFactory { + + AuthorizationDBAdaptor getAuthorizationDBAdaptor(String organization) throws CatalogDBException; + +} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/AuthorizationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/AuthorizationManager.java index 550faae0ea0..0b2661baa46 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/AuthorizationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/AuthorizationManager.java @@ -16,9 +16,13 @@ package org.opencb.opencga.catalog.auth.authorization; +import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.utils.ParamUtils; +import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.models.Acl; import org.opencb.opencga.core.models.AclEntryList; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.clinical.ClinicalAnalysisPermissions; import org.opencb.opencga.core.models.cohort.CohortPermissions; import org.opencb.opencga.core.models.common.Enums; @@ -80,124 +84,237 @@ static EnumSet getLockedAcls() { return EnumSet.noneOf(StudyPermissions.Permissions.class); } - void checkCanViewProject(long projectId, String userId) throws CatalogException; + void checkCanViewOrganization(String organizationId, String userId) throws CatalogException; - void checkCanEditProject(long projectId, String userId) throws CatalogException; + void checkCanViewProject(String organizationId, long projectId, String userId) throws CatalogException; - void checkStudyPermission(long studyId, String userId, StudyPermissions.Permissions permission) throws CatalogException; + void checkCanEditProject(String organizationId, long projectId, String userId) throws CatalogException; - void checkStudyPermission(long studyId, String userId, StudyPermissions.Permissions permission, String message) + void checkStudyPermission(String organizationId, long studyUid, JwtPayload payload, StudyPermissions.Permissions permission) throws CatalogException; - void checkCanEditStudy(long studyId, String userId) throws CatalogException; + void checkStudyPermission(String organizationId, long studyId, String userId, StudyPermissions.Permissions permission) + throws CatalogException; + + void checkCanEditStudy(String organizationId, long studyId, String userId) throws CatalogException; - void checkCanViewStudy(long studyId, String userId) throws CatalogException; + void checkCanViewStudy(String organizationId, long studyId, String userId) throws CatalogException; - void checkCanUpdatePermissionRules(long studyId, String userId) throws CatalogException; + void checkCanUpdatePermissionRules(String organizationId, long studyId, String userId) throws CatalogException; - void checkCreateDeleteGroupPermissions(long studyId, String userId, String group) throws CatalogException; + void checkCreateDeleteGroupPermissions(String organizationId, long studyId, String userId, String group) throws CatalogException; - void checkSyncGroupPermissions(long studyId, String userId, String group) throws CatalogException; + void checkSyncGroupPermissions(String organizationId, long studyId, String userId, String group) throws CatalogException; - void checkUpdateGroupPermissions(long studyId, String userId, String group, ParamUtils.BasicUpdateAction action) + void checkUpdateGroupPermissions(String organizationId, long studyId, String userId, String group, ParamUtils.BasicUpdateAction action) throws CatalogException; void checkNotAssigningPermissionsToAdminsGroup(List members) throws CatalogException; - void checkCanAssignOrSeePermissions(long studyId, String userId) throws CatalogException; + void checkCanAssignOrSeePermissions(String organizationId, long studyId, String userId) throws CatalogException; + + void checkCanCreateUpdateDeleteVariableSets(String organizationId, long studyId, String userId) throws CatalogException; + + default boolean isOpencgaAdministrator(JwtPayload payload) throws CatalogException { + return isOpencgaAdministrator(payload.getOrganization(), payload.getUserId()); + } + + boolean isOpencgaAdministrator(String organization, String userId) throws CatalogAuthorizationException; + + default void checkIsOpencgaAdministrator(JwtPayload payload) throws CatalogException { + checkIsOpencgaAdministrator(payload, null); + } + + default void checkIsOpencgaAdministrator(JwtPayload payload, String action) throws CatalogException { + checkIsOpencgaAdministrator(payload.getOrganization(), payload.getUserId(), action); + } + + default void checkIsOpencgaAdministrator(String organization, String userId) throws CatalogException { + checkIsOpencgaAdministrator(organization, userId, null); + } + + void checkIsOpencgaAdministrator(String organization, String userId, String action) throws CatalogException; + + /** + * Check if the given user is the organization owner. + * If the user is not the owner, it will throw an exception. + * + * @param organizationId Organization id + * @param userId User id + * @throws CatalogException CatalogException + */ + void checkIsAtLeastOrganizationOwner(String organizationId, String userId) throws CatalogException; - void checkCanCreateUpdateDeleteVariableSets(long studyId, String userId) throws CatalogException; + /** + * Check if the given user is the owner of the organization or if it is an admin. + * If the user is not the owner or an admin, it will throw an exception. + * + * @param organizationId Organization id + * @param userId User id + * @throws CatalogException CatalogException + */ + void checkIsAtLeastOrganizationOwnerOrAdmin(String organizationId, String userId) throws CatalogException; - Boolean isInstallationAdministrator(String user); + /** + * Check if the given user is the organization owner or one of the Opencga administrators. + * + * @param organization Organization + * @param userId User id + * @return true if the user is the organization owner or one of the Opencga administrators. + * @throws CatalogException CatalogDBException + */ + boolean isAtLeastOrganizationOwner(String organization, String userId) throws CatalogException; - void checkIsInstallationAdministrator(String user) throws CatalogException; + /** + * Check if the given user is the owner or one of the admins of the organization or one of the Opencga administrators. + * + * @param organization Organization + * @param userId User id + * @return true if the user is the owner or an admin of the organization or one of the Opencga administrators. + * @throws CatalogException CatalogDBException + */ + boolean isAtLeastOrganizationOwnerOrAdmin(String organization, String userId) throws CatalogException; - void checkIsOwnerOrAdmin(long studyId, String userId) throws CatalogException; + /** + * Check if the user is part of the {@link ParamConstants#ADMINS_GROUP} group of the study. + * Keep in mind that all organization admins and the organization owner are also study admins. + * It does not include the opencga admins. + * + * @param organizationId Organization id + * @param studyId Study id + * @param userId User id + * @return true if the user is part of the admins group of the study + * @throws CatalogException CatalogException + */ + boolean isAtLeastStudyAdministrator(String organizationId, long studyId, String userId) throws CatalogException; - Boolean isOwnerOrAdmin(long studyId, String userId) throws CatalogException; + /** + * Check if the user is part of the {@link ParamConstants#ADMINS_GROUP} group of the study. + * Keep in mind that all organization admins and the organization owner are also study admins. + * It does not include the opencga admins. + * If the check fails, it will throw an exception. + * + * @param organizationId Organization id + * @param studyId Study id + * @param userId User id + * @throws CatalogException CatalogException + */ + void checkIsAtLeastStudyAdministrator(String organizationId, long studyId, String userId) throws CatalogException; - void checkFilePermission(long studyId, long fileId, String userId, FilePermissions permission) throws CatalogException; + void checkFilePermission(String organizationId, long studyId, long fileId, String userId, FilePermissions permission) + throws CatalogException; - void checkSamplePermission(long studyId, long sampleId, String userId, SamplePermissions permission) + void checkSamplePermission(String organizationId, long studyId, long sampleId, String userId, SamplePermissions permission) throws CatalogException; - void checkIndividualPermission(long studyId, long individualId, String userId, IndividualPermissions permission) + void checkIndividualPermission(String organizationId, long studyId, long individualId, String userId, IndividualPermissions permission) throws CatalogException; - void checkJobPermission(long studyId, long jobId, String userId, JobPermissions permission) throws CatalogException; + void checkJobPermission(String organizationId, long studyId, long jobId, String userId, JobPermissions permission) + throws CatalogException; - void checkCohortPermission(long studyId, long cohortId, String userId, CohortPermissions permission) + void checkCohortPermission(String organizationId, long studyId, long cohortId, String userId, CohortPermissions permission) throws CatalogException; - void checkPanelPermission(long studyId, long panelId, String userId, PanelPermissions permission) + void checkPanelPermission(String organizationId, long studyId, long panelId, String userId, PanelPermissions permission) throws CatalogException; - void checkFamilyPermission(long studyId, long familyId, String userId, FamilyPermissions permission) + void checkFamilyPermission(String organizationId, long studyId, long familyId, String userId, FamilyPermissions permission) throws CatalogException; - void checkClinicalAnalysisPermission(long studyId, long analysisId, String userId, + void checkClinicalAnalysisPermission(String organizationId, long studyId, long analysisId, String userId, ClinicalAnalysisPermissions permission) throws CatalogException; + default List getEffectivePermissions(String organizationId, long studyUid, String resourceId, Enums.Resource resource) + throws CatalogException { + return getEffectivePermissions(organizationId, studyUid, Collections.singletonList(resourceId), Collections.emptyList(), resource); + } + + default List getEffectivePermissions(String organizationId, long studyUid, List resourceIdList, Enums.Resource resource) + throws CatalogException { + return getEffectivePermissions(organizationId, studyUid, resourceIdList, Collections.emptyList(), resource); + } + + default List getEffectivePermissions(String organizationId, long studyUid, String resourceId, String permission, + Enums.Resource resource) throws CatalogException { + return getEffectivePermissions(organizationId, studyUid, Collections.singletonList(resourceId), + Collections.singletonList(permission), resource); + } + + default List getEffectivePermissions(String organizationId, long studyUid, List resourceIdList, String permission, + Enums.Resource resource) throws CatalogException { + return getEffectivePermissions(organizationId, studyUid, resourceIdList, Collections.singletonList(permission), resource); + } + + List getEffectivePermissions(String organizationId, long studyUid, List resourceIdList, List permission, + Enums.Resource resource) throws CatalogException; + //------------------------- Study ACL ----------------------------- /** * Return all the ACLs defined in the study. * - * @param userId user id asking for the ACLs. - * @param studyId study id. + * @param organizationId Organization id. + * @param studyId study id. + * @param userId user id asking for the ACLs. * @return a list of studyAcls. * @throws CatalogException when the user asking to retrieve all the ACLs defined in the study does not have proper permissions. */ - OpenCGAResult> getAllStudyAcls(String userId, long studyId) throws CatalogException; + OpenCGAResult> getAllStudyAcls(String organizationId, long studyId, String userId) + throws CatalogException; /** * Return the ACL defined for the member. * - * @param userId user asking for the ACL. - * @param studyId study id. - * @param member member whose permissions will be retrieved. + * @param organizationId Organization id. + * @param studyId study id. + * @param member member whose permissions will be retrieved. + * @param userId user asking for the ACL. * @return the studyAcl for the member. * @throws CatalogException if the user does not have proper permissions to see the member permissions. */ - default OpenCGAResult> getStudyAcl(String userId, long studyId, String member) - throws CatalogException { - return getStudyAcl(userId, studyId, Collections.singletonList(member)); + default OpenCGAResult> getStudyAcl(String organizationId, long studyId, String member, + String userId) throws CatalogException { + return getStudyAcl(organizationId, studyId, Collections.singletonList(member), userId); } /** * Return the ACL defined for the member. * - * @param userId user asking for the ACL. - * @param studyId study id. - * @param members members whose permissions will be retrieved. + * @param organizationId Organization id. + * @param studyId study id. + * @param members members whose permissions will be retrieved. + * @param userId user asking for the ACL. * @return the studyAcl for the member. * @throws CatalogException if the user does not have proper permissions to see the member permissions. */ - OpenCGAResult> getStudyAcl(String userId, long studyId, List members) - throws CatalogException; + OpenCGAResult> getStudyAcl(String organizationId, long studyId, List members, + String userId) throws CatalogException; /** * Return the ACL defined for the member. * - * @param studyUid study id. - * @param members members whose permissions will be retrieved. + * @param organizationId Organization id. + * @param studyUid study id. + * @param members members whose permissions will be retrieved. * @return the studyAcl for the member. * @throws CatalogException if the user does not have proper permissions to see the member permissions. */ - default OpenCGAResult> getStudyAcl(long studyUid, List members) - throws CatalogException { - return getStudyAcl(Collections.singletonList(studyUid), members); + default OpenCGAResult> getStudyAcl(String organizationId, long studyUid, + List members) throws CatalogException { + return getStudyAcl(organizationId, Collections.singletonList(studyUid), members); } /** * Return the ACL defined for the member. * - * @param studyUids study uids. - * @param members members whose permissions will be retrieved. + * @param organizationId Organization id. + * @param studyUids study uids. + * @param members members whose permissions will be retrieved. * @return the studyAcl for the member. * @throws CatalogException if the user does not have proper permissions to see the member permissions. */ - OpenCGAResult> getStudyAcl(List studyUids, List members) + OpenCGAResult> getStudyAcl(String organizationId, List studyUids, List members) throws CatalogException; //------------------------- End of study ACL ---------------------- @@ -205,98 +322,111 @@ OpenCGAResult> getStudyAcl(List /** * Return the ACL defined for the member. * - * @param userId user asking for the ACL. - * @param studyId study uid. - * @param resourceUid Resource uid. - * @param members members whose permissions will be retrieved. - * @param resource Resource where those permissions need to be checked. - * @param clazz Permissions enum class. + * @param Permissions enum type. + * @param organizationId Organization id. + * @param studyId study uid. + * @param resourceUid Resource uid. + * @param members members whose permissions will be retrieved. + * @param resource Resource where those permissions need to be checked. + * @param clazz Permissions enum class. + * @param userId user asking for the ACL. * @return the studyAcl for the member. - * @param Permissions enum type. * @throws CatalogException if the user does not have proper permissions to see the member permissions. */ - default > OpenCGAResult> getAcl(String userId, long studyId, long resourceUid, List members, - Enums.Resource resource, Class clazz) throws CatalogException { - return getAcl(userId, studyId, Collections.singletonList(resourceUid), members, resource, clazz); + default > OpenCGAResult> getAcl(String organizationId, long studyId, long resourceUid, + List members, Enums.Resource resource, Class clazz, + String userId) throws CatalogException { + return getAcl(organizationId, studyId, Collections.singletonList(resourceUid), members, resource, clazz, userId); } /** * Return the ACL defined for the member. * - * @param userId user asking for the ACL. - * @param studyId study uid. - * @param resourceUids List of resource uids. - * @param members members whose permissions will be retrieved. - * @param resource Resource where those permissions need to be checked. - * @param clazz Permissions enum class. + * @param Permissions enum type. + * @param organizationId Organization id. + * @param studyId study uid. + * @param resourceUids List of resource uids. + * @param members members whose permissions will be retrieved. + * @param resource Resource where those permissions need to be checked. + * @param clazz Permissions enum class. + * @param userId user asking for the ACL. * @return the studyAcl for the member. - * @param Permissions enum type. * @throws CatalogException if the user does not have proper permissions to see the member permissions. */ - > OpenCGAResult> getAcl(String userId, long studyId, List resourceUids, List members, - Enums.Resource resource, Class clazz) throws CatalogException; + > OpenCGAResult> getAcl(String organizationId, long studyId, List resourceUids, + List members, Enums.Resource resource, Class clazz, String userId) + throws CatalogException; - default > OpenCGAResult> getAcls(long studyUid, long resourceUid, Enums.Resource resource, - Class clazz) throws CatalogException { - return getAcls(studyUid, Collections.singletonList(resourceUid), resource, clazz); + default > OpenCGAResult> getAcls(String organizationId, long studyUid, long resourceUid, + Enums.Resource resource, Class clazz) throws CatalogException { + return getAcls(organizationId, studyUid, Collections.singletonList(resourceUid), resource, clazz); } /** * Return the ACLs of the resources asked. * - * @param userId user asking for the ACL. - * @param studyId study uid. - * @param resourceUids List of resource uid. - * @param resource Resource where those permissions need to be checked. - * @param clazz Permissions enum class. + * @param Permissions enum type. + * @param organizationId Organization id. + * @param studyId study uid. + * @param resourceUids List of resource uid. + * @param resource Resource where those permissions need to be checked. + * @param clazz Permissions enum class. + * @param userId user asking for the ACL. * @return the studyAcl for the member. - * @param Permissions enum type. * @throws CatalogException if the user does not have proper permissions to see the member permissions. */ - > OpenCGAResult> getAcl(String userId, long studyId, List resourceUids, Enums.Resource resource, - Class clazz) throws CatalogException; - - > OpenCGAResult> getAcls(long studyUid, List resourceUids, Enums.Resource resource, - Class clazz) throws CatalogException; + > OpenCGAResult> getAcl(String organizationId, long studyId, List resourceUids, + Enums.Resource resource, Class clazz, String userId) + throws CatalogException; - > OpenCGAResult> getAcls(long studyUid, List resourceUids, List members, + > OpenCGAResult> getAcls(String organizationId, long studyUid, List resourceUids, Enums.Resource resource, Class clazz) throws CatalogException; - void setStudyAcls(List studyIds, List members, List permissions) throws CatalogException; + > OpenCGAResult> getAcls(String organizationId, long studyUid, List resourceUids, + List members, Enums.Resource resource, Class clazz) + throws CatalogException; - void addStudyAcls(List studyIds, List members, List permissions) throws CatalogException; + void setStudyAcls(String organizationId, List studyIds, List members, List permissions) throws CatalogException; - void removeStudyAcls(List studyIds, List members, @Nullable List permissions) throws CatalogException; + void addStudyAcls(String organizationId, List studyIds, List members, List permissions) throws CatalogException; - default void setAcls(long studyUid, List members, CatalogAclParams... aclParams) throws CatalogException { - setAcls(studyUid, members, Arrays.asList(aclParams)); + void removeStudyAcls(String organizationId, List studyIds, List members, @Nullable List permissions) + throws CatalogException; + + default void setAcls(String organizationId, long studyUid, List members, CatalogAclParams... aclParams) + throws CatalogException { + setAcls(organizationId, studyUid, members, Arrays.asList(aclParams)); } - void setAcls(long studyUid, List members, List aclParams) throws CatalogException; + void setAcls(String organizationId, long studyUid, List members, List aclParams) throws CatalogException; - default void addAcls(long studyId, List members, CatalogAclParams... aclParams) throws CatalogException { - addAcls(studyId, members, Arrays.asList(aclParams)); + default void addAcls(String organizationId, long studyId, List members, CatalogAclParams... aclParams) throws CatalogException { + addAcls(organizationId, studyId, members, Arrays.asList(aclParams)); } - void addAcls(long studyId, List members, List aclParams) throws CatalogException; + void addAcls(String organizationId, long studyId, List members, List aclParams) throws CatalogException; - default void removeAcls(List members, CatalogAclParams... aclParams) throws CatalogException { - removeAcls(members, Arrays.asList(aclParams)); + default void removeAcls(String organizationId, List members, CatalogAclParams... aclParams) throws CatalogException { + removeAcls(organizationId, members, Arrays.asList(aclParams)); } - void removeAcls(List members, List aclParams) throws CatalogException; + void removeAcls(String organizationId, List members, List aclParams) throws CatalogException; - void replicateAcls(List uids, AclEntryList aclEntryList, Enums.Resource resource) throws CatalogException; + void replicateAcls(String organizationId, List uids, AclEntryList aclEntryList, Enums.Resource resource) + throws CatalogException; - void resetPermissionsFromAllEntities(long studyId, List members) throws CatalogException; + void resetPermissionsFromAllEntities(String organizationId, long studyId, List members) throws CatalogException; - void applyPermissionRule(long studyId, PermissionRule permissionRule, Enums.Entity entry) throws CatalogException; + void applyPermissionRule(String organizationId, long studyId, PermissionRule permissionRule, Enums.Entity entry) + throws CatalogException; - void removePermissionRuleAndRemovePermissions(Study study, String permissionRuleId, Enums.Entity entry) throws CatalogException; + void removePermissionRuleAndRemovePermissions(String organizationId, Study study, String permissionRuleId, Enums.Entity entry) + throws CatalogException; - void removePermissionRuleAndRestorePermissions(Study study, String permissionRuleId, Enums.Entity entry) throws CatalogException; + void removePermissionRuleAndRestorePermissions(String organizationId, Study study, String permissionRuleId, Enums.Entity entry) + throws CatalogException; - void removePermissionRule(long studyId, String permissionRuleId, Enums.Entity entry) throws CatalogException; + void removePermissionRule(String organizationId, long studyId, String permissionRuleId, Enums.Entity entry) throws CatalogException; class CatalogAclParams { private List ids; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/AuthorizationMongoDBAdaptorFactory.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/AuthorizationMongoDBAdaptorFactory.java new file mode 100644 index 00000000000..4284ed7117c --- /dev/null +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/AuthorizationMongoDBAdaptorFactory.java @@ -0,0 +1,39 @@ +package org.opencb.opencga.catalog.auth.authorization; + +import org.opencb.opencga.catalog.db.mongodb.AuthorizationMongoDBAdaptor; +import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.core.config.Configuration; + +import java.util.HashMap; +import java.util.Map; + +public class AuthorizationMongoDBAdaptorFactory implements AuthorizationDBAdaptorFactory { + + private final MongoDBAdaptorFactory dbAdaptorFactory; + private final Configuration catalogConfiguration; + private final Map authorizationMongoDBAdaptorMap; + + public AuthorizationMongoDBAdaptorFactory(MongoDBAdaptorFactory dbAdaptorFactory, Configuration catalogConfiguration) + throws CatalogDBException { + this.dbAdaptorFactory = dbAdaptorFactory; + this.catalogConfiguration = catalogConfiguration; + this.authorizationMongoDBAdaptorMap = new HashMap<>(); + for (String organizationId : dbAdaptorFactory.getOrganizationIds()) { + OrganizationMongoDBAdaptorFactory orgDbAdaptorFactory = dbAdaptorFactory.getOrganizationMongoDBAdaptorFactory(organizationId); + authorizationMongoDBAdaptorMap.put(organizationId, new AuthorizationMongoDBAdaptor(orgDbAdaptorFactory, catalogConfiguration)); + } + } + + @Override + public AuthorizationMongoDBAdaptor getAuthorizationDBAdaptor(String organization) throws CatalogDBException { + if (authorizationMongoDBAdaptorMap.containsKey(organization)) { + return authorizationMongoDBAdaptorMap.get(organization); + } + OrganizationMongoDBAdaptorFactory orgDBAdaptorFactory = dbAdaptorFactory.getOrganizationMongoDBAdaptorFactory(organization); + authorizationMongoDBAdaptorMap.put(organization, new AuthorizationMongoDBAdaptor(orgDBAdaptorFactory, catalogConfiguration)); + return authorizationMongoDBAdaptorMap.get(organization); + } + +} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/CatalogAuthorizationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/CatalogAuthorizationManager.java index 8a34fc6e17b..7022be4a0e7 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/CatalogAuthorizationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/CatalogAuthorizationManager.java @@ -17,17 +17,21 @@ package org.opencb.opencga.catalog.auth.authorization; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.catalog.db.DBAdaptorFactory; import org.opencb.opencga.catalog.db.api.*; -import org.opencb.opencga.catalog.db.mongodb.AuthorizationMongoDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.exceptions.CatalogParameterException; +import org.opencb.opencga.catalog.managers.OrganizationManager; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.api.ParamConstants; -import org.opencb.opencga.core.config.Configuration; +import org.opencb.opencga.core.models.Acl; import org.opencb.opencga.core.models.AclEntryList; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.clinical.ClinicalAnalysisPermissions; import org.opencb.opencga.core.models.cohort.CohortPermissions; import org.opencb.opencga.core.models.common.Enums; @@ -35,6 +39,7 @@ import org.opencb.opencga.core.models.file.FilePermissions; import org.opencb.opencga.core.models.individual.IndividualPermissions; import org.opencb.opencga.core.models.job.JobPermissions; +import org.opencb.opencga.core.models.organizations.Organization; import org.opencb.opencga.core.models.panel.PanelPermissions; import org.opencb.opencga.core.models.sample.SamplePermissions; import org.opencb.opencga.core.models.study.Group; @@ -57,57 +62,65 @@ public class CatalogAuthorizationManager implements AuthorizationManager { public static final String MEMBERS_GROUP = ParamConstants.MEMBERS_GROUP; public static final String ADMINS_GROUP = ParamConstants.ADMINS_GROUP; - private static final String OPENCGA = ParamConstants.OPENCGA_USER_ID; private final Logger logger; - private final ProjectDBAdaptor projectDBAdaptor; - private final StudyDBAdaptor studyDBAdaptor; - private final FileDBAdaptor fileDBAdaptor; - private final JobDBAdaptor jobDBAdaptor; - private final SampleDBAdaptor sampleDBAdaptor; - private final IndividualDBAdaptor individualDBAdaptor; - private final CohortDBAdaptor cohortDBAdaptor; - private final PanelDBAdaptor panelDBAdaptor; - private final FamilyDBAdaptor familyDBAdaptor; - private final ClinicalAnalysisDBAdaptor clinicalAnalysisDBAdaptor; + private final DBAdaptorFactory dbAdaptorFactory; + private final AuthorizationDBAdaptorFactory authorizationDBAdaptorFactory; - private final AuthorizationDBAdaptor aclDBAdaptor; - - public CatalogAuthorizationManager(DBAdaptorFactory dbFactory, Configuration configuration) + public CatalogAuthorizationManager(DBAdaptorFactory dbFactory, AuthorizationDBAdaptorFactory authorizationDBAdaptorFactory) throws CatalogDBException { this.logger = LoggerFactory.getLogger(CatalogAuthorizationManager.class); - this.aclDBAdaptor = new AuthorizationMongoDBAdaptor(dbFactory, configuration); - - projectDBAdaptor = dbFactory.getCatalogProjectDbAdaptor(); - studyDBAdaptor = dbFactory.getCatalogStudyDBAdaptor(); - fileDBAdaptor = dbFactory.getCatalogFileDBAdaptor(); - jobDBAdaptor = dbFactory.getCatalogJobDBAdaptor(); - sampleDBAdaptor = dbFactory.getCatalogSampleDBAdaptor(); - individualDBAdaptor = dbFactory.getCatalogIndividualDBAdaptor(); - cohortDBAdaptor = dbFactory.getCatalogCohortDBAdaptor(); - panelDBAdaptor = dbFactory.getCatalogPanelDBAdaptor(); - familyDBAdaptor = dbFactory.getCatalogFamilyDBAdaptor(); - clinicalAnalysisDBAdaptor = dbFactory.getClinicalAnalysisDBAdaptor(); + this.dbAdaptorFactory = dbFactory; + this.authorizationDBAdaptorFactory = authorizationDBAdaptorFactory; } @Override - public void checkCanEditProject(long projectId, String userId) throws CatalogException { - if (isInstallationAdministrator(userId)) { + public void checkCanEditProject(String organizationId, long projectId, String userId) throws CatalogException { + if (isAtLeastOrganizationOwnerOrAdmin(organizationId, userId)) { return; } - if (projectDBAdaptor.getOwnerId(projectId).equals(userId)) { - return; + try { + checkUserIsStudyAdminInAllStudiesOfProject(organizationId, projectId, userId); + } catch (CatalogException e) { + throw new CatalogAuthorizationException("Permission denied: Only the organization owner, administrators, or users who are " + + "study administrators in all studies within the project can update the project.", e); + } + } + + private void checkUserIsStudyAdminInAllStudiesOfProject(String organizationId, long projectUid, String userId) throws CatalogException { + Query query = new Query(StudyDBAdaptor.QueryParams.PROJECT_UID.key(), projectUid); + QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, + Arrays.asList(StudyDBAdaptor.QueryParams.GROUPS.key(), StudyDBAdaptor.QueryParams.FQN.key())); + OpenCGAResult studyResult = dbAdaptorFactory.getCatalogStudyDBAdaptor(organizationId).get(query, options); + List nonAdminStudyIds = new ArrayList<>(); + for (Study study : studyResult.getResults()) { + for (Group group : study.getGroups()) { + if (group.getId().equals(ADMINS_GROUP)) { + if (!group.getUserIds().contains(userId)) { + nonAdminStudyIds.add(study.getFqn()); + } + break; + } + } + } + if (!nonAdminStudyIds.isEmpty()) { + throw new CatalogAuthorizationException("Permission denied: User " + userId + " is not an administrator of the following" + + " studies: " + String.join(", ", nonAdminStudyIds)); } - throw new CatalogAuthorizationException("Permission denied: Only the owner of the project can update it."); } @Override - public void checkCanViewProject(long projectId, String userId) throws CatalogException { - if (isInstallationAdministrator(userId)) { + public void checkCanViewOrganization(String organizationId, String userId) throws CatalogAuthorizationException { + if (isOpencgaAdministrator(organizationId, userId)) { return; } - if (projectDBAdaptor.getOwnerId(projectId).equals(userId)) { + checkUserExists(organizationId, userId); + } + + @Override + public void checkCanViewProject(String organizationId, long projectId, String userId) throws CatalogException { + if (isAtLeastOrganizationOwnerOrAdmin(organizationId, userId)) { return; } @@ -116,7 +129,7 @@ public void checkCanViewProject(long projectId, String userId) throws CatalogExc .append(StudyDBAdaptor.QueryParams.PROJECT_ID.key(), projectId) .append(StudyDBAdaptor.QueryParams.GROUP_USER_IDS.key(), userId); - if (studyDBAdaptor.count(query).getNumMatches() > 0) { + if (dbAdaptorFactory.getCatalogStudyDBAdaptor(organizationId).count(query).getNumMatches() > 0) { return; } @@ -124,110 +137,87 @@ public void checkCanViewProject(long projectId, String userId) throws CatalogExc } @Override - public void checkStudyPermission(long studyId, String userId, StudyPermissions.Permissions permission) throws CatalogException { - checkStudyPermission(studyId, userId, permission, permission.toString()); - } - - @Override - public void checkStudyPermission(long studyId, String userId, StudyPermissions.Permissions permission, String message) + public void checkStudyPermission(String organizationId, long studyUid, JwtPayload payload, StudyPermissions.Permissions permission) throws CatalogException { - if (isInstallationAdministrator(userId)) { + String userId = payload.getUserId(organizationId); + if (isOpencgaAdministrator(organizationId, userId)) { return; } else { - if (studyDBAdaptor.hasStudyPermission(studyId, userId, permission)) { + if (dbAdaptorFactory.getCatalogStudyDBAdaptor(organizationId).hasStudyPermission(studyUid, userId, permission)) { return; } } - throw CatalogAuthorizationException.deny(userId, message, "Study", studyId, null); + throw CatalogAuthorizationException.deny(userId, permission.name(), "Study", studyUid, null); } @Override - public void checkCanEditStudy(long studyId, String userId) throws CatalogException { - if (isInstallationAdministrator(userId)) { + public void checkStudyPermission(String organizationId, long studyId, String userId, StudyPermissions.Permissions permission) + throws CatalogException { + if (isAtLeastOrganizationOwnerOrAdmin(organizationId, userId)) { return; + } else { + if (dbAdaptorFactory.getCatalogStudyDBAdaptor(organizationId).hasStudyPermission(studyId, userId, permission)) { + return; + } } - - String ownerId = studyDBAdaptor.getOwnerId(studyId); - if (!ownerId.equals(userId) && !isAdministrativeUser(studyId, userId)) { - throw new CatalogAuthorizationException("Only owners or administrative users are allowed to modify a study"); - } + throw CatalogAuthorizationException.deny(userId, permission.name(), "Study", studyId, null); } @Override - public void checkCanViewStudy(long studyId, String userId) throws CatalogException { - if (isInstallationAdministrator(userId)) { - return; + public void checkCanEditStudy(String organizationId, long studyId, String userId) throws CatalogException { + if (!this.isAtLeastStudyAdministrator(organizationId, studyId, userId)) { + throw CatalogAuthorizationException.notStudyAdmin("modify a study"); } + } - String ownerId = studyDBAdaptor.getOwnerId(studyId); - if (ownerId.equals(userId)) { + @Override + public void checkCanViewStudy(String organizationId, long studyId, String userId) throws CatalogException { + if (isOpencgaAdministrator(organizationId, userId)) { return; } - OpenCGAResult groupBelonging = getGroupBelonging(studyId, userId); + OpenCGAResult groupBelonging = getGroupBelonging(organizationId, studyId, userId); if (groupBelonging.getNumResults() == 0) { - throw new CatalogAuthorizationException("Only the members of the study are allowed to see it"); + throw CatalogAuthorizationException.notStudyMember("see it"); } } @Override - public void checkCanUpdatePermissionRules(long studyId, String userId) throws CatalogException { - if (isInstallationAdministrator(userId)) { - return; - } - - String ownerId = studyDBAdaptor.getOwnerId(studyId); - if (!ownerId.equals(userId) && !isAdministrativeUser(studyId, userId)) { - throw new CatalogAuthorizationException("Only owners or administrative users are allowed to modify a update permission rules"); + public void checkCanUpdatePermissionRules(String organizationId, long studyId, String userId) throws CatalogException { + if (!isAtLeastStudyAdministrator(organizationId, studyId, userId)) { + throw CatalogAuthorizationException.notStudyAdmin("update the permission rules"); } } @Override - public void checkCreateDeleteGroupPermissions(long studyId, String userId, String group) throws CatalogException { + public void checkCreateDeleteGroupPermissions(String organizationId, long studyId, String userId, String group) + throws CatalogException { if (group.equals(MEMBERS_GROUP) || group.equals(ADMINS_GROUP)) { throw new CatalogAuthorizationException(group + " is a protected group that cannot be created or deleted."); } - if (isInstallationAdministrator(userId)) { - return; - } - - String ownerId = studyDBAdaptor.getOwnerId(studyId); - if (!userId.equals(ownerId) && !isAdministrativeUser(studyId, userId)) { - throw new CatalogAuthorizationException("Only administrative users are allowed to create/remove groups."); + if (!isAtLeastStudyAdministrator(organizationId, studyId, userId)) { + throw CatalogAuthorizationException.notStudyAdmin("create or remove groups."); } } @Override - public void checkSyncGroupPermissions(long studyId, String userId, String group) throws CatalogException { - checkCreateDeleteGroupPermissions(studyId, userId, group); + public void checkSyncGroupPermissions(String organizationId, long studyUid, String userId, String group) throws CatalogException { + checkCreateDeleteGroupPermissions(organizationId, studyUid, userId, group); } @Override - public void checkUpdateGroupPermissions(long studyId, String userId, String group, ParamUtils.BasicUpdateAction action) - throws CatalogException { - String ownerId = studyDBAdaptor.getOwnerId(studyId); - - if (userId.equals(ownerId)) { - // Granted permission but check it is a valid action - if (group.equals(MEMBERS_GROUP) - && (action != ParamUtils.BasicUpdateAction.ADD && action != ParamUtils.BasicUpdateAction.REMOVE)) { - throw new CatalogAuthorizationException("Only ADD or REMOVE actions are accepted for @members group."); - } - return; + public void checkUpdateGroupPermissions(String organizationId, long studyId, String userId, String group, + ParamUtils.BasicUpdateAction action) throws CatalogException { + if (MEMBERS_GROUP.equals(group) + && (action != ParamUtils.BasicUpdateAction.ADD && action != ParamUtils.BasicUpdateAction.REMOVE)) { + throw new CatalogAuthorizationException("Only ADD or REMOVE actions are accepted for " + MEMBERS_GROUP + " group."); } - - if (group.equals(ADMINS_GROUP)) { - throw new CatalogAuthorizationException("Only the owner of the study can assign/remove users to the administrative group."); - } - - if (!isInstallationAdministrator(userId) && !isAdministrativeUser(studyId, userId)) { - throw new CatalogAuthorizationException("Only administrative users are allowed to assign/remove users to groups."); + if (ADMINS_GROUP.equals(group) && !isAtLeastOrganizationOwnerOrAdmin(organizationId, userId)) { + throw CatalogAuthorizationException.notOrganizationOwnerOrAdmin("assign or remove users to the " + ADMINS_GROUP + " group."); } - - // Check it is a valid action - if (group.equals(MEMBERS_GROUP) && (action != ParamUtils.BasicUpdateAction.ADD && action != ParamUtils.BasicUpdateAction.REMOVE)) { - throw new CatalogAuthorizationException("Only ADD or REMOVE actions are accepted for @members group."); + if (!isAtLeastStudyAdministrator(organizationId, studyId, userId)) { + throw CatalogAuthorizationException.notStudyAdmin("assign or remove users to groups."); } } @@ -241,67 +231,84 @@ public void checkNotAssigningPermissionsToAdminsGroup(List members) thro } @Override - public void checkCanAssignOrSeePermissions(long studyId, String userId) throws CatalogException { - if (isInstallationAdministrator(userId)) { - return; + public void checkCanAssignOrSeePermissions(String organizationId, long studyId, String userId) throws CatalogException { + if (!isAtLeastStudyAdministrator(organizationId, studyId, userId)) { + throw CatalogAuthorizationException.notStudyAdmin("assign or see all permissions"); } + } - String ownerId = studyDBAdaptor.getOwnerId(studyId); - if (!ownerId.equals(userId) && !isAdministrativeUser(studyId, userId)) { - throw new CatalogAuthorizationException("Only owners or administrative users are allowed to assign or see all permissions"); + @Override + public void checkCanCreateUpdateDeleteVariableSets(String organizationId, long studyId, String userId) throws CatalogException { + if (!isAtLeastOrganizationOwnerOrAdmin(organizationId, userId)) { + throw CatalogAuthorizationException.notOrganizationOwnerOrAdmin("create, update or delete variable sets."); } } @Override - public void checkCanCreateUpdateDeleteVariableSets(long studyId, String userId) throws CatalogException { - if (isInstallationAdministrator(userId)) { - return; + public boolean isOpencgaAdministrator(String organization, String userId) throws CatalogAuthorizationException { + if (ParamConstants.ADMIN_ORGANIZATION.equals(organization) || userId.startsWith(ParamConstants.ADMIN_ORGANIZATION + ":")) { + // Check user exists in ADMIN ORGANIZATION + String user = userId.replace(ParamConstants.ADMIN_ORGANIZATION + ":", ""); + checkUserExists(ParamConstants.ADMIN_ORGANIZATION, user); + return true; } + return false; + } - String ownerId = studyDBAdaptor.getOwnerId(studyId); - - if (!ownerId.equals(userId) && !isAdministrativeUser(studyId, userId)) { - throw new CatalogAuthorizationException("Only owners or administrative users are allowed to create/update/delete variable " - + "sets"); + @Override + public void checkIsOpencgaAdministrator(String organizationId, String userId, String action) throws CatalogException { + if (!isOpencgaAdministrator(organizationId, userId)) { + throw CatalogAuthorizationException.opencgaAdminOnlySupportedOperation(action); } } @Override - public Boolean isInstallationAdministrator(String user) { - return OPENCGA.equals(user); + public void checkIsAtLeastOrganizationOwner(String organizationId, String userId) throws CatalogException { + if (!isAtLeastOrganizationOwner(organizationId, userId)) { + throw CatalogAuthorizationException.notOrganizationOwner(); + } } @Override - public void checkIsInstallationAdministrator(String user) throws CatalogException { - if (!isInstallationAdministrator(user)) { - throw new CatalogAuthorizationException("Only ADMINISTRATOR users are allowed to perform this action"); + public void checkIsAtLeastOrganizationOwnerOrAdmin(String organizationId, String userId) throws CatalogException { + if (!isAtLeastOrganizationOwnerOrAdmin(organizationId, userId)) { + throw CatalogAuthorizationException.notOrganizationOwnerOrAdmin(); } } @Override - public void checkIsOwnerOrAdmin(long studyId, String userId) throws CatalogException { - if (isInstallationAdministrator(userId)) { - return; + public boolean isAtLeastOrganizationOwner(String organizationId, String userId) throws CatalogException { + if (isOpencgaAdministrator(organizationId, userId)) { + return true; } + OrganizationDBAdaptor organizationDBAdaptor = dbAdaptorFactory.getCatalogOrganizationDBAdaptor(organizationId); + Organization organization = organizationDBAdaptor.get(OrganizationManager.INCLUDE_ORGANIZATION_ADMINS).first(); + return organization.getOwner().equals(userId); + } - if (!isOwnerOrAdmin(studyId, userId)) { - throw new CatalogAuthorizationException("Only owners or administrative users are allowed to perform this action"); + @Override + public boolean isAtLeastOrganizationOwnerOrAdmin(String organizationId, String userId) throws CatalogException { + if (isOpencgaAdministrator(organizationId, userId)) { + return true; } + OrganizationDBAdaptor organizationDBAdaptor = dbAdaptorFactory.getCatalogOrganizationDBAdaptor(organizationId); + Organization organization = organizationDBAdaptor.get(OrganizationManager.INCLUDE_ORGANIZATION_ADMINS).first(); + return userId.equals(organization.getOwner()) || organization.getAdmins().contains(userId); } @Override - public Boolean isOwnerOrAdmin(long studyId, String userId) throws CatalogException { - String ownerId = studyDBAdaptor.getOwnerId(studyId); - - if (!ownerId.equals(userId) && !isAdministrativeUser(studyId, userId)) { - return false; + public void checkIsAtLeastStudyAdministrator(String organizationId, long studyId, String userId) throws CatalogException { + if (!isAtLeastStudyAdministrator(organizationId, studyId, userId)) { + throw CatalogAuthorizationException.notStudyAdmin("perform this action"); } - return true; } - - private boolean isAdministrativeUser(long studyId, String user) throws CatalogException { - OpenCGAResult groupBelonging = getGroupBelonging(studyId, user); + @Override + public boolean isAtLeastStudyAdministrator(String organizationId, long studyId, String user) throws CatalogException { + if (isAtLeastOrganizationOwnerOrAdmin(organizationId, user)) { + return true; + } + OpenCGAResult groupBelonging = getGroupBelonging(organizationId, studyId, user); for (Group group : groupBelonging.getResults()) { if (group.getId().equals(ADMINS_GROUP)) { return true; @@ -311,21 +318,22 @@ private boolean isAdministrativeUser(long studyId, String user) throws CatalogEx } @Override - public void checkFilePermission(long studyId, long fileId, String userId, FilePermissions permission) + public void checkFilePermission(String organizationId, long studyId, long fileId, String userId, FilePermissions permission) throws CatalogException { Query query = new Query() .append(FileDBAdaptor.QueryParams.UID.key(), fileId) .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), studyId) .append(ParamConstants.ACL_PARAM, userId + ":" + permission.name()); - if (checkUserPermission(userId, query, fileDBAdaptor)) { + if (checkUserPermission(organizationId, userId, query, dbAdaptorFactory.getCatalogFileDBAdaptor(organizationId))) { return; } throw CatalogAuthorizationException.deny(userId, permission.toString(), "File", fileId, null); } - private boolean checkUserPermission(String userId, Query query, CoreDBAdaptor dbAdaptor) throws CatalogException { - if (isInstallationAdministrator(userId)) { + private boolean checkUserPermission(String organizationId, String userId, Query query, CoreDBAdaptor dbAdaptor) + throws CatalogException { + if (isAtLeastOrganizationOwnerOrAdmin(organizationId, userId)) { return true; } else { if (dbAdaptor.count(query, userId).getNumMatches() == 1) { @@ -336,55 +344,56 @@ private boolean checkUserPermission(String userId, Query query, CoreDBAdaptor db } @Override - public void checkSamplePermission(long studyId, long sampleId, String userId, SamplePermissions permission) + public void checkSamplePermission(String organizationId, long studyId, long sampleId, String userId, SamplePermissions permission) throws CatalogException { Query query = new Query() .append(SampleDBAdaptor.QueryParams.UID.key(), sampleId) .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), studyId) .append(ParamConstants.ACL_PARAM, userId + ":" + permission.name()); - if (checkUserPermission(userId, query, sampleDBAdaptor)) { + if (checkUserPermission(organizationId, userId, query, dbAdaptorFactory.getCatalogSampleDBAdaptor(organizationId))) { return; } throw CatalogAuthorizationException.deny(userId, permission.toString(), "Sample", sampleId, null); } @Override - public void checkIndividualPermission(long studyId, long individualId, String userId, + public void checkIndividualPermission(String organizationId, long studyId, long individualId, String userId, IndividualPermissions permission) throws CatalogException { Query query = new Query() .append(IndividualDBAdaptor.QueryParams.UID.key(), individualId) .append(IndividualDBAdaptor.QueryParams.STUDY_UID.key(), studyId) .append(ParamConstants.ACL_PARAM, userId + ":" + permission.name()); - if (checkUserPermission(userId, query, individualDBAdaptor)) { + if (checkUserPermission(organizationId, userId, query, dbAdaptorFactory.getCatalogIndividualDBAdaptor(organizationId))) { return; } throw CatalogAuthorizationException.deny(userId, permission.toString(), "Individual", individualId, null); } @Override - public void checkJobPermission(long studyId, long jobId, String userId, JobPermissions permission) throws CatalogException { + public void checkJobPermission(String organizationId, long studyId, long jobId, String userId, JobPermissions permission) + throws CatalogException { Query query = new Query() .append(JobDBAdaptor.QueryParams.UID.key(), jobId) .append(JobDBAdaptor.QueryParams.STUDY_UID.key(), studyId) .append(ParamConstants.ACL_PARAM, userId + ":" + permission.name()); - if (checkUserPermission(userId, query, jobDBAdaptor)) { + if (checkUserPermission(organizationId, userId, query, dbAdaptorFactory.getCatalogJobDBAdaptor(organizationId))) { return; } throw CatalogAuthorizationException.deny(userId, permission.toString(), "Job", jobId, null); } @Override - public void checkCohortPermission(long studyId, long cohortId, String userId, CohortPermissions permission) + public void checkCohortPermission(String organizationId, long studyId, long cohortId, String userId, CohortPermissions permission) throws CatalogException { Query query = new Query() .append(CohortDBAdaptor.QueryParams.UID.key(), cohortId) .append(CohortDBAdaptor.QueryParams.STUDY_UID.key(), studyId) .append(ParamConstants.ACL_PARAM, userId + ":" + permission.name()); - if (checkUserPermission(userId, query, cohortDBAdaptor)) { + if (checkUserPermission(organizationId, userId, query, dbAdaptorFactory.getCatalogCohortDBAdaptor(organizationId))) { return; } throw CatalogAuthorizationException.deny(userId, permission.toString(), "Cohort", cohortId, null); @@ -392,28 +401,28 @@ public void checkCohortPermission(long studyId, long cohortId, String userId, Co } @Override - public void checkPanelPermission(long studyId, long panelId, String userId, PanelPermissions permission) + public void checkPanelPermission(String organizationId, long studyId, long panelId, String userId, PanelPermissions permission) throws CatalogException { Query query = new Query() .append(PanelDBAdaptor.QueryParams.UID.key(), panelId) .append(PanelDBAdaptor.QueryParams.STUDY_UID.key(), studyId) .append(ParamConstants.ACL_PARAM, userId + ":" + permission.name()); - if (checkUserPermission(userId, query, panelDBAdaptor)) { + if (checkUserPermission(organizationId, userId, query, dbAdaptorFactory.getCatalogPanelDBAdaptor(organizationId))) { return; } throw CatalogAuthorizationException.deny(userId, permission.toString(), "Panel", panelId, null); } @Override - public void checkFamilyPermission(long studyId, long familyId, String userId, FamilyPermissions permission) + public void checkFamilyPermission(String organizationId, long studyId, long familyId, String userId, FamilyPermissions permission) throws CatalogException { Query query = new Query() .append(FamilyDBAdaptor.QueryParams.UID.key(), familyId) .append(FamilyDBAdaptor.QueryParams.STUDY_UID.key(), studyId) .append(ParamConstants.ACL_PARAM, userId + ":" + permission.name()); - if (checkUserPermission(userId, query, familyDBAdaptor)) { + if (checkUserPermission(organizationId, userId, query, dbAdaptorFactory.getCatalogFamilyDBAdaptor(organizationId))) { return; } throw CatalogAuthorizationException.deny(userId, permission.toString(), "Family", familyId, null); @@ -421,91 +430,128 @@ public void checkFamilyPermission(long studyId, long familyId, String userId, Fa } @Override - public void checkClinicalAnalysisPermission(long studyId, long analysisId, String userId, + public void checkClinicalAnalysisPermission(String organizationId, long studyId, long analysisId, String userId, ClinicalAnalysisPermissions permission) throws CatalogException { Query query = new Query() .append(ClinicalAnalysisDBAdaptor.QueryParams.UID.key(), analysisId) .append(ClinicalAnalysisDBAdaptor.QueryParams.STUDY_UID.key(), studyId) .append(ParamConstants.ACL_PARAM, userId + ":" + permission.name()); - if (checkUserPermission(userId, query, clinicalAnalysisDBAdaptor)) { + if (checkUserPermission(organizationId, userId, query, dbAdaptorFactory.getClinicalAnalysisDBAdaptor(organizationId))) { return; } throw CatalogAuthorizationException.deny(userId, permission.toString(), "ClinicalAnalysis", analysisId, null); } @Override - public OpenCGAResult> getAllStudyAcls(String userId, long studyId) - throws CatalogException { - checkCanAssignOrSeePermissions(studyId, userId); - return aclDBAdaptor.get(studyId, null, null, Enums.Resource.STUDY, StudyPermissions.Permissions.class); + public List getEffectivePermissions(String organizationId, long studyUid, List resourceIdList, List permissions, + Enums.Resource resource) throws CatalogException { + HashSet resourcePermissions = new HashSet<>(resource.getFullPermissionList()); + for (String permission : permissions) { + if (!resourcePermissions.contains(permission)) { + throw new CatalogParameterException("Permission '" + permission + "' does not correspond with any of the available" + + " permissions. This is the full list of possible permissions: " + StringUtils.join(resourcePermissions, ", ")); + } + } + + List acls = authorizationDBAdaptorFactory.getAuthorizationDBAdaptor(organizationId) + .effectivePermissions(studyUid, resourceIdList, resource); + if (CollectionUtils.isNotEmpty(permissions)) { + // Filter out other permissions + for (Acl acl : acls) { + List permissionList = new ArrayList<>(permissions.size()); + for (Acl.Permission aclPermission : acl.getPermissions()) { + if (permissions.contains(aclPermission.getId())) { + permissionList.add(aclPermission); + } + } + acl.setPermissions(permissionList); + } + } + return acls; } @Override - public OpenCGAResult> getStudyAcl(String userId, long studyUid, List members) + public OpenCGAResult> getAllStudyAcls(String organizationId, long studyId, String userId) throws CatalogException { - checkCanSeePermissions(userId, studyUid, members); - Map> userGroups = extractUserGroups(studyUid, members); - return aclDBAdaptor.get(studyUid, members, userGroups, Enums.Resource.STUDY, StudyPermissions.Permissions.class); + checkCanAssignOrSeePermissions(organizationId, studyId, userId); + return authorizationDBAdaptorFactory.getAuthorizationDBAdaptor(organizationId) + .get(studyId, null, null, Enums.Resource.STUDY, StudyPermissions.Permissions.class); } @Override - public OpenCGAResult> getStudyAcl(List studyUids, List members) - throws CatalogException { + public OpenCGAResult> getStudyAcl(String organizationId, long studyUid, List members, + String userId) throws CatalogException { + checkCanSeePermissions(organizationId, studyUid, members, userId); + Map> userGroups = extractUserGroups(organizationId, studyUid, members); + return authorizationDBAdaptorFactory.getAuthorizationDBAdaptor(organizationId) + .get(studyUid, members, userGroups, Enums.Resource.STUDY, StudyPermissions.Permissions.class); + } + + @Override + public OpenCGAResult> getStudyAcl(String organizationId, List studyUids, + List members) throws CatalogException { OpenCGAResult> result = OpenCGAResult.empty(); for (Long studyUid : studyUids) { - Map> userGroups = extractUserGroups(studyUid, members); - result.append(aclDBAdaptor.get(studyUid, members, userGroups, Enums.Resource.STUDY, StudyPermissions.Permissions.class)); + Map> userGroups = extractUserGroups(organizationId, studyUid, members); + result.append(authorizationDBAdaptorFactory.getAuthorizationDBAdaptor(organizationId) + .get(studyUid, members, userGroups, Enums.Resource.STUDY, StudyPermissions.Permissions.class)); } return result; } @Override - public > OpenCGAResult> getAcl(String userId, long studyUid, List resourceUids, - List members, Enums.Resource resource, Class clazz) - throws CatalogException { - checkCanSeePermissions(userId, studyUid, members); - Map> userGroups = extractUserGroups(studyUid, members); - return aclDBAdaptor.get(resourceUids, members, userGroups, resource, clazz); + public > OpenCGAResult> getAcl(String organizationId, long studyUid, List resourceUids, + List members, Enums.Resource resource, Class clazz, + String userId) throws CatalogException { + checkCanSeePermissions(organizationId, studyUid, members, userId); + Map> userGroups = extractUserGroups(organizationId, studyUid, members); + return authorizationDBAdaptorFactory.getAuthorizationDBAdaptor(organizationId) + .get(resourceUids, members, userGroups, resource, clazz); } @Override - public > OpenCGAResult> getAcl(String userId, long studyUid, List resourceUids, - Enums.Resource resource, Class clazz) throws CatalogException { - checkCanAssignOrSeePermissions(studyUid, userId); - return aclDBAdaptor.get(resourceUids, null, null, resource, clazz); + public > OpenCGAResult> getAcl(String organizationId, long studyUid, List resourceUids, + Enums.Resource resource, Class clazz, String userId) + throws CatalogException { + checkCanAssignOrSeePermissions(organizationId, studyUid, userId); + return authorizationDBAdaptorFactory.getAuthorizationDBAdaptor(organizationId) + .get(resourceUids, null, null, resource, clazz); } @Override - public > OpenCGAResult> getAcls(long studyUid, List resourceUids, Enums.Resource resource, - Class clazz) throws CatalogException { - return aclDBAdaptor.get(resourceUids, null, null, resource, clazz); + public > OpenCGAResult> getAcls(String organizationId, long studyUid, List resourceUids, + Enums.Resource resource, Class clazz) throws CatalogException { + return authorizationDBAdaptorFactory.getAuthorizationDBAdaptor(organizationId) + .get(resourceUids, null, null, resource, clazz); } @Override - public > OpenCGAResult> getAcls(long studyUid, List resourceUids, List members, - Enums.Resource resource, Class clazz) throws CatalogException { - Map> userGroups = extractUserGroups(studyUid, members); - return aclDBAdaptor.get(resourceUids, members, userGroups, resource, clazz); + public > OpenCGAResult> getAcls(String organizationId, long studyUid, List resourceUids, + List members, Enums.Resource resource, Class clazz) + throws CatalogException { + Map> userGroups = extractUserGroups(organizationId, studyUid, members); + return authorizationDBAdaptorFactory.getAuthorizationDBAdaptor(organizationId) + .get(resourceUids, members, userGroups, resource, clazz); } - private void checkCanSeePermissions(String userId, long studyId, List members) throws CatalogException { + private void checkCanSeePermissions(String organizationId, long studyId, List members, String userId) throws CatalogException { try { - checkCanAssignOrSeePermissions(studyId, userId); + checkCanAssignOrSeePermissions(organizationId, studyId, userId); } catch (CatalogException e) { // It will be OK if the userId asking for the ACLs wants to see its own permissions for (String member : members) { - checkAskingOwnPermissions(userId, member, studyId); + checkAskingOwnPermissions(organizationId, studyId, member, userId); } } } - private Map> extractUserGroups(long studyId, List members) throws CatalogException { + private Map> extractUserGroups(String organizationId, long studyId, List members) throws CatalogException { Map> userGroups = new HashMap<>(); List userList = members.stream().filter(m -> !m.startsWith("@")).collect(Collectors.toList()); if (!userList.isEmpty()) { // If member is a user, we will also store the groups the user might belong to fetch those permissions as well - OpenCGAResult groups = getGroupBelonging(studyId, userList); + OpenCGAResult groups = getGroupBelonging(organizationId, studyId, userList); // Fill in map with the results for (String user : userList) { @@ -523,14 +569,15 @@ private Map> extractUserGroups(long studyId, List m } @Override - public void resetPermissionsFromAllEntities(long studyId, List members) throws CatalogException { - aclDBAdaptor.resetMembersFromAllEntries(studyId, members); + public void resetPermissionsFromAllEntities(String organizationId, long studyId, List members) throws CatalogException { + authorizationDBAdaptorFactory.getAuthorizationDBAdaptor(organizationId) + .resetMembersFromAllEntries(studyId, members); } - private void checkAskingOwnPermissions(String userId, String member, long studyId) throws CatalogException { + private void checkAskingOwnPermissions(String organizationId, long studyId, String member, String userId) throws CatalogException { if (member.startsWith("@")) { //group // If the userId does not belong to the group... - OpenCGAResult groupsBelonging = getGroupBelonging(studyId, userId); + OpenCGAResult groupsBelonging = getGroupBelonging(organizationId, studyId, userId); for (Group group : groupsBelonging.getResults()) { if (group.getId().equals(member)) { return; @@ -548,36 +595,44 @@ private void checkAskingOwnPermissions(String userId, String member, long studyI } @Override - public void setStudyAcls(List studyIds, List members, List permissions) throws CatalogException { - aclDBAdaptor.setToMembers(studyIds, members, getImplicitPermissions(permissions, Enums.Resource.STUDY)); + public void setStudyAcls(String organizationId, List studyIds, List members, List permissions) + throws CatalogException { + authorizationDBAdaptorFactory.getAuthorizationDBAdaptor(organizationId) + .setToMembers(studyIds, members, getImplicitPermissions(permissions, Enums.Resource.STUDY)); } @Override - public void addStudyAcls(List studyIds, List members, List permissions) throws CatalogException { - aclDBAdaptor.addToMembers(studyIds, members, getImplicitPermissions(permissions, Enums.Resource.STUDY)); + public void addStudyAcls(String organizationId, List studyIds, List members, List permissions) + throws CatalogException { + authorizationDBAdaptorFactory.getAuthorizationDBAdaptor(organizationId) + .addToMembers(studyIds, members, getImplicitPermissions(permissions, Enums.Resource.STUDY)); } @Override - public void removeStudyAcls(List studyIds, List members, @Nullable List permissions) throws CatalogException { - removeAcls(members, new CatalogAclParams(studyIds, permissions, Enums.Resource.STUDY)); + public void removeStudyAcls(String organizationId, List studyIds, List members, @Nullable List permissions) + throws CatalogException { + removeAcls(organizationId, members, new CatalogAclParams(studyIds, permissions, Enums.Resource.STUDY)); } @Override - public void setAcls(long studyUid, List members, List aclParams) throws CatalogException { + public void setAcls(String organizationId, long studyUid, List members, List aclParams) + throws CatalogException { setImplicitPermissions(aclParams); - aclDBAdaptor.setToMembers(studyUid, members, aclParams); + authorizationDBAdaptorFactory.getAuthorizationDBAdaptor(organizationId) + .setToMembers(studyUid, members, aclParams); } @Override - public void addAcls(long studyId, List members, List aclParams) throws CatalogException { + public void addAcls(String organizationId, long studyId, List members, List aclParams) + throws CatalogException { setImplicitPermissions(aclParams); - aclDBAdaptor.addToMembers(studyId, members, aclParams); + authorizationDBAdaptorFactory.getAuthorizationDBAdaptor(organizationId).addToMembers(studyId, members, aclParams); } @Override - public void removeAcls(List members, List aclParams) throws CatalogException { + public void removeAcls(String organizationId, List members, List aclParams) throws CatalogException { setDependentPermissions(aclParams); - aclDBAdaptor.removeFromMembers(members, aclParams); + authorizationDBAdaptorFactory.getAuthorizationDBAdaptor(organizationId).removeFromMembers(members, aclParams); } private void setDependentPermissions(List aclParams) throws CatalogAuthorizationException { @@ -920,55 +975,62 @@ private List getImplicitPermissions(List permissions, Enums.Reso } @Override - public void replicateAcls(List uids, AclEntryList aclEntryList, Enums.Resource resource) throws CatalogException { + public void replicateAcls(String organizationId, List uids, AclEntryList aclEntryList, Enums.Resource resource) + throws CatalogException { if (CollectionUtils.isEmpty(uids)) { throw new CatalogDBException("Missing identifiers to set acls"); } if (CollectionUtils.isEmpty(aclEntryList.getAcl())) { return; } - aclDBAdaptor.setAcls(uids, aclEntryList, resource); + authorizationDBAdaptorFactory.getAuthorizationDBAdaptor(organizationId).setAcls(uids, aclEntryList, resource); } @Override - public void applyPermissionRule(long studyId, PermissionRule permissionRule, Enums.Entity entry) throws CatalogException { + public void applyPermissionRule(String organizationId, long studyId, PermissionRule permissionRule, Enums.Entity entry) + throws CatalogException { // 1. We obtain which of those members are actually users to add them to the @members group automatically List userList = permissionRule.getMembers().stream() .filter(member -> !member.startsWith("@")) .collect(Collectors.toList()); if (CollectionUtils.isNotEmpty(userList)) { // We first add the member to the @members group in case they didn't belong already - studyDBAdaptor.addUsersToGroup(studyId, MEMBERS_GROUP, userList); + dbAdaptorFactory.getCatalogStudyDBAdaptor(organizationId).addUsersToGroup(studyId, MEMBERS_GROUP, userList); } // 2. We can apply the permission rules - aclDBAdaptor.applyPermissionRules(studyId, permissionRule, entry); + authorizationDBAdaptorFactory.getAuthorizationDBAdaptor(organizationId) + .applyPermissionRules(studyId, permissionRule, entry); } @Override - public void removePermissionRuleAndRemovePermissions(Study study, String permissionRuleId, Enums.Entity entry) + public void removePermissionRuleAndRemovePermissions(String organizationId, Study study, String permissionRuleId, Enums.Entity entry) throws CatalogException { ParamUtils.checkObj(permissionRuleId, "PermissionRule id"); ParamUtils.checkObj(entry, "Entity"); - aclDBAdaptor.removePermissionRuleAndRemovePermissions(study, permissionRuleId, entry); + authorizationDBAdaptorFactory.getAuthorizationDBAdaptor(organizationId) + .removePermissionRuleAndRemovePermissions(study, permissionRuleId, entry); } @Override - public void removePermissionRuleAndRestorePermissions(Study study, String permissionRuleId, Enums.Entity entry) + public void removePermissionRuleAndRestorePermissions(String organizationId, Study study, String permissionRuleId, Enums.Entity entry) throws CatalogException { ParamUtils.checkObj(permissionRuleId, "PermissionRule id"); ParamUtils.checkObj(entry, "Entity"); - aclDBAdaptor.removePermissionRuleAndRestorePermissions(study, permissionRuleId, entry); + authorizationDBAdaptorFactory.getAuthorizationDBAdaptor(organizationId) + .removePermissionRuleAndRestorePermissions(study, permissionRuleId, entry); } @Override - public void removePermissionRule(long studyId, String permissionRuleId, Enums.Entity entry) throws CatalogException { + public void removePermissionRule(String organizationId, long studyId, String permissionRuleId, Enums.Entity entry) + throws CatalogException { ParamUtils.checkObj(permissionRuleId, "PermissionRule id"); ParamUtils.checkObj(entry, "Entity"); - aclDBAdaptor.removePermissionRule(studyId, permissionRuleId, entry); + authorizationDBAdaptorFactory.getAuthorizationDBAdaptor(organizationId) + .removePermissionRule(studyId, permissionRuleId, entry); } /* @@ -977,20 +1039,29 @@ public void removePermissionRule(long studyId, String permissionRuleId, Enums.En ==================================== */ + private void checkUserExists(String organizationId, String userId) throws CatalogAuthorizationException { + try { + dbAdaptorFactory.getCatalogUserDBAdaptor(organizationId).checkId(userId); + } catch (CatalogException e) { + throw new CatalogAuthorizationException("User '" + userId + "' not authorized to see Org '" + organizationId + "'.", e); + } + } + /** * Retrieves the groupId where the members belongs to. * - * @param studyId study id. - * @param members List of user ids. + * @param organizationId Organization id. + * @param studyId study id. + * @param members List of user ids. * @return the group id of the user. Null if the user does not take part of any group. * @throws CatalogException when there is any database error. */ - OpenCGAResult getGroupBelonging(long studyId, List members) throws CatalogException { - return studyDBAdaptor.getGroup(studyId, null, members); + OpenCGAResult getGroupBelonging(String organizationId, long studyId, List members) throws CatalogException { + return dbAdaptorFactory.getCatalogStudyDBAdaptor(organizationId).getGroup(studyId, null, members); } - OpenCGAResult getGroupBelonging(long studyId, String members) throws CatalogException { - return getGroupBelonging(studyId, Arrays.asList(members.split(","))); + OpenCGAResult getGroupBelonging(String organizationId, long studyId, String members) throws CatalogException { + return getGroupBelonging(organizationId, studyId, Arrays.asList(members.split(","))); } public static void checkPermissions(List permissions, Function getValue) throws CatalogException { diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/AbstractDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/AbstractDBAdaptor.java index af5ffcf4588..4187aca3582 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/AbstractDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/AbstractDBAdaptor.java @@ -16,17 +16,17 @@ package org.opencb.opencga.catalog.db; +import org.apache.commons.lang3.StringUtils; import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.Event; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.catalog.db.api.DBIterator; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.core.response.OpenCGAResult; import org.slf4j.Logger; -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; +import java.util.*; public abstract class AbstractDBAdaptor { @@ -101,6 +101,21 @@ protected void checkParameter(Object param, String name) throws CatalogDBExcepti } } + protected void checkUpdatedParams(ObjectMap parameters, List updateableKeys) throws CatalogParameterException { + Set keysToUpdate = parameters.keySet(); + Set updateableKeysSet = new HashSet<>(updateableKeys); + + List unexpectedKeys = new ArrayList<>(keysToUpdate.size()); + for (String key : keysToUpdate) { + if (!updateableKeysSet.contains(key)) { + unexpectedKeys.add(key); + } + } + if (!unexpectedKeys.isEmpty()) { + throw new CatalogParameterException("Unexpected fields passed to update: " + StringUtils.join(unexpectedKeys, ", ")); + } + } + public interface FilterOption { String getKey(); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/DBAdaptorFactory.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/DBAdaptorFactory.java index 86ccefa2653..883943f1a4b 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/DBAdaptorFactory.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/DBAdaptorFactory.java @@ -16,14 +16,17 @@ package org.opencb.opencga.catalog.db; -import org.opencb.commons.datastore.mongodb.MongoDBCollection; +import org.apache.commons.lang3.StringUtils; +import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.catalog.db.api.*; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.core.config.Admin; import org.opencb.opencga.core.config.Configuration; +import org.opencb.opencga.core.models.organizations.Organization; +import org.opencb.opencga.core.response.OpenCGAResult; -import java.util.Map; +import java.util.List; /** * Created by hpccoll1 on 19/06/15. @@ -33,8 +36,9 @@ public interface DBAdaptorFactory extends AutoCloseable { /** * Says if the catalog database is ready to be used. If false, needs to be initialized. * @return boolean + * @throws CatalogDBException CatalogDBException. */ - boolean isCatalogDBReady(); + boolean isCatalogDBReady() throws CatalogDBException; /** * Create all collections for the database. @@ -52,16 +56,13 @@ public interface DBAdaptorFactory extends AutoCloseable { */ void initialiseMetaCollection(Admin admin) throws CatalogException; - /** - * Creates the indexes needed to make queries faster. - * - * @throws CatalogDBException if there is any problem creating the indexes. - */ - void createIndexes() throws CatalogDBException; + default String getCatalogDatabase(String prefix, String organization) { + String dbPrefix = StringUtils.isEmpty(prefix) ? "opencga" : prefix; + dbPrefix = dbPrefix.endsWith("_") ? dbPrefix : dbPrefix + "_"; + return (dbPrefix + "catalog_" + organization).toLowerCase(); + } - String getCatalogDatabase(String prefix); - - boolean getDatabaseStatus(); + boolean getDatabaseStatus() throws CatalogDBException; /** * Removes the catalog database. @@ -72,35 +73,45 @@ public interface DBAdaptorFactory extends AutoCloseable { void close(); - MigrationDBAdaptor getMigrationDBAdaptor(); + void createIndexes(String organization) throws CatalogDBException; + + List getOrganizationIds() throws CatalogDBException; + + MigrationDBAdaptor getMigrationDBAdaptor(String organization) throws CatalogDBException; + + MetaDBAdaptor getCatalogMetaDBAdaptor(String organization) throws CatalogDBException; + + OpenCGAResult createOrganization(Organization organization, QueryOptions options, String userId) throws CatalogException; + + void deleteOrganization(Organization organization) throws CatalogDBException; - MetaDBAdaptor getCatalogMetaDBAdaptor(); + NoteDBAdaptor getCatalogNoteDBAdaptor(String organization) throws CatalogDBException; - UserDBAdaptor getCatalogUserDBAdaptor(); + OrganizationDBAdaptor getCatalogOrganizationDBAdaptor(String organization) throws CatalogDBException; - ProjectDBAdaptor getCatalogProjectDbAdaptor(); + UserDBAdaptor getCatalogUserDBAdaptor(String organization) throws CatalogDBException; - StudyDBAdaptor getCatalogStudyDBAdaptor(); + ProjectDBAdaptor getCatalogProjectDbAdaptor(String organization) throws CatalogDBException; - FileDBAdaptor getCatalogFileDBAdaptor(); + StudyDBAdaptor getCatalogStudyDBAdaptor(String organization) throws CatalogDBException; - SampleDBAdaptor getCatalogSampleDBAdaptor(); + FileDBAdaptor getCatalogFileDBAdaptor(String organization) throws CatalogDBException; - IndividualDBAdaptor getCatalogIndividualDBAdaptor(); + SampleDBAdaptor getCatalogSampleDBAdaptor(String organization) throws CatalogDBException; - JobDBAdaptor getCatalogJobDBAdaptor(); + IndividualDBAdaptor getCatalogIndividualDBAdaptor(String organization) throws CatalogDBException; - AuditDBAdaptor getCatalogAuditDbAdaptor(); + JobDBAdaptor getCatalogJobDBAdaptor(String organization) throws CatalogDBException; - CohortDBAdaptor getCatalogCohortDBAdaptor(); + AuditDBAdaptor getCatalogAuditDbAdaptor(String organization) throws CatalogDBException; - PanelDBAdaptor getCatalogPanelDBAdaptor(); + CohortDBAdaptor getCatalogCohortDBAdaptor(String organization) throws CatalogDBException; - FamilyDBAdaptor getCatalogFamilyDBAdaptor(); + PanelDBAdaptor getCatalogPanelDBAdaptor(String organization) throws CatalogDBException; - ClinicalAnalysisDBAdaptor getClinicalAnalysisDBAdaptor(); + FamilyDBAdaptor getCatalogFamilyDBAdaptor(String organization) throws CatalogDBException; - InterpretationDBAdaptor getInterpretationDBAdaptor(); + ClinicalAnalysisDBAdaptor getClinicalAnalysisDBAdaptor(String organization) throws CatalogDBException; - Map getMongoDBCollectionMap(); + InterpretationDBAdaptor getInterpretationDBAdaptor(String organization) throws CatalogDBException; } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/AnnotationSetDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/AnnotationSetDBAdaptor.java index a06f3762b4d..ccca18225c9 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/AnnotationSetDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/AnnotationSetDBAdaptor.java @@ -21,6 +21,7 @@ import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.core.models.common.AnnotationSet; import org.opencb.opencga.core.models.study.Variable; @@ -62,12 +63,10 @@ OpenCGAResult update(Query query, ObjectMap parameters, List variab * @param variableSetId variable set id to identify the annotations that will add a new annotation. * @param variable new variable that will be added. * @return a OpenCGAResult object. - * @throws CatalogDBException if the variable could not be added to an existing annotationSet. - * @throws CatalogParameterException if there is any unexpected parameter. - * @throws CatalogAuthorizationException if the operation is not authorized. + * @throws CatalogException if the variable could not be added to an existing annotationSet, there is any unexpected parameter or + * the operation is not authorized. */ - OpenCGAResult addVariableToAnnotations(long studyUid, long variableSetId, Variable variable) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult addVariableToAnnotations(long studyUid, long variableSetId, Variable variable) throws CatalogException; // /** // * This method will rename the id of all the annotations corresponding to the variableSetId changing oldName per newName. @@ -88,12 +87,10 @@ OpenCGAResult addVariableToAnnotations(long studyUid, long variableSetId, Variab * @param variableSetId variable set id for which the annotationSets have to delete the annotation. * @param annotationName Annotation name. * @return a OpenCGAResult object. - * @throws CatalogDBException when there is an error in the database. - * @throws CatalogParameterException if there is any unexpected parameter. - * @throws CatalogAuthorizationException if the operation is not authorized. + * @throws CatalogException when there is an error in the database, there is any unexpected parameter or the operation is not + * authorized. */ - OpenCGAResult removeAnnotationField(long studyUid, long variableSetId, String annotationName) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult removeAnnotationField(long studyUid, long variableSetId, String annotationName) throws CatalogException; /** * Makes a groupBy to obtain the different values that every annotation has and the total number of each. diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ClinicalAnalysisDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ClinicalAnalysisDBAdaptor.java index 9ba04a571c1..3935f24836d 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ClinicalAnalysisDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ClinicalAnalysisDBAdaptor.java @@ -55,8 +55,6 @@ enum QueryParams implements QueryParam { DISORDER_NAME("disorder.name", TEXT, ""), TYPE("type", TEXT, ""), ATTRIBUTES("attributes", TEXT, ""), // "Format: where is [<|<=|>|>=|==|!=|~|!~]" - NATTRIBUTES("nattributes", DECIMAL, ""), // "Format: where is [<|<=|>|>=|==|!=|~|!~]" - BATTRIBUTES("battributes", BOOLEAN, ""), // "Format: where is [==|!=]" STATUS("status", OBJECT, ""), STATUS_ID("status.id", TEXT, ""), STATUS_DATE("status.date", TEXT, ""), @@ -80,8 +78,10 @@ enum QueryParams implements QueryParam { RESPONSIBLE("responsible", OBJECT, ""), FLAGS("flags", OBJECT, ""), FLAGS_ID("flags.id", TEXT, ""), + VERSION("version", INTEGER, ""), RELEASE("release", INTEGER, ""), - PANEL_LOCK("panelLock", BOOLEAN, ""), + SNAPSHOT("snapshot", INTEGER, ""), + PANEL_LOCKED("panelLocked", BOOLEAN, ""), LOCKED("locked", BOOLEAN, ""), SAMPLE("sample", TEXT_ARRAY, ""), // Alias to search for samples within proband.samples or family.members.samples @@ -237,8 +237,7 @@ default void checkId(long clinicalAnalysisId) throws CatalogDBException, Catalog OpenCGAResult nativeInsert(Map clinicalAnalysis, String userId) throws CatalogDBException; OpenCGAResult insert(long studyId, ClinicalAnalysis clinicalAnalysis, List variableSetList, - List clinicalAuditList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + List clinicalAuditList, QueryOptions options) throws CatalogException; OpenCGAResult update(long id, ObjectMap parameters, List variableSetList, List clinicalAuditList, QueryOptions queryOptions) diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/CohortDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/CohortDBAdaptor.java index e7f0804914e..c95d006161d 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/CohortDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/CohortDBAdaptor.java @@ -70,8 +70,6 @@ enum QueryParams implements QueryParam { ANNOTATION("annotation", TEXT_ARRAY, ""), ATTRIBUTES("attributes", TEXT, "Format: where is [<|<=|>|>=|==|!=|~|!~]"), - NATTRIBUTES("nattributes", DECIMAL, "Format: where is [<|<=|>|>=|==|!=|~|!~]"), - BATTRIBUTES("battributes", BOOLEAN, "Format: where is [==|!=]"), STUDY_UID("studyUid", INTEGER_ARRAY, ""), STUDY("study", INTEGER_ARRAY, ""); // Alias to studyId in the database. Only for the webservices. @@ -135,8 +133,7 @@ default void checkId(long cohortId) throws CatalogDBException, CatalogParameterE OpenCGAResult nativeInsert(Map cohort, String userId) throws CatalogDBException; - OpenCGAResult insert(long studyId, Cohort cohort, List variableSetList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult insert(long studyId, Cohort cohort, List variableSetList, QueryOptions options) throws CatalogException; OpenCGAResult get(long cohortId, QueryOptions options) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FamilyDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FamilyDBAdaptor.java index 463ab21bd60..fe9fd976e64 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FamilyDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FamilyDBAdaptor.java @@ -56,8 +56,6 @@ enum QueryParams implements QueryParam { DESCRIPTION("description", TEXT, ""), EXPECTED_SIZE("expectedSize", INTEGER, ""), ATTRIBUTES("attributes", TEXT, ""), // "Format: where is [<|<=|>|>=|==|!=|~|!~]" - NATTRIBUTES("nattributes", DECIMAL, ""), // "Format: where is [<|<=|>|>=|==|!=|~|!~]" - BATTRIBUTES("battributes", BOOLEAN, ""), // "Format: where is [==|!=]" STATUS("status", TEXT_ARRAY, ""), STATUS_ID("status.id", TEXT, ""), STATUS_DATE("status.date", TEXT, ""), @@ -151,7 +149,7 @@ default void checkId(long familyId) throws CatalogDBException, CatalogParameterE OpenCGAResult nativeInsert(Map family, String userId) throws CatalogDBException; OpenCGAResult insert(long studyId, Family family, List members, List variableSetList, - QueryOptions options) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + QueryOptions options) throws CatalogException; OpenCGAResult get(long familyId, QueryOptions options) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java index 2690fb210ab..a341840a6bf 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java @@ -103,8 +103,6 @@ enum QueryParams implements QueryParam { INTERNAL_MISSING_SAMPLES_NON_EXISTING("internal.missingSamples.nonExisting", TEXT_ARRAY, ""), ATTRIBUTES("attributes", TEXT, ""), // "Format: where is [<|<=|>|>=|==|!=|~|!~]" - NATTRIBUTES("nattributes", DECIMAL, ""), // "Format: where is [<|<=|>|>=|==|!=|~|!~]" - BATTRIBUTES("battributes", BOOLEAN, ""), // "Format: where is [==|!=]" STATS("stats", TEXT, ""), NSTATS("nstats", DECIMAL, ""), @@ -235,13 +233,10 @@ default void checkId(long fileId) throws CatalogDBException, CatalogParameterExc * @param variableSetList Variable set list. * @param options Options to filter the output that will be returned after the insertion of the file. * @return A OpenCGAResult object containing the time spent. - * @throws CatalogDBException when the file could not be inserted due to different reasons. - * @throws CatalogParameterException if there is any formatting error. - * @throws CatalogAuthorizationException if the user is not authorised to perform the query. + * @throws CatalogException when the file could not be inserted due to any formatting error or the user is not authorised. */ OpenCGAResult insert(long studyId, File file, List existingSamples, List nonExistingSamples, - List variableSetList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + List variableSetList, QueryOptions options) throws CatalogException; /*** * Inserts the passed file in the database. @@ -254,13 +249,11 @@ OpenCGAResult insert(long studyId, File file, List existingSamples, List * @param variableSetList Variable set list. * @param options Options to filter the output that will be returned after the insertion of the file. * @return A OpenCGAResult object containing the time spent. - * @throws CatalogDBException when the file could not be inserted due to different reasons. - * @throws CatalogParameterException if there is any formatting error. - * @throws CatalogAuthorizationException if the user is not authorised to perform the query. + * @throws CatalogException when the file could not be inserted due any formatting error or the user is not authorised. */ OpenCGAResult insertWithVirtualFile(long studyId, File file, File virtualFile, List existingSamples, List nonExistingSamples, List variableSetList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + throws CatalogException; /*** * Retrieves the file from the database containing the fileId given. diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/IndividualDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/IndividualDBAdaptor.java index 0100dc55391..b6dbc94b861 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/IndividualDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/IndividualDBAdaptor.java @@ -93,8 +93,6 @@ enum QueryParams implements QueryParam { KARYOTYPIC_SEX("karyotypicSex", TEXT, ""), LIFE_STATUS("lifeStatus", TEXT, ""), ATTRIBUTES("attributes", TEXT, ""), // "Format: where is [<|<=|>|>=|==|!=|~|!~]" - NATTRIBUTES("nattributes", DECIMAL, ""), // "Format: where is [<|<=|>|>=|==|!=|~|!~]" - BATTRIBUTES("battributes", BOOLEAN, ""), // "Format: where is [==|!=]" DELETED(ParamConstants.DELETED_PARAM, BOOLEAN, ""), @@ -165,7 +163,7 @@ default void checkId(long individualId) throws CatalogDBException, CatalogParame OpenCGAResult nativeInsert(Map individual, String userId) throws CatalogDBException; OpenCGAResult insert(long studyId, Individual individual, List variableSetList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + throws CatalogException; OpenCGAResult get(long individualId, QueryOptions options) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/InterpretationDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/InterpretationDBAdaptor.java index 8c95202e0de..a49e7fd6262 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/InterpretationDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/InterpretationDBAdaptor.java @@ -24,6 +24,7 @@ import org.opencb.commons.datastore.core.QueryParam; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.api.ParamConstants; @@ -41,6 +42,7 @@ enum QueryParams implements QueryParam { ID("id", TEXT, ""), UID("uid", INTEGER, ""), UUID("uuid", TEXT, ""), + NAME("name", TEXT, ""), CLINICAL_ANALYSIS_ID("clinicalAnalysisId", TEXT, ""), DESCRIPTION("description", TEXT, ""), INTERNAL_STATUS("internal.status", TEXT, ""), @@ -135,8 +137,7 @@ default void checkId(long interpretationId) throws CatalogDBException, CatalogPa OpenCGAResult nativeInsert(Map interpretation, String userId) throws CatalogDBException; OpenCGAResult insert(long studyId, Interpretation interpretation, ParamUtils.SaveInterpretationAs action, - List clinicalAuditList) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + List clinicalAuditList) throws CatalogException; OpenCGAResult update(long uid, ObjectMap parameters, List clinicalAuditList, ParamUtils.SaveInterpretationAs action, QueryOptions queryOptions) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/JobDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/JobDBAdaptor.java index b34721cc1f7..ebbc7b62143 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/JobDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/JobDBAdaptor.java @@ -54,8 +54,7 @@ default void checkId(long jobId) throws CatalogDBException, CatalogParameterExce OpenCGAResult nativeInsert(Map job, String userId) throws CatalogDBException; - OpenCGAResult insert(long studyId, Job job, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult insert(long studyId, Job job, QueryOptions options) throws CatalogException; default OpenCGAResult restore(Query query, QueryOptions queryOptions) throws CatalogDBException { //return updateStatus(query, new Job.JobStatus(Job.JobStatus.PREPARED)); @@ -126,6 +125,7 @@ enum QueryParams implements QueryParam { INTERNAL_STATUS_DATE("internal.status.date", TEXT, ""), INTERNAL_WEBHOOK("internal.webhook", OBJECT, ""), INTERNAL_EVENTS("internal.events", OBJECT, ""), + INTERNAL_KILL_JOB_REQUESTED("internal.killJobRequested", BOOLEAN, ""), OUT_DIR("outDir", OBJECT, ""), INPUT("input", OBJECT, ""), diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/MigrationDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/MigrationDBAdaptor.java index d7181c4ff51..f936fb07d4a 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/MigrationDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/MigrationDBAdaptor.java @@ -3,7 +3,7 @@ import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryParam; import org.opencb.opencga.catalog.exceptions.CatalogDBException; -import org.opencb.opencga.catalog.migration.MigrationRun; +import org.opencb.opencga.core.models.migration.MigrationRun; import org.opencb.opencga.core.response.OpenCGAResult; import java.util.Collections; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/NoteDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/NoteDBAdaptor.java new file mode 100644 index 00000000000..eeecd6d55df --- /dev/null +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/NoteDBAdaptor.java @@ -0,0 +1,87 @@ +package org.opencb.opencga.catalog.db.api; + +import org.apache.commons.collections4.map.LinkedMap; +import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.commons.datastore.core.QueryParam; +import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; +import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.exceptions.CatalogParameterException; +import org.opencb.opencga.core.models.notes.Note; +import org.opencb.opencga.core.response.OpenCGAResult; + +import java.util.Map; + +import static org.opencb.commons.datastore.core.QueryParam.Type.*; + +public interface NoteDBAdaptor extends DBAdaptor { + + OpenCGAResult insert(Note note) throws CatalogException; + + default OpenCGAResult get(long noteUid, QueryOptions options) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + Query query = new Query(QueryParams.UID.key(), noteUid); + return get(query, options); + } + + enum QueryParams implements QueryParam { + UID("uid", LONG, ""), + STUDY_UID("studyUid", LONG, ""), + ID("id", STRING, ""), + UUID("uuid", STRING, ""), + SCOPE("scope", STRING, ""), + STUDY("study", STRING, ""), + TAGS("tags", TEXT_ARRAY, ""), + USER_ID("userId", STRING, ""), + VISIBILITY("visibility", STRING, ""), + VERSION("version", INTEGER, ""), + CREATION_DATE("creationDate", DATE, ""), + MODIFICATION_DATE("modificationDate", DATE, ""), + VALUE_TYPE("valueType", STRING, ""), + VALUE("value", OBJECT, ""); + + private static Map map; + + static { + map = new LinkedMap(); + for (QueryParams params : QueryParams.values()) { + map.put(params.key(), params); + } + } + + private final String key; + private Type type; + private String description; + + QueryParams(String key, Type type, String description) { + this.key = key; + this.type = type; + this.description = description; + } + + public static Map getMap() { + return map; + } + + public static QueryParams getParam(String key) { + return map.get(key); + } + + @Override + public String key() { + return key; + } + + @Override + public Type type() { + return type; + } + + @Override + public String description() { + return description; + } + } + +} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/OrganizationDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/OrganizationDBAdaptor.java new file mode 100644 index 00000000000..93e65c63121 --- /dev/null +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/OrganizationDBAdaptor.java @@ -0,0 +1,248 @@ +package org.opencb.opencga.catalog.db.api; + +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.commons.datastore.core.QueryParam; +import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; +import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.exceptions.CatalogParameterException; +import org.opencb.opencga.core.models.organizations.Organization; +import org.opencb.opencga.core.response.OpenCGAResult; + +import java.util.HashMap; +import java.util.Map; + +import static org.opencb.commons.datastore.core.QueryParam.Type.*; + +public interface OrganizationDBAdaptor extends Iterable { + + String IS_ORGANIZATION_ADMIN_OPTION = "isOrgAdmin"; + String AUTH_ORIGINS_FIELD = "authenticationOrigins"; + + enum QueryParams implements QueryParam { + UID("uid", LONG, ""), + ID("id", STRING, ""), + UUID("uuid", STRING, ""), + NAME("name", STRING, ""), + OWNER("owner", STRING, ""), + ADMINS("admins", TEXT_ARRAY, ""), + INTERNAL("internal", OBJECT, ""), + INTERNAL_MIGRATION_EXECUTIONS("internal.migrationExecutions", OBJECT, ""), + CONFIGURATION("configuration", OBJECT, ""), + CONFIGURATION_OPTIMIZATIONS("configuration.optimizations", OBJECT, ""), + CONFIGURATION_AUTHENTICATION_ORIGINS("configuration." + AUTH_ORIGINS_FIELD, OBJECT, ""), + CONFIGURATION_AUTHENTICATION_ORIGINS_ID("configuration." + AUTH_ORIGINS_FIELD + ".id", STRING, ""), + CONFIGURATION_AUTHENTICATION_ORIGINS_OPTIONS("configuration." + AUTH_ORIGINS_FIELD + ".options", OBJECT, ""), + CONFIGURATION_TOKEN("configuration.token", OBJECT, ""), + CREATION_DATE("creationDate", DATE, ""), + MODIFICATION_DATE("modificationDate", DATE, ""), + PROJECTS("projects", OBJECT, ""), + NOTES("notes", OBJECT, ""), + ATTRIBUTES("attributes", OBJECT, ""); + + private static Map map = new HashMap<>(); + + static { + for (QueryParams params : QueryParams.values()) { + map.put(params.key(), params); + } + } + + private final String key; + private Type type; + private String description; + + QueryParams(String key, Type type, String description) { + this.key = key; + this.type = type; + this.description = description; + } + + @Override + public String key() { + return key; + } + + @Override + public Type type() { + return type; + } + + @Override + public String description() { + return description; + } + + public static Map getMap() { + return map; + } + + public static QueryParams getParam(String key) { + return map.get(key); + } + } + +// default boolean exists(long projectId) throws CatalogDBException { +// return count(new Query(QueryParams.UID.key(), projectId)).getNumMatches() > 0; +// } +// +// default void checkId(long projectId) throws CatalogDBException { +// if (projectId < 0) { +// throw CatalogDBException.newInstance("Project id '{}' is not valid: ", projectId); +// } +// +// if (!exists(projectId)) { +// throw CatalogDBException.newInstance("Project id '{}' does not exist", projectId); +// } +// } +// +// OpenCGAResult nativeInsert(Map project, String userId) throws CatalogDBException; +// + OpenCGAResult insert(Organization organization, QueryOptions options) throws CatalogException; + + OpenCGAResult get(String userId, QueryOptions options) throws CatalogDBException; + + OpenCGAResult get(QueryOptions options) throws CatalogDBException; +// +// OpenCGAResult incrementCurrentRelease(long projectId) throws CatalogDBException; +// +// long getId(String userId, String projectAlias) throws CatalogDBException; +// +// String getOwnerId(long projectId) throws CatalogDBException; +// +// +// default OpenCGAResult count() throws CatalogDBException { +// return count(new Query()); +// } +// +// OpenCGAResult count(Query query) throws CatalogDBException; +// +// OpenCGAResult count(Query query, String user, StudyPermissions.Permissions studyPermission) +// throws CatalogDBException, CatalogAuthorizationException; +// +// default OpenCGAResult distinct(String field) throws CatalogDBException { +// return distinct(new Query(), field); +// } +// +// OpenCGAResult distinct(Query query, String field) throws CatalogDBException; +// +// +// default OpenCGAResult stats() { +// return stats(new Query()); +// } +// +// OpenCGAResult stats(Query query); +// +// +// OpenCGAResult get(Query query, QueryOptions options) throws CatalogDBException; +// +// OpenCGAResult get(Query query, QueryOptions options, String user) throws CatalogDBException, CatalogParameterException; +// +// default List> get(List queries, QueryOptions options) throws CatalogDBException { +// Objects.requireNonNull(queries); +// List> queryResults = new ArrayList<>(queries.size()); +// for (Query query : queries) { +// queryResults.add(get(query, options)); +// } +// return queryResults; +// } +// +// OpenCGAResult nativeGet(Query query, QueryOptions options) throws CatalogDBException; +// +// OpenCGAResult nativeGet(Query query, QueryOptions options, String user) +// throws CatalogDBException, CatalogAuthorizationException; +// +// default List nativeGet(List queries, QueryOptions options) throws CatalogDBException { +// Objects.requireNonNull(queries); +// List queryResults = new ArrayList<>(queries.size()); +// for (Query query : queries) { +// queryResults.add(nativeGet(query, options)); +// } +// return queryResults; +// } +// + OpenCGAResult update(String organizationId, ObjectMap parameters, QueryOptions queryOptions) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; +// +// OpenCGAResult update(Query query, ObjectMap parameters, QueryOptions queryOptions) throws CatalogDBException; +// + OpenCGAResult delete(Organization organization) throws CatalogDBException; +// +// OpenCGAResult delete(Query query) throws CatalogDBException; +// +// default OpenCGAResult delete(long id, QueryOptions queryOptions) throws CatalogDBException { +// throw new NotImplementedException(""); +// } +// +// @Deprecated +// default OpenCGAResult delete(Query query, QueryOptions queryOptions) throws CatalogDBException { +// throw new NotImplementedException(""); +// } +// +// @Deprecated +// default OpenCGAResult remove(long id, QueryOptions queryOptions) throws CatalogDBException { +// throw new NotImplementedException(""); +// } +// +// @Deprecated +// default OpenCGAResult remove(Query query, QueryOptions queryOptions) throws CatalogDBException { +// throw new NotImplementedException(""); +// } +// +// OpenCGAResult restore(long id, QueryOptions queryOptions) +// throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; +// +// OpenCGAResult restore(Query query, QueryOptions queryOptions) throws CatalogDBException; +// +// +//// OpenCGAResult updateStatus(Query query, Status status) throws CatalogDBException; +// +// + @Override + default DBIterator iterator() { + try { + return iterator(new QueryOptions()); + } catch (CatalogDBException e) { + throw new RuntimeException(e); + } + } + + DBIterator iterator(QueryOptions options) throws CatalogDBException; + +// default DBIterator nativeIterator() throws CatalogDBException { +// return nativeIterator(new Query(), new QueryOptions()); +// } +// +// DBIterator nativeIterator(Query query, QueryOptions options) throws CatalogDBException; +// +// DBIterator iterator(Query query, QueryOptions options, String user) +// throws CatalogDBException, CatalogAuthorizationException; +// +// DBIterator nativeIterator(Query query, QueryOptions options, String user) +// throws CatalogDBException, CatalogAuthorizationException; +// +// OpenCGAResult rank(Query query, String field, int numResults, boolean asc) throws CatalogDBException; +// +// OpenCGAResult groupBy(Query query, String field, QueryOptions options) throws CatalogDBException; +// +// OpenCGAResult groupBy(Query query, List fields, QueryOptions options) throws CatalogDBException; +// +// OpenCGAResult groupBy(Query query, String field, QueryOptions options, String user) +// throws CatalogDBException, CatalogAuthorizationException; +// +// OpenCGAResult groupBy(Query query, List fields, QueryOptions options, String user) +// throws CatalogDBException, CatalogAuthorizationException; +// +// @Override +// default void forEach(Consumer action) { +// try { +// forEach(new Query(), action, new QueryOptions()); +// } catch (CatalogDBException e) { +// throw new RuntimeException(e); +// } +// } +// +// void forEach(Query query, Consumer action, QueryOptions options) throws CatalogDBException; + +} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/PanelDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/PanelDBAdaptor.java index d9ebc9b82ce..63bdd0d5e18 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/PanelDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/PanelDBAdaptor.java @@ -64,8 +64,6 @@ enum QueryParams implements QueryParam { STATS("stats", TEXT_ARRAY, ""), ATTRIBUTES("attributes", TEXT, ""), // "Format: where is [<|<=|>|>=|==|!=|~|!~]" - NATTRIBUTES("nattributes", DECIMAL, ""), // "Format: where is [<|<=|>|>=|==|!=|~|!~]" - BATTRIBUTES("battributes", BOOLEAN, ""), // "Format: where is [==|!=]" TAGS("tags", TEXT_ARRAY, ""), CATEGORIES("categories", TEXT_ARRAY, ""), @@ -155,11 +153,9 @@ default void checkUid(long panelUid) throws CatalogDBException, CatalogParameter } } - OpenCGAResult insert(long studyUid, List panelList) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult insert(long studyUid, List panelList) throws CatalogException; - OpenCGAResult insert(long studyId, Panel panel, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult insert(long studyId, Panel panel, QueryOptions options) throws CatalogException; OpenCGAResult get(long panelId, QueryOptions options) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ProjectDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ProjectDBAdaptor.java index e41e9029160..1ff667bdbac 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ProjectDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ProjectDBAdaptor.java @@ -23,6 +23,7 @@ import org.opencb.commons.datastore.core.QueryParam; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.core.models.project.Project; import org.opencb.opencga.core.models.study.StudyPermissions; @@ -58,14 +59,11 @@ enum QueryParams implements QueryParam { INTERNAL_STATUS_ID("internal.status.id", TEXT, ""), INTERNAL_STATUS_MSG("internal.status.msg", TEXT, ""), INTERNAL_STATUS_DATE("internal.status.date", TEXT, ""), - USER_ID("userId", TEXT, ""), INTERNAL_DATASTORES("internal.datastores", TEXT_ARRAY, ""), INTERNAL_DATASTORES_VARIANT("internal.datastores.variant", TEXT_ARRAY, ""), INTERNAL("internal", TEXT_ARRAY, ""), ATTRIBUTES("attributes", TEXT, ""), // "Format: where is [<|<=|>|>=|==|!=|~|!~]" - NATTRIBUTES("nattributes", DECIMAL, ""), // "Format: where is [<|<=|>|>=|==|!=|~|!~]" - BATTRIBUTES("battributes", BOOLEAN, ""), // "Format: where is [==|!=]" OWNER("owner", TEXT, ""), @@ -136,20 +134,12 @@ default void checkId(long projectId) throws CatalogDBException { OpenCGAResult nativeInsert(Map project, String userId) throws CatalogDBException; - OpenCGAResult insert(Project project, String userId, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; - - OpenCGAResult get(String userId, QueryOptions options) throws CatalogDBException; + OpenCGAResult insert(Project project, QueryOptions options) throws CatalogException; OpenCGAResult get(long project, QueryOptions options) throws CatalogDBException; OpenCGAResult incrementCurrentRelease(long projectId) throws CatalogDBException; - long getId(String userId, String projectAlias) throws CatalogDBException; - - String getOwnerId(long projectId) throws CatalogDBException; - - default OpenCGAResult count() throws CatalogDBException { return count(new Query()); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/SampleDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/SampleDBAdaptor.java index abc3dee02d6..485ab7b10dd 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/SampleDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/SampleDBAdaptor.java @@ -62,8 +62,8 @@ default void checkId(long sampleId) throws CatalogDBException, CatalogParameterE OpenCGAResult nativeInsert(Map sample, String userId) throws CatalogDBException; - OpenCGAResult insert(long studyId, Sample sample, List variableSetList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult insert(long studyId, Sample sample, List variableSetList, QueryOptions options) + throws CatalogException; OpenCGAResult get(long sampleId, QueryOptions options) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; @@ -86,13 +86,11 @@ OpenCGAResult updateProjectRelease(long studyId, int release) */ OpenCGAResult unmarkPermissionRule(long studyId, String permissionRuleId) throws CatalogException; - default OpenCGAResult setRgaIndexes(long studyUid, RgaIndex rgaIndex) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + default OpenCGAResult setRgaIndexes(long studyUid, RgaIndex rgaIndex) throws CatalogException { return setRgaIndexes(studyUid, Collections.emptyList(), rgaIndex); } - OpenCGAResult setRgaIndexes(long studyUid, List sampleUids, RgaIndex rgaIndex) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult setRgaIndexes(long studyUid, List sampleUids, RgaIndex rgaIndex) throws CatalogException; enum QueryParams implements QueryParam { ID("id", TEXT, ""), @@ -116,8 +114,6 @@ enum QueryParams implements QueryParam { COHORT_IDS("cohortIds", TEXT_ARRAY, ""), SOMATIC("somatic", BOOLEAN, ""), ATTRIBUTES("attributes", TEXT, ""), // "Format: where is [<|<=|>|>=|==|!=|~|!~]" - NATTRIBUTES("nattributes", DECIMAL, ""), // "Format: where is [<|<=|>|>=|==|!=|~|!~]" - BATTRIBUTES("battributes", BOOLEAN, ""), // "Format: where is [==|!=]" STATUS("status", TEXT_ARRAY, ""), STATUS_ID("status.id", TEXT, ""), STATUS_DATE("status.date", TEXT, ""), diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/StudyDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/StudyDBAdaptor.java index 7ac9ad282ff..0afd3f255d4 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/StudyDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/StudyDBAdaptor.java @@ -16,6 +16,7 @@ package org.opencb.opencga.catalog.db.api; +import com.mongodb.client.ClientSession; import org.apache.commons.collections4.map.LinkedMap; import org.apache.commons.lang3.NotImplementedException; import org.opencb.commons.datastore.core.ObjectMap; @@ -24,9 +25,11 @@ import org.opencb.commons.datastore.core.QueryParam; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.models.common.Enums; +import org.opencb.opencga.core.models.file.File; import org.opencb.opencga.core.models.project.Project; import org.opencb.opencga.core.models.study.*; import org.opencb.opencga.core.response.OpenCGAResult; @@ -186,9 +189,9 @@ default void checkId(long studyId) throws CatalogDBException { } } - OpenCGAResult nativeInsert(Map study, String userId) throws CatalogDBException; + OpenCGAResult nativeInsert(Map study) throws CatalogDBException; - OpenCGAResult insert(Project project, Study study, QueryOptions options) throws CatalogDBException; + OpenCGAResult insert(Project project, Study study, List files, QueryOptions options) throws CatalogDBException; boolean hasStudyPermission(long studyId, String user, StudyPermissions.Permissions permission) throws CatalogDBException; @@ -198,12 +201,6 @@ default void checkId(long studyId) throws CatalogDBException { long getId(long projectId, String studyAlias) throws CatalogDBException; - long getProjectUidByStudyUid(long studyUid) throws CatalogDBException; - - String getProjectIdByStudyUid(long studyUid) throws CatalogDBException; - - String getOwnerId(long studyId) throws CatalogDBException; - OpenCGAResult createGroup(long studyId, Group group) throws CatalogDBException; /** @@ -253,8 +250,7 @@ OpenCGAResult setUsersToGroup(long studyId, String groupId, List */ OpenCGAResult removeUsersFromGroup(long studyId, String groupId, List members) throws CatalogDBException; - OpenCGAResult removeUsersFromAllGroups(long studyId, List users) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult removeUsersFromAllGroups(long studyId, List users) throws CatalogException; /** * Delete a group. @@ -277,12 +273,9 @@ OpenCGAResult removeUsersFromAllGroups(long studyId, List users) * @param groupList List containing possible groups that are synced and where the user should be added to. * @param authOrigin Authentication origin of the synced groups. * @return OpenCGAResult object. - * @throws CatalogDBException CatalogDBException - * @throws CatalogParameterException CatalogParameterException - * @throws CatalogAuthorizationException CatalogAuthorizationException + * @throws CatalogException CatalogException */ - OpenCGAResult resyncUserWithSyncedGroups(String user, List groupList, String authOrigin) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult resyncUserWithSyncedGroups(String user, List groupList, String authOrigin) throws CatalogException; /** * ADD or REMOVE user to list of provided groups. @@ -292,13 +285,10 @@ OpenCGAResult resyncUserWithSyncedGroups(String user, List groupL * @param groupList List of group ids. * @param action Update action [ADD, REMOVE] * @return OpenCGAResult object. - * @throws CatalogDBException CatalogDBException - * @throws CatalogParameterException CatalogParameterException - * @throws CatalogAuthorizationException CatalogAuthorizationException + * @throws CatalogException CatalogException */ - OpenCGAResult updateUserFromGroups(String user, List studyUids, List groupList, - ParamUtils.AddRemoveAction action) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult updateUserFromGroups(String user, List studyUids, List groupList, ParamUtils.AddRemoveAction action) + throws CatalogException; /** * Create the permission rule to the list of permission rules defined for the entry in the studyId. @@ -385,13 +375,13 @@ default void checkVariableSetExists(String variableSetId, long studyId) throws C OpenCGAResult createVariableSet(long studyId, VariableSet variableSet) throws CatalogDBException; OpenCGAResult addFieldToVariableSet(long studyUid, long variableSetId, Variable variable, String user) - throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException; + throws CatalogException; OpenCGAResult renameFieldVariableSet(long variableSetId, String oldName, String newName, String user) throws CatalogDBException, CatalogAuthorizationException; OpenCGAResult removeFieldFromVariableSet(long studyUid, long variableSetId, String name, String user) - throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException; + throws CatalogException; OpenCGAResult getVariableSet(long variableSetUid, QueryOptions options) throws CatalogDBException; @@ -415,12 +405,9 @@ OpenCGAResult getVariableSet(long variableSetId, QueryOptions optio OpenCGAResult getVariableSets(Query query, QueryOptions queryOptions, String user) throws CatalogDBException, CatalogAuthorizationException; - OpenCGAResult deleteVariableSet(long studyUid, VariableSet variableSet, boolean force) - throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException; + OpenCGAResult deleteVariableSet(long studyUid, VariableSet variableSet, boolean force) throws CatalogException; - long getStudyIdByVariableSetId(long variableSetId) throws CatalogDBException; - - OpenCGAResult getStudiesFromUser(String userId, QueryOptions queryOptions) throws CatalogDBException; + void updateDiskUsage(ClientSession clientSession, long studyId, long size) throws CatalogDBException; enum QueryParams implements QueryParam { ID("id", TEXT, ""), @@ -434,6 +421,7 @@ enum QueryParams implements QueryParam { DESCRIPTION("description", TEXT, ""), TYPE("type", OBJECT, ""), SOURCES("sources", TEXT_ARRAY, ""), + NOTES("notes", OBJECT, ""), STATUS("status", TEXT_ARRAY, ""), STATUS_ID("status.id", TEXT, ""), STATUS_DATE("status.date", TEXT, ""), @@ -457,8 +445,6 @@ enum QueryParams implements QueryParam { PROJECT_UUID("projectUuid", TEXT, ""), ADDITIONAL_INFO("additionalInfo", TEXT_ARRAY, ""), ATTRIBUTES("attributes", TEXT, ""), // "Format: where is [<|<=|>|>=|==|!=|~|!~]", - NATTRIBUTES("nattributes", DECIMAL, ""), // "Format: where is [<|<=|>|>=|==|!=|~|!~]" - BATTRIBUTES("battributes", BOOLEAN, ""), // "Format: where is [==|!=]" RELEASE("release", INTEGER, ""), GROUPS("groups", TEXT_ARRAY, ""), @@ -471,9 +457,6 @@ enum QueryParams implements QueryParam { PERMISSION_RULES("permissionRules", TEXT_ARRAY, ""), - OWNER("_ownerId", TEXT, ""), - COHORTS("cohorts", TEXT_ARRAY, ""), - DELETED("deleted", BOOLEAN, ""), VARIABLE_SET("variableSets", TEXT_ARRAY, ""), diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/UserDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/UserDBAdaptor.java index a56676d09bd..84efa64aea2 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/UserDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/UserDBAdaptor.java @@ -22,10 +22,7 @@ import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.QueryParam; -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; -import org.opencb.opencga.catalog.exceptions.CatalogDBException; -import org.opencb.opencga.catalog.exceptions.CatalogParameterException; +import org.opencb.opencga.catalog.exceptions.*; import org.opencb.opencga.core.models.user.User; import org.opencb.opencga.core.models.user.UserFilter; import org.opencb.opencga.core.response.OpenCGAResult; @@ -75,7 +72,7 @@ default void checkIds(List userIds) throws CatalogDBException, CatalogPa void authenticate(String userId, String password) throws CatalogDBException, CatalogAuthenticationException; OpenCGAResult insert(User user, String password, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + throws CatalogException; OpenCGAResult get(String userId, QueryOptions options) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; @@ -97,9 +94,10 @@ OpenCGAResult delete(String userId, QueryOptions queryOptions) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; OpenCGAResult changePassword(String userId, String oldPassword, String newPassword) - throws CatalogDBException, CatalogAuthenticationException; + throws CatalogException; - OpenCGAResult resetPassword(String userId, String email, String newCryptPass) throws CatalogDBException; + OpenCGAResult resetPassword(String userId, String email, String newCryptPass) + throws CatalogException; // Config operations OpenCGAResult setConfig(String userId, String name, Map config) throws CatalogDBException; @@ -118,26 +116,24 @@ enum QueryParams implements QueryParam { NAME("name", TEXT_ARRAY, ""), EMAIL("email", TEXT_ARRAY, ""), ORGANIZATION("organization", TEXT_ARRAY, ""), + CREATION_DATE("creationDate", TEXT_ARRAY, ""), + MODIFICATION_DATE("modificationDate", TEXT_ARRAY, ""), + DEPRECATED_ACCOUNT("account", OBJECT, ""), // Deprecated since 3.2.1 #TASK-6494 TODO: Remove in future releases + DEPRECATED_ACCOUNT_AUTHENTICATION_ID("account.authentication.id", TEXT, ""), // Deprecated since 3.2.1 #TASK-6494 + INTERNAL("internal", OBJECT, ""), INTERNAL_STATUS_ID("internal.status.id", TEXT, ""), INTERNAL_STATUS_DATE("internal.status.date", TEXT, ""), - ACCOUNT("account", TEXT_ARRAY, ""), - ACCOUNT_TYPE("account.type", TEXT, ""), - ACCOUNT_AUTHENTICATION_ID("account.authentication.id", TEXT, ""), - ACCOUNT_CREATION_DATE("account.creationDate", TEXT, ""), - SIZE("size", INTEGER_ARRAY, ""), - QUOTA("quota", INTEGER_ARRAY, ""), + INTERNAL_ACCOUNT("internal.account", TEXT_ARRAY, ""), + INTERNAL_ACCOUNT_AUTHENTICATION_ID("internal.account.authentication.id", TEXT, ""), + INTERNAL_ACCOUNT_FAILED_ATTEMPTS("internal.account.failedAttempts", INTEGER, ""), + INTERNAL_ACCOUNT_CREATION_DATE("internal.account.creationDate", TEXT, ""), + INTERNAL_ACCOUNT_EXPIRATION_DATE("internal.account.expirationDate", TEXT, ""), + INTERNAL_ACCOUNT_PASSWORD_EXPIRATION_DATE("internal.account.password.expirationDate", TEXT, ""), + INTERNAL_ACCOUNT_PASSWORD_LAST_MODIFIED("internal.account.password.lastModified", TEXT, ""), + QUOTA("quota", OBJECT, ""), ATTRIBUTES("attributes", TEXT, ""), // "Format: where is [<|<=|>|>=|==|!=|~|!~]" - NATTRIBUTES("nattributes", DECIMAL, ""), // "Format: where is [<|<=|>|>=|==|!=|~|!~]" - BATTRIBUTES("battributes", BOOLEAN, ""), // "Format: where is [==|!=]" PROJECTS("projects", TEXT_ARRAY, ""), - PROJECTS_UID("projects.uid", INTEGER_ARRAY, ""), - PROJECTS_ID("projects.id", INTEGER_ARRAY, ""), - PROJECT_NAME("projects.name", TEXT_ARRAY, ""), - PROJECT_ORGANIZATION("projects.organization", TEXT_ARRAY, ""), - PROJECT_STATUS("projects.status", TEXT_ARRAY, ""), - - SHARED_PROJECTS("sharedProjects", TEXT_ARRAY, ""), TOOL_ID("tools.id", INTEGER_ARRAY, ""), TOOL_NAME("tools.name", TEXT_ARRAY, ""), diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AnnotationMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AnnotationMongoDBAdaptor.java index d1e48be9331..02ce5649729 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AnnotationMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AnnotationMongoDBAdaptor.java @@ -31,6 +31,7 @@ import org.opencb.opencga.catalog.db.mongodb.converters.AnnotationConverter; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.utils.AnnotationUtils; import org.opencb.opencga.catalog.utils.Constants; @@ -687,8 +688,7 @@ private List getNewAnnotationList(List annotationSetLis return annotationList; } - public OpenCGAResult addVariableToAnnotations(long studyUid, long variableSetId, Variable variable) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult addVariableToAnnotations(long studyUid, long variableSetId, Variable variable) throws CatalogException { long startTime = startQuery(); // We generate the generic document that should be inserted @@ -764,12 +764,9 @@ public OpenCGAResult addVariableToAnnotations(long studyUid, long variableSetId, * @param variableSetId Variable set id. * @param fieldId Field id corresponds with the variable name whose annotations have to be removed. * @return A OpenCGAResult object. - * @throws CatalogDBException if there is any unexpected error. - * @throws CatalogParameterException if there is any unexpected parameter. - * @throws CatalogAuthorizationException if the operation is not authorized. + * @throws CatalogException if there is any unexpected error or parameter, or if the operation is not authorized. */ - public OpenCGAResult removeAnnotationField(long studyUid, long variableSetId, String fieldId) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult removeAnnotationField(long studyUid, long variableSetId, String fieldId) throws CatalogException { long startTime = startQuery(); UpdateDocument updateDocument = new UpdateDocument(); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBAdaptor.java index 26d65edff3f..ac6dd7af9fe 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBAdaptor.java @@ -20,6 +20,7 @@ import com.mongodb.client.model.Aggregates; import com.mongodb.client.model.Filters; import com.mongodb.client.model.Projections; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.bson.Document; import org.bson.conversions.Bson; @@ -29,17 +30,16 @@ import org.opencb.commons.datastore.core.QueryParam; import org.opencb.commons.datastore.mongodb.MongoDBCollection; import org.opencb.commons.datastore.mongodb.MongoDBIterator; -import org.opencb.commons.utils.CollectionUtils; import org.opencb.opencga.catalog.auth.authorization.AuthorizationDBAdaptor; import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager; import org.opencb.opencga.catalog.auth.authorization.CatalogAuthorizationManager; -import org.opencb.opencga.catalog.db.DBAdaptorFactory; import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; -import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.exceptions.CatalogParameterException; +import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.Configuration; +import org.opencb.opencga.core.models.Acl; import org.opencb.opencga.core.models.AclEntry; import org.opencb.opencga.core.models.AclEntryList; import org.opencb.opencga.core.models.common.Enums; @@ -60,15 +60,12 @@ */ public class AuthorizationMongoDBAdaptor extends MongoDBAdaptor implements AuthorizationDBAdaptor { - private final Map> dbCollectionMap = new HashMap<>(); - private static final String ANONYMOUS = "*"; static final String MEMBER_WITH_INTERNAL_ACL = "_withInternalAcls"; - public AuthorizationMongoDBAdaptor(DBAdaptorFactory dbFactory, Configuration configuration) { + public AuthorizationMongoDBAdaptor(OrganizationMongoDBAdaptorFactory dbFactory, Configuration configuration) { super(configuration, LoggerFactory.getLogger(AuthorizationMongoDBAdaptor.class)); - this.dbAdaptorFactory = (MongoDBAdaptorFactory) dbFactory; - initCollectionConnections(); + this.dbAdaptorFactory = dbFactory; } enum QueryParams implements QueryParam { @@ -118,30 +115,6 @@ public static QueryParams getParam(String key) { } } - private void initCollectionConnections() { - this.dbCollectionMap.put(Enums.Resource.STUDY, - Collections.singletonList(dbAdaptorFactory.getCatalogStudyDBAdaptor().getStudyCollection())); - this.dbCollectionMap.put(Enums.Resource.COHORT, - Collections.singletonList(dbAdaptorFactory.getCatalogCohortDBAdaptor().getCohortCollection())); - this.dbCollectionMap.put(Enums.Resource.FILE, - Collections.singletonList(dbAdaptorFactory.getCatalogFileDBAdaptor().getCollection())); - this.dbCollectionMap.put(Enums.Resource.JOB, - Collections.singletonList(dbAdaptorFactory.getCatalogJobDBAdaptor().getJobCollection())); - this.dbCollectionMap.put(Enums.Resource.CLINICAL_ANALYSIS, - Collections.singletonList(dbAdaptorFactory.getClinicalAnalysisDBAdaptor().getCollection())); - - // Versioned models will always have first the main collection and second the archive collection - this.dbCollectionMap.put(Enums.Resource.INDIVIDUAL, Arrays.asList(dbAdaptorFactory.getCatalogIndividualDBAdaptor().getCollection(), - dbAdaptorFactory.getCatalogIndividualDBAdaptor().getIndividualArchiveCollection())); - this.dbCollectionMap.put(Enums.Resource.SAMPLE, Arrays.asList(dbAdaptorFactory.getCatalogSampleDBAdaptor().getCollection(), - dbAdaptorFactory.getCatalogSampleDBAdaptor().getArchiveSampleCollection())); - this.dbCollectionMap.put(Enums.Resource.DISEASE_PANEL, - Arrays.asList(dbAdaptorFactory.getCatalogPanelDBAdaptor().getPanelCollection(), - dbAdaptorFactory.getCatalogPanelDBAdaptor().getPanelArchiveCollection())); - this.dbCollectionMap.put(Enums.Resource.FAMILY, Arrays.asList(dbAdaptorFactory.getCatalogFamilyDBAdaptor().getCollection(), - dbAdaptorFactory.getCatalogFamilyDBAdaptor().getArchiveFamilyCollection())); - } - private List getFullPermissions(Enums.Resource resource) { List permissionList = new ArrayList<>(resource.getFullPermissionList()); permissionList.add("NONE"); @@ -174,13 +147,14 @@ private void validateEntry(Enums.Resource entry) throws CatalogDBException { * @param membersList Members for which we want to fetch the permissions. If empty, it should return the permissions for all members. * @param entry Entity where the query will be performed. * @return A map of [acl, user_defined_acl] -> user -> List of permissions and the string id of the resource queried. + * @throws CatalogDBException CatalogDBException. */ - private EntryPermission internalGet(long resourceId, List membersList, Enums.Resource entry) { + private EntryPermission internalGet(long resourceId, List membersList, Enums.Resource entry) throws CatalogDBException { EntryPermission entryPermission = new EntryPermission(); List members = (membersList == null ? Collections.emptyList() : membersList); - MongoDBCollection collection = dbCollectionMap.get(entry).get(0); + MongoDBCollection collection = getMainCollection(entry); List aggregation = new ArrayList<>(); aggregation.add(Aggregates.match(Filters.eq(PRIVATE_UID, resourceId))); @@ -263,6 +237,225 @@ private EntryPermission internalGet(long resourceId, List membersList, E return entryPermission; } + /** + * Fetch the effective permissions of every user. + * + * @param studyUid Study where the resources belong. + * @param resourceIdList List of resource ids. + * @param entry Entity where the query will be performed. + * @return A list of ACL object. + * @throws CatalogDBException CatalogDBException if the study or the resourcers cannot be found. + */ + public List effectivePermissions(long studyUid, List resourceIdList, Enums.Resource entry) throws CatalogDBException { + // Get groups and array of ACLs from the study document + MongoDBCollection studyCollection = getMainCollection(Enums.Resource.STUDY); + Bson studyQuery = Filters.eq(PRIVATE_UID, studyUid); + Bson studyProjection = Projections.include(StudyDBAdaptor.QueryParams.GROUPS.key(), QueryParams.ACL.key()); + DataResult studyResult = studyCollection.find(studyQuery, studyProjection, null); + if (studyResult.getNumMatches() == 0) { + throw new CatalogDBException("Study uid '" + studyUid + "' not found"); + } + Document studyDocument = studyResult.first(); + + boolean simplifyPermissions = simplifyPermissions(); + Map> groupsMap = getGroupUsersMap(studyDocument); + Map> studyUserPermissionsMap = extractUserPermissionsMap(groupsMap, studyDocument, simplifyPermissions); + + // Retrieve ACL list for the resources requested + MongoDBCollection collection = getMainCollection(entry); + Bson query = Filters.and( + Filters.eq(PRIVATE_STUDY_UID, studyUid), + Filters.in(ID, resourceIdList) + ); + Bson projection = Projections.include(QueryParams.ID.key(), QueryParams.ACL.key()); + + logger.debug("Get Acl: {}", query.toBsonDocument()); + DataResult dataResult = collection.find(query, projection, null); + Map dataResultMap = new HashMap<>(); + for (Document result : dataResult.getResults()) { + dataResultMap.put(result.getString(ID), result); + } + + // Process resourceIdList in order + List aclList = new ArrayList<>(resourceIdList.size()); + for (String resourceId : resourceIdList) { + if (!dataResultMap.containsKey(resourceId)) { + throw new CatalogDBException("Resource id '" + resourceId + "' not found."); + } + Document resourceDocument = dataResultMap.get(resourceId); + Map> resourceUserPermissionsMap = extractUserPermissionsMap(groupsMap, resourceDocument, + simplifyPermissions); + Acl acl = convertPermissionsToAcl(groupsMap, studyUserPermissionsMap, resourceUserPermissionsMap, resourceId, entry); + aclList.add(acl); + } + + return aclList; + } + + private Acl convertPermissionsToAcl(Map> groupsMap, Map> studyUserPermissionsMap, + Map> resourceUserPermissionsMap, String id, Enums.Resource resource) { + Set adminUsers = groupsMap.get(ParamConstants.ADMINS_GROUP); + + // Init permission map + Map> permissionMap = new HashMap<>(); + List resourcePermissions = resource.getFullPermissionList(); + for (String permission : resource.getFullPermissionList()) { + permissionMap.put(permission, new HashSet<>()); + } + + // Store permissions at the resource level + for (Map.Entry> entry : resourceUserPermissionsMap.entrySet()) { + for (String permission : entry.getValue()) { + permissionMap.get(permission).add(entry.getKey()); + } + } + + // List of correspondence of study permissions + Map studyPermissionsToResourcePermissionsMap = new HashMap<>(); + for (String resourcePermission : resourcePermissions) { + String studyPermission; + if (!"NONE".equals(resourcePermission)) { + studyPermission = resource.toStudyPermission(resourcePermission); + } else { + studyPermission = "NONE"; + } + studyPermissionsToResourcePermissionsMap.put(studyPermission, resourcePermission); + } + + // Iterate and only store permissions at the study level if no permissions were given at the resource level + for (Map.Entry> entry : studyUserPermissionsMap.entrySet()) { + String userId = entry.getKey(); + Set studyPermissions = entry.getValue(); + if (resourceUserPermissionsMap.get(userId).isEmpty()) { + // Loop over all study permissions given to the user + for (String studyPermission : studyPermissions) { + if (studyPermissionsToResourcePermissionsMap.containsKey(studyPermission)) { + String resourcePermission = studyPermissionsToResourcePermissionsMap.get(studyPermission); + permissionMap.get(resourcePermission).add(userId); + } + } + } + } + + Set usersWithNoAccess = new HashSet<>(studyUserPermissionsMap.keySet()); + usersWithNoAccess.removeAll(adminUsers); + // Generate ACL object + List permissionList = new ArrayList<>(resourcePermissions.size()); + for (String resourcePermission : resourcePermissions) { + Set userIdSet = permissionMap.get(resourcePermission); + if (!"NONE".equals(resourcePermission)) { + // Remove users with access from usersWithNoAccess set + usersWithNoAccess.removeAll(userIdSet); + // Add admin users to users with this permission + userIdSet.addAll(adminUsers); + + permissionList.add(new Acl.Permission(resourcePermission, new ArrayList<>(userIdSet))); + } + } + permissionList.add(new Acl.Permission("NONE", new ArrayList<>(usersWithNoAccess))); + + return new Acl(id, resource.name(), permissionList, TimeUtils.getDate().getTime()); + } + + private Map> extractUserPermissionsMap(Map> groupsMap, Document document, + boolean simplifyPermissions) { + Set allUsers = groupsMap.get(ParamConstants.MEMBERS_GROUP); + + // Map of userId - List of permissions + Map> userPermissionsMap = new HashMap<>(); + for (String userId : allUsers) { + userPermissionsMap.put(userId, new HashSet<>()); + } + + // Group ACLs + List aclList = document.getList(QueryParams.ACL.key(), String.class); + if (CollectionUtils.isNotEmpty(aclList)) { + List personalAcls = new ArrayList<>(aclList.size()); + List groupAcls = new ArrayList<>(aclList.size()); + List anonymousAcls = new ArrayList<>(aclList.size()); + + for (String acl : aclList) { + if (acl.startsWith("@")) { + groupAcls.add(acl); + } else if (acl.startsWith(ParamConstants.ANONYMOUS_USER_ID + INTERNAL_DELIMITER)) { + anonymousAcls.add(acl); + } else { + personalAcls.add(acl); + } + } + + Set userIdsWithPermissions = new HashSet<>(); + + // Personal ACLs + for (String acl : personalAcls) { + String[] split = acl.split(INTERNAL_DELIMITER, 2); + String userId = split[0]; + String permission = split[1]; + if (!userPermissionsMap.containsKey(userId)) { + throw new IllegalStateException("User id '" + userId + "' with permissions was not found in the '" + + ParamConstants.MEMBERS_GROUP + "' group."); + } + userIdsWithPermissions.add(userId); + userPermissionsMap.get(userId).add(permission); + } + + // Anonymous ACLs + List anonymousPermissions = new ArrayList<>(anonymousAcls.size()); + for (String acl : anonymousAcls) { + String[] split = acl.split(INTERNAL_DELIMITER, 2); + String permission = split[1]; + anonymousPermissions.add(permission); + } + // Assign anonymous permissions + if (!anonymousPermissions.isEmpty()) { + for (Map.Entry> tmpEntry : userPermissionsMap.entrySet()) { + // Only add permissions if "simplifyPermissions" or if the user hasn't been given any acls personally + if (simplifyPermissions || tmpEntry.getValue().isEmpty()) { + tmpEntry.getValue().addAll(anonymousPermissions); + } + } + } + + // Group ACLs + Map> groupPermissions = new HashMap<>(); + for (String acl : groupAcls) { + String[] split = acl.split(INTERNAL_DELIMITER, 2); + String groupId = split[0]; + String permission = split[1]; + + if (!groupPermissions.containsKey(groupId)) { + groupPermissions.put(groupId, new LinkedList<>()); + } + groupPermissions.get(groupId).add(permission); + } + // Assign group permissions + for (Map.Entry> tmpEntry : groupPermissions.entrySet()) { + String groupId = tmpEntry.getKey(); + List tmpPermissionList = tmpEntry.getValue(); + + for (String userId : groupsMap.get(groupId)) { + if (simplifyPermissions || !userIdsWithPermissions.contains(userId)) { + userPermissionsMap.get(userId).addAll(tmpPermissionList); + } + } + } + } + return userPermissionsMap; + } + + private static Map> getGroupUsersMap(Document studyDocument) { + // Generate a map of group - set of userIds + Map> groupsMap = new HashMap<>(); + List groups = studyDocument.getList(StudyDBAdaptor.QueryParams.GROUPS.key(), Document.class); + for (Document group : groups) { + String groupId = group.getString(ID); + List userIds = group.getList("userIds", String.class); + groupsMap.put(groupId, new HashSet<>(userIds)); + } + + return groupsMap; + } + static class EntryPermission { /** * Entry id. @@ -383,7 +576,7 @@ public OpenCGAResult removeFromStudy(long studyId, String member, Enums.Resou @Override public OpenCGAResult setToMembers(long studyId, List members, List aclParams) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + throws CatalogException { return runTransaction(clientSession -> { long startTime = startQuery(); @@ -394,8 +587,8 @@ public OpenCGAResult setToMembers(long studyId, List members, List members, List studyIds, List members, List permissions) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult setToMembers(List studyIds, List members, List permissions) throws CatalogException { return runTransaction(clientSession -> { long startTime = startQuery(); for (Long studyId : studyIds) { @@ -457,7 +649,7 @@ private void setToMembers(List resourceIds, List members, List members, List aclParams) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + throws CatalogException { return runTransaction(clientSession -> { long startTime = startQuery(); addToMembersGroupInStudy(studyId, members, clientSession); @@ -466,8 +658,8 @@ public OpenCGAResult addToMembers(long studyId, List members, List resourceIds, List members, List studyIds, List members, List permissions) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult addToMembers(List studyIds, List members, List permissions) throws CatalogException { return runTransaction((clientSession) -> { long startTime = startQuery(); for (Long studyId : studyIds) { @@ -522,19 +713,20 @@ private void addToMembersGroupInStudy(long studyId, List members, Client .collect(Collectors.toList()); if (CollectionUtils.isNotEmpty(userList)) { // We first add the member to the @members group in case they didn't belong already - dbAdaptorFactory.getCatalogStudyDBAdaptor().addUsersToGroup(clientSession, studyId, CatalogAuthorizationManager.MEMBERS_GROUP, - userList); + dbAdaptorFactory.getCatalogStudyDBAdaptor() + .addUsersToGroup(clientSession, studyId, CatalogAuthorizationManager.MEMBERS_GROUP, userList); } } @Override public OpenCGAResult removeFromMembers(List members, List aclParams) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + throws CatalogException { return runTransaction(clientSession -> { long startTime = startQuery(); for (AuthorizationManager.CatalogAclParams aclParam : aclParams) { - removeFromMembers(clientSession, aclParam.getIds(), members, aclParam.getPermissions(), aclParam.getResource()); + removeFromMembers(clientSession, aclParam.getIds(), members, aclParam.getPermissions(), + aclParam.getResource()); } return endWrite(startTime, aclParams.get(0).getIds().size(), aclParams.get(0).getIds().size(), null); @@ -570,8 +762,7 @@ private void removeFromMembers(ClientSession clientSession, List resourceI } @Override - public OpenCGAResult resetMembersFromAllEntries(long studyId, List members) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult resetMembersFromAllEntries(long studyId, List members) throws CatalogException { if (members == null || members.isEmpty()) { throw new CatalogDBException("Missing 'members' array."); } @@ -589,7 +780,7 @@ public OpenCGAResult resetMembersFromAllEntries(long studyId, List membe removePermissions(clientSession, studyId, members, Enums.Resource.DISEASE_PANEL); removePermissions(clientSession, studyId, members, Enums.Resource.FAMILY); removePermissions(clientSession, studyId, members, Enums.Resource.CLINICAL_ANALYSIS); - removeFromMembers(clientSession, Arrays.asList(studyId), members, null, Enums.Resource.STUDY); + removeFromMembers(clientSession, Collections.singletonList(studyId), members, null, Enums.Resource.STUDY); return endWrite(tmpStartTime, -1, -1, null); }); @@ -597,14 +788,13 @@ public OpenCGAResult resetMembersFromAllEntries(long studyId, List membe // TODO: Make this method transactional @Override - public OpenCGAResult setAcls(List resourceIds, AclEntryList acls, Enums.Resource resource) - throws CatalogDBException { + public OpenCGAResult setAcls(List resourceIds, AclEntryList acls, Enums.Resource resource) throws CatalogDBException { validateEntry(resource); for (long resourceId : resourceIds) { // Get current permissions for resource and override with new ones set for members (already existing or not) - Map>> currentPermissions = internalGet(resourceId, Collections.emptyList(), resource) - .getPermissions(); + Map>> currentPermissions = internalGet(resourceId, Collections.emptyList(), + resource).getPermissions(); for (AclEntry acl : acls.getAcl()) { // We add the NONE permission by default so when a user is removed some permissions (not reset), the NONE permission remains List permissions = acl.getPermissions().stream().map(Enum::name).collect(Collectors.toList()); @@ -633,8 +823,8 @@ public OpenCGAResult setAcls(List resourceIds, AclEntryList acls, Enums return OpenCGAResult.empty(); } - private void setMembersHaveInternalPermissionsDefined(long studyId, List members, Enums.Resource resource, - ClientSession clientSession) { + private void setMembersHaveInternalPermissionsDefined(ClientSession clientSession, long studyId, List members, + Enums.Resource resource) throws CatalogDBException { Document queryDocument = new Document() .append(PRIVATE_UID, studyId); @@ -691,61 +881,62 @@ public OpenCGAResult removePermissionRuleAndRemovePermissions(Study study, Strin .append(PERMISSION_RULES_APPLIED, permissionRuleId); QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(QueryParams.ACL.key(), QueryParams.USER_DEFINED_ACLS.key(), PERMISSION_RULES_APPLIED, PRIVATE_UID)); - MongoDBIterator iterator = dbCollectionMap.get(entry.getResource()).get(0).iterator(query, options); - while (iterator.hasNext()) { - Document myDocument = iterator.next(); - Set effectivePermissions = new HashSet<>(); - Set manualPermissions = new HashSet<>(); - Set permissionRulesApplied = new HashSet<>(); - - List currentAclList = (List) myDocument.get(QueryParams.ACL.key()); - List currentManualAclList = (List) myDocument.get(QueryParams.USER_DEFINED_ACLS.key()); - List currentPermissionRulesApplied = (List) myDocument.get(PERMISSION_RULES_APPLIED); - - // TODO: Control that if there are no more permissions set for a user or group, we should also remove the NONE permission - // Remove permissions from the permission rule - for (String permission : currentAclList) { - if (!permissionsToRemove.contains(permission)) { - effectivePermissions.add(permission); + try (MongoDBIterator iterator = getMainCollection(entry.getResource()).iterator(query, options)) { + while (iterator.hasNext()) { + Document myDocument = iterator.next(); + Set effectivePermissions = new HashSet<>(); + Set manualPermissions = new HashSet<>(); + Set permissionRulesApplied = new HashSet<>(); + + List currentAclList = (List) myDocument.get(QueryParams.ACL.key()); + List currentManualAclList = (List) myDocument.get(QueryParams.USER_DEFINED_ACLS.key()); + List currentPermissionRulesApplied = (List) myDocument.get(PERMISSION_RULES_APPLIED); + + // TODO: Control that if there are no more permissions set for a user or group, we should also remove the NONE permission + // Remove permissions from the permission rule + for (String permission : currentAclList) { + if (!permissionsToRemove.contains(permission)) { + effectivePermissions.add(permission); + } } - } - // Remove permissions from the permission rule from the internal manual permissions list - if (currentManualAclList != null) { - for (String permission : currentManualAclList) { - if (!permissionsToRemove.contains(permission)) { - manualPermissions.add(permission); + // Remove permissions from the permission rule from the internal manual permissions list + if (currentManualAclList != null) { + for (String permission : currentManualAclList) { + if (!permissionsToRemove.contains(permission)) { + manualPermissions.add(permission); + } } } - } - for (String tmpPermissionRuleId : currentPermissionRulesApplied) { - // We apply the rest of permission rules except the one to be deleted - if (!tmpPermissionRuleId.equals(permissionRuleId)) { - PermissionRule tmpPermissionRule = permissionRuleMap.get(tmpPermissionRuleId); - List tmpPermissionList = new ArrayList<>(tmpPermissionRule.getPermissions()); - tmpPermissionList.add("NONE"); - List permissionArray = createPermissionArray(tmpPermissionRule.getMembers(), tmpPermissionList); + for (String tmpPermissionRuleId : currentPermissionRulesApplied) { + // We apply the rest of permission rules except the one to be deleted + if (!tmpPermissionRuleId.equals(permissionRuleId)) { + PermissionRule tmpPermissionRule = permissionRuleMap.get(tmpPermissionRuleId); + List tmpPermissionList = new ArrayList<>(tmpPermissionRule.getPermissions()); + tmpPermissionList.add("NONE"); + List permissionArray = createPermissionArray(tmpPermissionRule.getMembers(), tmpPermissionList); - effectivePermissions.addAll(permissionArray); - permissionRulesApplied.add(tmpPermissionRuleId); + effectivePermissions.addAll(permissionArray); + permissionRulesApplied.add(tmpPermissionRuleId); + } } - } - Document tmpQuery = new Document() - .append(PRIVATE_UID, myDocument.get(PRIVATE_UID)) - .append(PRIVATE_STUDY_UID, study.getUid()); + Document tmpQuery = new Document() + .append(PRIVATE_UID, myDocument.get(PRIVATE_UID)) + .append(PRIVATE_STUDY_UID, study.getUid()); - Document update = new Document("$set", new Document() - .append(QueryParams.ACL.key(), effectivePermissions) - .append(QueryParams.USER_DEFINED_ACLS.key(), manualPermissions) - .append(PERMISSION_RULES_APPLIED, permissionRulesApplied)); + Document update = new Document("$set", new Document() + .append(QueryParams.ACL.key(), effectivePermissions) + .append(QueryParams.USER_DEFINED_ACLS.key(), manualPermissions) + .append(PERMISSION_RULES_APPLIED, permissionRulesApplied)); - logger.debug("Remove permission rule id and permissions from {}: Query {}, Update {}", entry, tmpQuery.toBsonDocument(), - update.toBsonDocument()); - DataResult result = update(null, tmpQuery, update, entry.getResource()); - if (result.getNumUpdated() == 0) { - throw new CatalogException("Could not update and remove permission rule from entry " + myDocument.get(PRIVATE_UID)); + logger.debug("Remove permission rule id and permissions from {}: Query {}, Update {}", entry, tmpQuery.toBsonDocument(), + update.toBsonDocument()); + DataResult result = update(null, tmpQuery, update, entry.getResource()); + if (result.getNumUpdated() == 0) { + throw new CatalogException("Could not update and remove permission rule from entry " + myDocument.get(PRIVATE_UID)); + } } } @@ -776,57 +967,58 @@ public OpenCGAResult removePermissionRuleAndRestorePermissions(Study study, Stri .append(PERMISSION_RULES_APPLIED, permissionRuleId); QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(QueryParams.ACL.key(), QueryParams.USER_DEFINED_ACLS.key(), PERMISSION_RULES_APPLIED, PRIVATE_UID)); - MongoDBIterator iterator = dbCollectionMap.get(entry.getResource()).get(0).iterator(query, options); - while (iterator.hasNext()) { - Document myDocument = iterator.next(); - Set effectivePermissions = new HashSet<>(); - Set permissionRulesApplied = new HashSet<>(); - - List currentAclList = (List) myDocument.get(QueryParams.ACL.key()); - List currentManualAclList = (List) myDocument.get(QueryParams.USER_DEFINED_ACLS.key()); - List currentPermissionRulesApplied = (List) myDocument.get(PERMISSION_RULES_APPLIED); - - // TODO: Control that if there are no more permissions set for a user or group, we should also remove the NONE permission - // Remove permissions from the permission rule - for (String permission : currentAclList) { - if (!permissionsToRemove.contains(permission)) { - effectivePermissions.add(permission); + try (MongoDBIterator iterator = getMainCollection(entry.getResource()).iterator(query, options)) { + while (iterator.hasNext()) { + Document myDocument = iterator.next(); + Set effectivePermissions = new HashSet<>(); + Set permissionRulesApplied = new HashSet<>(); + + List currentAclList = (List) myDocument.get(QueryParams.ACL.key()); + List currentManualAclList = (List) myDocument.get(QueryParams.USER_DEFINED_ACLS.key()); + List currentPermissionRulesApplied = (List) myDocument.get(PERMISSION_RULES_APPLIED); + + // TODO: Control that if there are no more permissions set for a user or group, we should also remove the NONE permission + // Remove permissions from the permission rule + for (String permission : currentAclList) { + if (!permissionsToRemove.contains(permission)) { + effectivePermissions.add(permission); + } } - } - // Restore manual permissions - if (currentManualAclList != null) { - for (String permission : currentManualAclList) { - effectivePermissions.add(permission); + // Restore manual permissions + if (currentManualAclList != null) { + for (String permission : currentManualAclList) { + effectivePermissions.add(permission); + } } - } - for (String tmpPermissionRuleId : currentPermissionRulesApplied) { - // We apply the rest of permission rules except the one to be deleted - if (!tmpPermissionRuleId.equals(permissionRuleId)) { - PermissionRule tmpPermissionRule = permissionRuleMap.get(tmpPermissionRuleId); - List tmpPermissionList = new ArrayList<>(tmpPermissionRule.getPermissions()); - tmpPermissionList.add("NONE"); - List permissionArray = createPermissionArray(tmpPermissionRule.getMembers(), tmpPermissionList); + for (String tmpPermissionRuleId : currentPermissionRulesApplied) { + // We apply the rest of permission rules except the one to be deleted + if (!tmpPermissionRuleId.equals(permissionRuleId)) { + PermissionRule tmpPermissionRule = permissionRuleMap.get(tmpPermissionRuleId); + List tmpPermissionList = new ArrayList<>(tmpPermissionRule.getPermissions()); + tmpPermissionList.add("NONE"); + List permissionArray = createPermissionArray(tmpPermissionRule.getMembers(), tmpPermissionList); - effectivePermissions.addAll(permissionArray); - permissionRulesApplied.add(tmpPermissionRuleId); + effectivePermissions.addAll(permissionArray); + permissionRulesApplied.add(tmpPermissionRuleId); + } } - } - Document tmpQuery = new Document() - .append(PRIVATE_UID, myDocument.get(PRIVATE_UID)) - .append(PRIVATE_STUDY_UID, study.getUid()); + Document tmpQuery = new Document() + .append(PRIVATE_UID, myDocument.get(PRIVATE_UID)) + .append(PRIVATE_STUDY_UID, study.getUid()); - Document update = new Document("$set", new Document() - .append(QueryParams.ACL.key(), effectivePermissions) - .append(PERMISSION_RULES_APPLIED, permissionRulesApplied)); + Document update = new Document("$set", new Document() + .append(QueryParams.ACL.key(), effectivePermissions) + .append(PERMISSION_RULES_APPLIED, permissionRulesApplied)); - logger.debug("Remove permission rule id and restoring permissions from {}: Query {}, Update {}", entry, - tmpQuery.toBsonDocument(), update.toBsonDocument()); - DataResult result = update(null, tmpQuery, update, entry.getResource()); - if (result.getNumUpdated() == 0) { - throw new CatalogException("Could not update and remove permission rule from entry " + myDocument.get(PRIVATE_UID)); + logger.debug("Remove permission rule id and restoring permissions from {}: Query {}, Update {}", entry, + tmpQuery.toBsonDocument(), update.toBsonDocument()); + DataResult result = update(null, tmpQuery, update, entry.getResource()); + if (result.getNumUpdated() == 0) { + throw new CatalogException("Could not update and remove permission rule from entry " + myDocument.get(PRIVATE_UID)); + } } } @@ -909,7 +1101,8 @@ private Bson parseQuery(Query query, Document rawQuery, Enums.Resource entry) th } } - private void removePermissions(ClientSession clientSession, long studyId, List users, Enums.Resource resource) { + private void removePermissions(ClientSession clientSession, long studyId, List users, Enums.Resource resource) + throws CatalogDBException { List permissions = getFullPermissions(resource); List removePermissions = createPermissionArray(users, permissions); @@ -956,14 +1149,59 @@ private List createPermissionArray(List members, List pe return myPermissions; } - private DataResult update(ClientSession clientSession, Bson query, Bson update, Enums.Resource resource) { - List collections = dbCollectionMap.get(resource); + private DataResult update(ClientSession clientSession, Bson query, Bson update, Enums.Resource resource) throws CatalogDBException { QueryOptions options = new QueryOptions(MongoDBCollection.MULTI, true); - DataResult result = collections.get(0).update(clientSession, query, update, options); - if (collections.size() == 2) { - collections.get(1).update(clientSession, query, update, options); + DataResult result = getMainCollection(resource).update(clientSession, query, update, options); + if (hasArchiveCollection(resource)) { + getArchiveCollection(resource).update(clientSession, query, update, options); } return result; } + + private MongoDBCollection getMainCollection(Enums.Resource resource) throws CatalogDBException { + switch (resource) { + case STUDY: + return dbAdaptorFactory.getCatalogStudyDBAdaptor().getStudyCollection(); + case COHORT: + return dbAdaptorFactory.getCatalogCohortDBAdaptor().getCohortCollection(); + case INDIVIDUAL: + return dbAdaptorFactory.getCatalogIndividualDBAdaptor().getIndividualCollection(); + case JOB: + return dbAdaptorFactory.getCatalogJobDBAdaptor().getJobCollection(); + case FILE: + return dbAdaptorFactory.getCatalogFileDBAdaptor().getCollection(); + case SAMPLE: + return dbAdaptorFactory.getCatalogSampleDBAdaptor().getCollection(); + case DISEASE_PANEL: + return dbAdaptorFactory.getCatalogPanelDBAdaptor().getPanelCollection(); + case FAMILY: + return dbAdaptorFactory.getCatalogFamilyDBAdaptor().getCollection(); + case CLINICAL_ANALYSIS: + case CLINICAL: + return dbAdaptorFactory.getClinicalAnalysisDBAdaptor().getCollection(); + default: + throw new CatalogDBException("Unexpected resource '" + resource + "' parameter received."); + } + } + + private MongoDBCollection getArchiveCollection(Enums.Resource resource) throws CatalogDBException { + switch (resource) { + case INDIVIDUAL: + return dbAdaptorFactory.getCatalogIndividualDBAdaptor().getIndividualArchiveCollection(); + case SAMPLE: + return dbAdaptorFactory.getCatalogSampleDBAdaptor().getArchiveSampleCollection(); + case DISEASE_PANEL: + return dbAdaptorFactory.getCatalogPanelDBAdaptor().getPanelArchiveCollection(); + case FAMILY: + return dbAdaptorFactory.getCatalogFamilyDBAdaptor().getArchiveFamilyCollection(); + default: + throw new CatalogDBException("Unexpected resource '" + resource + "' parameter received."); + } + } + + private boolean hasArchiveCollection(Enums.Resource resource) { + return resource == Enums.Resource.INDIVIDUAL || resource == Enums.Resource.SAMPLE || resource == Enums.Resource.DISEASE_PANEL + || resource == Enums.Resource.FAMILY; + } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBUtils.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBUtils.java index 72b5d841744..fb8a3952f50 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBUtils.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBUtils.java @@ -25,7 +25,6 @@ import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.core.api.ParamConstants; -import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.study.StudyPermissions; @@ -41,8 +40,7 @@ public class AuthorizationMongoDBUtils { static final String OPENCGA = "opencga"; - static final String PRIVATE_OWNER_ID = "_ownerId"; - private static final String PRIVATE_ACL = "_acl"; + public static final String PRIVATE_ACL = "_acl"; private static final String VARIABLE_SETS = "variableSets"; private static final String ANNOTATION_SETS = AnnotationMongoDBAdaptor.AnnotationSetParams.ANNOTATION_SETS.key(); @@ -58,12 +56,19 @@ public class AuthorizationMongoDBUtils { private static final Pattern REGISTERED_USERS_PATTERN = Pattern.compile("^" + REGISTERED_USERS); private static final Pattern ANONYMOUS_PATTERN = Pattern.compile("^\\" + ANONYMOUS); - public static boolean checkCanViewStudy(Document study, String user) { - // 0. If the user corresponds with the owner, we don't have to check anything else - if (study.getString(PRIVATE_OWNER_ID).equals(user)) { - return true; - } - if (OPENCGA.equals(user)) { + /** + * Checks whether the user belongs to the administration organization. + * + * @param organizationId Organization the user wants to access. + * @param user User id. + * @return True if the user is a super administrator, False otherwise. + */ + public static boolean isOpencgaAdministrator(String organizationId, String user) { + return ParamConstants.ADMIN_ORGANIZATION.equals(organizationId) || user.startsWith(ParamConstants.ADMIN_ORGANIZATION + ":"); + } + + public static boolean checkCanViewStudy(String organizationId, Document study, String user) { + if (isOpencgaAdministrator(organizationId, user)) { return true; } // If user does not exist in the members group, the user will not have any permission @@ -73,15 +78,8 @@ public static boolean checkCanViewStudy(Document study, String user) { return false; } - public static boolean checkStudyPermission(Document study, String user, String studyPermission) { - // 0. If the user corresponds with the owner, we don't have to check anything else - if (study.getString(PRIVATE_OWNER_ID).equals(user)) { - return true; - } - if (OPENCGA.equals(user)) { - return true; - } - if (getAdminUsers(study).contains(user)) { + public static boolean checkStudyPermission(String organizationId, Document study, String user, String studyPermission) { + if (isAtLeastOrganizationOwnerOrStudyAdmin(organizationId, study, user)) { return true; } @@ -101,9 +99,20 @@ public static boolean checkStudyPermission(Document study, String user, String s } } + public static boolean isAtLeastOrganizationOwnerOrStudyAdmin(String organizationId, Document study, String user) { + if (isOpencgaAdministrator(organizationId, user)) { + return true; + } + if (getAdminUsers(study).contains(user)) { + return true; + } + return false; + } + /** * Removes annotation sets from results if the user does not have the proper permissions. * + * @param organizationId Organization id. * @param study study document. * @param entry Annotable document entry. * @param user user. @@ -111,16 +120,12 @@ public static boolean checkStudyPermission(Document study, String user, String s * @param entryPermission entry permission to check. * @return the document modified. */ - public static Document filterAnnotationSets(Document study, Document entry, String user, String studyPermission, + public static Document filterAnnotationSets(String organizationId, Document study, Document entry, String user, String studyPermission, String entryPermission) { if (study == null || entry == null || user == null) { return entry; } - // If the user corresponds with the owner, we don't have to check anything else - if (study.getString(PRIVATE_OWNER_ID).equals(user)) { - return entry; - } if (OPENCGA.equals(user)) { return entry; } @@ -138,8 +143,8 @@ public static Document filterAnnotationSets(Document study, Document entry, Stri entry.put(ANNOTATION_SETS, Collections.emptyList()); } else { // Check if the user has the CONFIDENTIAL PERMISSION - boolean confidential = - checkStudyPermission(study, user, StudyPermissions.Permissions.CONFIDENTIAL_VARIABLE_SET_ACCESS.toString()); + boolean confidential = checkStudyPermission(organizationId, study, user, + StudyPermissions.Permissions.CONFIDENTIAL_VARIABLE_SET_ACCESS.toString()); if (!confidential) { // If the user does not have the confidential permission, we will have to remove those annotation sets coming from // confidential variable sets @@ -194,35 +199,18 @@ private static int getPermissionType(Enums.Resource resource) throws CatalogPara /** * If query contains {@link ParamConstants#ACL_PARAM}, it will parse the value to generate the corresponding mongo query documents. * - * @param study Queried study document. - * @param query Original query. - * @param resource Affected resource. - * @param user User performing the query. - * @return A list of documents to satisfy the ACL query. - * @throws CatalogDBException when there is a DB error. - * @throws CatalogParameterException if there is any formatting error. - * @throws CatalogAuthorizationException if the user is not authorised to perform the query. - */ - public static List parseAclQuery(Document study, Query query, Enums.Resource resource, String user) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { - return parseAclQuery(study, query, resource, user, null); - } - - /** - * If query contains {@link ParamConstants#ACL_PARAM}, it will parse the value to generate the corresponding mongo query documents. - * - * @param study Queried study document. - * @param query Original query. - * @param resource Affected resource. - * @param user User performing the query. - * @param configuration Configuration object. + * @param study Queried study document. + * @param query Original query. + * @param resource Affected resource. + * @param user User performing the query. + * @param simplifyPermissions Boolean indicating whether permission check can be simplified. * @return A list of documents to satisfy the ACL query. * @throws CatalogDBException when there is a DB error. * @throws CatalogParameterException if there is any formatting error. * @throws CatalogAuthorizationException if the user is not authorised to perform the query. */ public static List parseAclQuery(Document study, Query query, Enums.Resource resource, String user, - Configuration configuration) + boolean simplifyPermissions) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { List aclDocuments = new LinkedList<>(); if (!query.containsKey(ParamConstants.ACL_PARAM)) { @@ -241,14 +229,10 @@ public static List parseAclQuery(Document study, Query query, Enums.Re List permissions = Arrays.asList(userPermission[1].split(",")); // If user is not checking its own permissions and it is not the owner or admin of the study, we fail - if (!user.equals(affectedUser) && !study.getString(PRIVATE_OWNER_ID).equals(user) && !getAdminUsers(study).contains(user)) { + if (!user.equals(affectedUser) && !getAdminUsers(study).contains(user)) { throw new CatalogAuthorizationException("Only study owners or admins are authorised to see other user's permissions."); } - // 0. If the user is the admin or corresponds with the owner, we don't have to check anything else - if (study.getString(PRIVATE_OWNER_ID).equals(affectedUser)) { - return aclDocuments; - } if (OPENCGA.equals(affectedUser)) { return aclDocuments; } @@ -262,11 +246,6 @@ public static List parseAclQuery(Document study, Query query, Enums.Re + study.getString(StudyDBAdaptor.QueryParams.ID.key())); } - boolean simplifyPermissionCheck = false; - if (configuration != null && configuration.getOptimizations() != null) { - simplifyPermissionCheck = configuration.getOptimizations().isSimplifyPermissions(); - } - boolean isAnonymousPresent = false; boolean isRegisteredUsersPresent = false; List groups; @@ -300,7 +279,7 @@ public static List parseAclQuery(Document study, Query query, Enums.Re } Document queryDocument = getAuthorisedEntries(affectedUser, groups, permission, isRegisteredUsersPresent, isAnonymousPresent, - simplifyPermissionCheck); + simplifyPermissions); if (hasStudyPermissions) { // The user has permissions defined globally, so we also have to check the entries where the user/groups/members/* have no // permissions defined as the user will also be allowed to see them @@ -315,23 +294,14 @@ public static List parseAclQuery(Document study, Query query, Enums.Re return aclDocuments; } - public static Document getQueryForAuthorisedEntries(Document study, String user, String permission, Enums.Resource resource) - throws CatalogAuthorizationException, CatalogParameterException { - return getQueryForAuthorisedEntries(study, user, permission, resource, null); - } - public static Document getQueryForAuthorisedEntries(Document study, String user, String permission, Enums.Resource resource, - Configuration configuration) + boolean simplifyPermissions) throws CatalogAuthorizationException, CatalogParameterException { if (StringUtils.isEmpty(user)) { return new Document(); } - // 0. If the user is the admin or corresponds with the owner, we don't have to check anything else - if (study.getString(PRIVATE_OWNER_ID).equals(user)) { - return new Document(); - } - if (OPENCGA.equals(user)) { + if (OPENCGA.equals(user) || ParamConstants.OPENCGA_USER_FQN.equals(user)) { return new Document(); } if (getAdminUsers(study).contains(user)) { @@ -344,11 +314,6 @@ public static Document getQueryForAuthorisedEntries(Document study, String user, + study.getString(StudyDBAdaptor.QueryParams.ID.key())); } - boolean simplifyPermissionCheck = false; - if (configuration != null && configuration.getOptimizations() != null) { - simplifyPermissionCheck = configuration.getOptimizations().isSimplifyPermissions(); - } - String studyPermission = StudyPermissions.Permissions.getStudyPermission(permission, getPermissionType(resource)).name(); // 0. Check if anonymous has any permission defined (just for performance) @@ -377,8 +342,8 @@ public static Document getQueryForAuthorisedEntries(Document study, String user, } Document queryDocument = getAuthorisedEntries(user, groups, permission, isRegisteredUsersPresent, isAnonymousPresent, - simplifyPermissionCheck); - if (hasStudyPermissions && !simplifyPermissionCheck) { + simplifyPermissions); + if (hasStudyPermissions && !simplifyPermissions) { // The user has permissions defined globally, so we also have to check the entries where the user/groups/members/* have no // permissions defined as the user will also be allowed to see them queryDocument = new Document("$or", Arrays.asList( diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java index 078554b4737..7a359e5a7eb 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java @@ -31,7 +31,10 @@ import org.opencb.commons.datastore.core.*; import org.opencb.commons.datastore.mongodb.MongoDBCollection; import org.opencb.commons.datastore.mongodb.MongoDBIterator; -import org.opencb.opencga.catalog.db.api.*; +import org.opencb.opencga.catalog.db.api.ClinicalAnalysisDBAdaptor; +import org.opencb.opencga.catalog.db.api.DBIterator; +import org.opencb.opencga.catalog.db.api.FileDBAdaptor; +import org.opencb.opencga.catalog.db.api.InterpretationDBAdaptor; import org.opencb.opencga.catalog.db.mongodb.converters.ClinicalAnalysisConverter; import org.opencb.opencga.catalog.db.mongodb.iterators.ClinicalAnalysisCatalogMongoDBIterator; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; @@ -80,16 +83,23 @@ public class ClinicalAnalysisMongoDBAdaptor extends AnnotationMongoDBAdaptor va try { return runTransaction(clientSession -> transactionalUpdate(clientSession, result.first(), parameters, variableSetList, clinicalAuditList, queryOptions)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update clinical analysis {}: {}", clinicalAnalysisId, e.getMessage(), e); throw new CatalogDBException("Could not update clinical analysis " + clinicalAnalysisId + ": " + e.getMessage(), e.getCause()); } @@ -284,61 +294,63 @@ OpenCGAResult transactionalUpdate(ClientSession clientSession, ClinicalAnalysis String clinicalAnalysisId = clinical.getId(); long clinicalAnalysisUid = clinical.getUid(); - DataResult result = updateAnnotationSets(clientSession, clinical.getStudyUid(), clinicalAnalysisUid, parameters, variableSetList, - queryOptions, false); + Query tmpQuery = new Query() + .append(STUDY_UID.key(), clinical.getStudyUid()) + .append(QueryParams.UID.key(), clinicalAnalysisUid); + Bson bsonQuery = parseQuery(tmpQuery); + return versionedMongoDBAdaptor.update(clientSession, bsonQuery, (entryList) -> { + DataResult result = updateAnnotationSets(clientSession, clinical.getStudyUid(), clinicalAnalysisUid, parameters, + variableSetList, queryOptions, false); - // Perform the update - Query query = new Query(QueryParams.UID.key(), clinicalAnalysisUid); - UpdateDocument updateDocument = parseAndValidateUpdateParams(parameters, clinicalAuditList, query, queryOptions); + // Perform the update + UpdateDocument updateDocument = parseAndValidateUpdateParams(parameters, clinicalAuditList, tmpQuery, queryOptions); - Document updateOperation = updateDocument.toFinalUpdateDocument(); - List events = new ArrayList<>(); - if (!updateOperation.isEmpty() || !updateDocument.getNestedUpdateList().isEmpty()) { - DataResult update; + Document updateOperation = updateDocument.toFinalUpdateDocument(); + List events = new ArrayList<>(); + if (!updateOperation.isEmpty() || !updateDocument.getNestedUpdateList().isEmpty()) { + DataResult update; - if (!updateOperation.isEmpty()) { - Bson bsonQuery = Filters.eq(PRIVATE_UID, clinicalAnalysisUid); + if (!updateOperation.isEmpty()) { + logger.debug("Update clinical analysis. Query: {}, Update: {}", bsonQuery.toBsonDocument(), updateDocument); + update = clinicalCollection.update(clientSession, bsonQuery, updateOperation, null); - logger.debug("Update clinical analysis. Query: {}, Update: {}", bsonQuery.toBsonDocument(), updateDocument); - update = clinicalCollection.update(clientSession, bsonQuery, updateOperation, null); + if (update.getNumMatches() == 0) { + throw CatalogDBException.uidNotFound("Clinical Analysis", clinicalAnalysisUid); + } + if (update.getNumUpdated() == 0) { + events.add(new Event(Event.Type.WARNING, clinicalAnalysisId, "Clinical Analysis was already updated")); + } - if (update.getNumMatches() == 0) { - throw CatalogDBException.uidNotFound("Clinical Analysis", clinicalAnalysisUid); - } - if (update.getNumUpdated() == 0) { - events.add(new Event(Event.Type.WARNING, clinicalAnalysisId, "Clinical Analysis was already updated")); - } + if (parameters.getBoolean(LOCKED.key())) { + // Propagate locked value to Interpretations + logger.debug("Propagating case lock to all the Interpretations"); + dbAdaptorFactory.getInterpretationDBAdaptor().propagateLockedFromClinicalAnalysis(clientSession, clinical, + parameters.getBoolean(LOCKED.key())); + } - if (parameters.getBoolean(LOCKED.key())) { - // Propagate locked value to Interpretations - logger.debug("Propagating case lock to all the Interpretations"); - dbAdaptorFactory.getInterpretationDBAdaptor().propagateLockedFromClinicalAnalysis(clientSession, clinical, - parameters.getBoolean(LOCKED.key())); + logger.debug("Clinical Analysis {} successfully updated", clinicalAnalysisId); } - logger.debug("Clinical Analysis {} successfully updated", clinicalAnalysisId); - } - - if (!updateDocument.getNestedUpdateList().isEmpty()) { - for (NestedArrayUpdateDocument nestedDocument : updateDocument.getNestedUpdateList()) { + if (!updateDocument.getNestedUpdateList().isEmpty()) { + // Nested documents are used to update reports + for (NestedArrayUpdateDocument nestedDocument : updateDocument.getNestedUpdateList()) { + Bson tmpBsonQuery = parseQuery(nestedDocument.getQuery().append(QueryParams.UID.key(), clinicalAnalysisUid)); + logger.debug("Update nested element from Clinical Analysis. Query: {}, Update: {}", + tmpBsonQuery.toBsonDocument(), nestedDocument.getSet()); + update = clinicalCollection.update(clientSession, tmpBsonQuery, nestedDocument.getSet(), null); - Bson bsonQuery = parseQuery(nestedDocument.getQuery().append(QueryParams.UID.key(), clinicalAnalysisUid)); - logger.debug("Update nested element from Clinical Analysis. Query: {}, Update: {}", - bsonQuery.toBsonDocument(), nestedDocument.getSet()); - - update = clinicalCollection.update(clientSession, bsonQuery, nestedDocument.getSet(), null); - - if (update.getNumMatches() == 0) { - throw CatalogDBException.uidNotFound("Clinical Analysis", clinicalAnalysisUid); + if (update.getNumMatches() == 0) { + throw CatalogDBException.uidNotFound("Clinical Analysis", clinicalAnalysisUid); + } } } - } - } else if (result.getNumUpdated() == 0) { - throw new CatalogDBException("Nothing to update"); - } + } else if (result.getNumUpdated() == 0) { + throw new CatalogDBException("Nothing to update"); + } - return endWrite(tmpStartTime, 1, 1, events); + return endWrite(tmpStartTime, 1, 1, events); + }, null, null); } @Override @@ -356,22 +368,24 @@ OpenCGAResult transactionalUpdate(ClientSession clientSession, Document updateOperation = updateDocument.toFinalUpdateDocument(); if (!updateOperation.isEmpty()) { - logger.debug("Update clinical analysis. Query: {}, Update: {}", query.toBsonDocument(), updateDocument); - DataResult update = clinicalCollection.update(clientSession, query, updateOperation, null); - - if (updateDocument.getSet().getBoolean(LOCKED.key(), false)) { - // Propagate locked value to Interpretations - logger.debug("Propagating case lock to all the Interpretations"); - MongoDBIterator iterator = clinicalCollection.iterator(clientSession, query, null, clinicalConverter, - ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS); - while (iterator.hasNext()) { - ClinicalAnalysis clinical = iterator.next(); - dbAdaptorFactory.getInterpretationDBAdaptor().propagateLockedFromClinicalAnalysis(clientSession, clinical, true); + return versionedMongoDBAdaptor.update(clientSession, query, entryList -> { + logger.debug("Update clinical analysis. Query: {}, Update: {}", query.toBsonDocument(), updateDocument); + DataResult update = clinicalCollection.update(clientSession, query, updateOperation, null); + + if (updateDocument.getSet().getBoolean(LOCKED.key(), false)) { + // Propagate locked value to Interpretations + logger.debug("Propagating case lock to all the Interpretations"); + MongoDBIterator iterator = clinicalCollection.iterator(clientSession, query, null, clinicalConverter, + ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS); + while (iterator.hasNext()) { + ClinicalAnalysis clinical = iterator.next(); + dbAdaptorFactory.getInterpretationDBAdaptor().propagateLockedFromClinicalAnalysis(clientSession, clinical, true); + } } - } - logger.debug("{} clinical analyses successfully updated", update.getNumUpdated()); - return endWrite(tmpStartTime, update.getNumMatches(), update.getNumUpdated(), Collections.emptyList()); + logger.debug("{} clinical analyses successfully updated", update.getNumUpdated()); + return endWrite(tmpStartTime, update.getNumMatches(), update.getNumUpdated(), Collections.emptyList()); + }, null, null); } else { throw new CatalogDBException("Nothing to update"); } @@ -409,7 +423,7 @@ UpdateDocument parseAndValidateUpdateParams(ObjectMap parameters, List delete(ClinicalAnalysis clinicalAnalysis, List privateDelete(clientSession, clinicalAnalysis, clinicalAuditList)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete Clinical Analysis {}: {}", clinicalAnalysis.getId(), e.getMessage(), e); throw new CatalogDBException("Could not delete Clinical Analysis " + clinicalAnalysis.getId() + ": " + e.getMessage(), e.getCause()); @@ -699,7 +713,7 @@ public OpenCGAResult delete(Query query, List c try { result.append(runTransaction(clientSession -> privateDelete(clientSession, clinicalAnalysis, clinicalAuditList))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete Clinical Analysis {}: {}", clinicalAnalysis.getId(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, clinicalAnalysis.getId(), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -767,52 +781,18 @@ OpenCGAResult privateDelete(ClientSession clientSession, ClinicalAnalysis cli } } + // Add Audit to ClinicalAnalysis + transactionalUpdate(clientSession, clinicalAnalysis, new ObjectMap(), Collections.emptyList(), clinicalAuditList, + QueryOptions.empty()); + + // And delete ClinicalAnalysis Query query = new Query() - .append(STUDY_UID.key(), clinicalAnalysis.getStudyUid()) + .append(QueryParams.STUDY_UID.key(), clinicalAnalysis.getStudyUid()) .append(UID.key(), clinicalAnalysis.getUid()); - OpenCGAResult result = nativeGet(clientSession, query, QueryOptions.empty()); - if (result.getNumResults() == 0) { - throw new CatalogDBException("Internal error: Clinical Analysis '" + clinicalAnalysis.getId() + "' not found."); - } - - String clinicalId = result.first().getString(QueryParams.ID.key()); - long clinicalUid = result.first().getLong(PRIVATE_UID); - long studyUid = result.first().getLong(PRIVATE_STUDY_UID); - - logger.debug("Deleting Clinical Analysis {} ({})", clinicalId, clinicalUid); - - // Delete any documents that might have been already deleted with that id - Bson bsonQuery = new Document() - .append(QueryParams.ID.key(), clinicalId) - .append(PRIVATE_STUDY_UID, studyUid); - deletedClinicalCollection.remove(clientSession, bsonQuery, new QueryOptions(MongoDBCollection.MULTI, true)); - - // Set status to DELETED - nestedPut(QueryParams.INTERNAL_STATUS.key(), getMongoDBDocument(new InternalStatus(InternalStatus.DELETED), "status"), - result.first()); - - // Add audit - List auditList = result.first().getList(AUDIT.key(), Document.class); - for (ClinicalAudit clinicalAudit : clinicalAuditList) { - auditList.add(getMongoDBDocument(clinicalAudit, "ClinicalAudit")); - } - result.first().put(AUDIT.key(), auditList); - - // Insert the document in the DELETE collection - deletedClinicalCollection.insert(clientSession, replaceDotsInKeys(result.first()), null); - logger.debug("Inserted Clinical Analysis uid '{}' in DELETE collection", clinicalUid); - - // Remove the document from the main Clinical collection - bsonQuery = parseQuery(new Query(QueryParams.UID.key(), clinicalUid)); - DataResult remove = clinicalCollection.remove(clientSession, bsonQuery, null); - if (remove.getNumMatches() == 0) { - throw new CatalogDBException("Clinical Analysis " + clinicalId + " not found"); - } - if (remove.getNumDeleted() == 0) { - throw new CatalogDBException("Clinical Analysis " + clinicalId + " could not be deleted"); - } + Bson bsonQuery = parseQuery(query); + versionedMongoDBAdaptor.delete(clientSession, bsonQuery); - logger.debug("Clinical Analysis {}({}) deleted", clinicalId, clinicalUid); + logger.debug("Clinical Analysis {}({}) deleted", clinicalAnalysis.getId(), clinicalAnalysis.getUid()); return endWrite(tmpStartTime, 1, 0, 0, 1, Collections.emptyList()); } @@ -915,7 +895,7 @@ public DBIterator iterator(long studyUid, Query query, QueryOp query.put(PRIVATE_STUDY_UID, studyUid); MongoDBIterator mongoCursor = getMongoCursor(query, options, user); Document studyDocument = getStudyDocument(null, studyUid); - UnaryOperator iteratorFilter = (d) -> filterAnnotationSets(studyDocument, d, user, + UnaryOperator iteratorFilter = (d) -> filterAnnotationSets(dbAdaptorFactory.getOrganizationId(), studyDocument, d, user, StudyPermissions.Permissions.VIEW_CLINICAL_ANNOTATIONS.name(), ClinicalAnalysisPermissions.VIEW_ANNOTATIONS.name()); return new ClinicalAnalysisCatalogMongoDBIterator(mongoCursor, null, clinicalConverter, iteratorFilter, dbAdaptorFactory, studyUid, @@ -930,7 +910,7 @@ public DBIterator nativeIterator(long studyUid, Query query, QueryOptions option query.put(PRIVATE_STUDY_UID, studyUid); MongoDBIterator mongoCursor = getMongoCursor(query, queryOptions, user); Document studyDocument = getStudyDocument(null, studyUid); - UnaryOperator iteratorFilter = (d) -> filterAnnotationSets(studyDocument, d, user, + UnaryOperator iteratorFilter = (d) -> filterAnnotationSets(dbAdaptorFactory.getOrganizationId(), studyDocument, d, user, StudyPermissions.Permissions.VIEW_CLINICAL_ANNOTATIONS.name(), ClinicalAnalysisPermissions.VIEW_ANNOTATIONS.name()); return new ClinicalAnalysisCatalogMongoDBIterator(mongoCursor, null, null, iteratorFilter, dbAdaptorFactory, studyUid, user, @@ -956,7 +936,7 @@ private MongoDBIterator getMongoCursor(ClientSession clientSession, Qu } else { qOptions = new QueryOptions(); } - qOptions = filterQueryOptions(qOptions, Arrays.asList(ID, PRIVATE_UID, PRIVATE_STUDY_UID)); + qOptions = filterQueryOptionsToIncludeKeys(qOptions, Arrays.asList(ID, PRIVATE_UID, PRIVATE_STUDY_UID)); qOptions = removeInnerProjections(qOptions, PROBAND.key()); qOptions = removeInnerProjections(qOptions, FAMILY.key()); qOptions = removeInnerProjections(qOptions, PANELS.key()); @@ -965,12 +945,8 @@ private MongoDBIterator getMongoCursor(ClientSession clientSession, Qu qOptions = removeInnerProjections(qOptions, QueryParams.SECONDARY_INTERPRETATIONS.key()); logger.debug("Clinical analysis query : {}", bson.toBsonDocument()); - - if (!query.getBoolean(QueryParams.DELETED.key())) { - return clinicalCollection.iterator(clientSession, bson, null, null, qOptions); - } else { - return deletedClinicalCollection.iterator(clientSession, bson, null, null, qOptions); - } + MongoDBCollection collection = getQueryCollection(query, clinicalCollection, archiveClinicalCollection, deletedClinicalCollection); + return collection.iterator(clientSession, bson, null, null, qOptions); } @Override @@ -1045,8 +1021,7 @@ public OpenCGAResult nativeInsert(Map clinicalAnalysis, String u @Override public OpenCGAResult insert(long studyId, ClinicalAnalysis clinicalAnalysis, List variableSetList, - List clinicalAuditList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + List clinicalAuditList, QueryOptions options) throws CatalogException { try { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); @@ -1078,7 +1053,7 @@ ClinicalAnalysis insert(ClientSession clientSession, long studyId, ClinicalAnaly throw CatalogDBException.alreadyExists("ClinicalAnalysis", "id", clinicalAnalysis.getId()); } - long clinicalUid = getNewUid(); + long clinicalUid = getNewUid(clientSession); clinicalAnalysis.setAudit(clinicalAudit); @@ -1103,7 +1078,7 @@ ClinicalAnalysis insert(ClientSession clientSession, long studyId, ClinicalAnaly clinicalDocument.put(PRIVATE_DUE_DATE, TimeUtils.toDate(clinicalAnalysis.getDueDate())); logger.debug("Inserting ClinicalAnalysis '{}' ({})...", clinicalAnalysis.getId(), clinicalAnalysis.getUid()); - clinicalCollection.insert(clientSession, clinicalDocument, null); + versionedMongoDBAdaptor.insert(clientSession, clinicalDocument); logger.debug("ClinicalAnalysis '{}' successfully inserted", clinicalAnalysis.getId()); return clinicalAnalysis; @@ -1249,7 +1224,7 @@ void updateClinicalAnalysisPanelReferences(ClientSession clientSession, Panel pa Query query = new Query() .append(STUDY_UID.key(), panel.getStudyUid()) .append(PANELS_UID.key(), panel.getUid()) - .append(PANEL_LOCK.key(), false) + .append(PANEL_LOCKED.key(), false) .append(LOCKED.key(), false); QueryOptions include = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList( PANELS_UID.key(), @@ -1313,29 +1288,36 @@ private Bson parseQuery(Query query, Document extraQuery, String user) if (queryCopy.containsKey(QueryParams.STUDY_UID.key()) && (StringUtils.isNotEmpty(user) || queryCopy.containsKey(ParamConstants.ACL_PARAM))) { Document studyDocument = getStudyDocument(null, queryCopy.getLong(QueryParams.STUDY_UID.key())); + boolean simplifyPermissions = simplifyPermissions(); if (queryCopy.containsKey(ParamConstants.ACL_PARAM)) { andBsonList.addAll(AuthorizationMongoDBUtils.parseAclQuery(studyDocument, queryCopy, Enums.Resource.CLINICAL_ANALYSIS, user, - configuration)); + simplifyPermissions)); } else { if (containsAnnotationQuery(query)) { andBsonList.add(getQueryForAuthorisedEntries(studyDocument, user, - ClinicalAnalysisPermissions.VIEW_ANNOTATIONS.name(), Enums.Resource.CLINICAL_ANALYSIS, configuration)); + ClinicalAnalysisPermissions.VIEW_ANNOTATIONS.name(), Enums.Resource.CLINICAL_ANALYSIS, simplifyPermissions)); } else { andBsonList.add(getQueryForAuthorisedEntries(studyDocument, user, ClinicalAnalysisPermissions.VIEW.name(), - Enums.Resource.CLINICAL_ANALYSIS, configuration)); + Enums.Resource.CLINICAL_ANALYSIS, simplifyPermissions)); } } queryCopy.remove(ParamConstants.ACL_PARAM); } + if ("all".equalsIgnoreCase(queryCopy.getString(QueryParams.VERSION.key()))) { + queryCopy.put(Constants.ALL_VERSIONS, true); + queryCopy.remove(QueryParams.VERSION.key()); + } + boolean uidVersionQueryFlag = versionedMongoDBAdaptor.generateUidVersionQuery(queryCopy, andBsonList); + for (Map.Entry entry : queryCopy.entrySet()) { String key = entry.getKey().split("\\.")[0]; QueryParams queryParam = QueryParams.getParam(entry.getKey()) != null ? QueryParams.getParam(entry.getKey()) : QueryParams.getParam(key); if (queryParam == null) { - if (Constants.PRIVATE_ANNOTATION_PARAM_TYPES.equals(entry.getKey())) { + if (Constants.ALL_VERSIONS.equals(entry.getKey()) || Constants.PRIVATE_ANNOTATION_PARAM_TYPES.equals(entry.getKey())) { continue; } throw new CatalogDBException("Unexpected parameter " + entry.getKey() + ". The parameter does not exist or cannot be " @@ -1381,7 +1363,7 @@ private Bson parseQuery(Query query, Document extraQuery, String user) case INTERNAL_STATUS: case INTERNAL_STATUS_ID: // Convert the status to a positive status - queryCopy.put(queryParam.key(), InternalStatus.getPositiveStatus(ClinicalAnalysisStatus.STATUS_LIST, + queryCopy.put(queryParam.key(), InternalStatus.getPositiveStatus(InternalStatus.STATUS_LIST, queryCopy.getString(queryParam.key()))); addAutoOrQuery(INTERNAL_STATUS_ID.key(), queryParam.key(), queryCopy, INTERNAL_STATUS_ID.type(), andBsonList); break; @@ -1391,11 +1373,14 @@ private Bson parseQuery(Query query, Document extraQuery, String user) queryCopy.get(Constants.PRIVATE_ANNOTATION_PARAM_TYPES, ObjectMap.class)); } break; + case SNAPSHOT: + addAutoOrQuery(RELEASE_FROM_VERSION, queryParam.key(), queryCopy, queryParam.type(), andBsonList); + break; // Other parameter that can be queried. case ID: case UUID: case TYPE: - case PANEL_LOCK: + case PANEL_LOCKED: case LOCKED: case FILES_UID: case PROBAND_UID: @@ -1408,6 +1393,7 @@ private Bson parseQuery(Query query, Document extraQuery, String user) case PRIORITY_ID: case FLAGS_ID: case QUALITY_CONTROL_SUMMARY: + case VERSION: case RELEASE: case COMMENTS_DATE: addAutoOrQuery(queryParam.key(), queryParam.key(), queryCopy, queryParam.type(), andBsonList); @@ -1421,6 +1407,17 @@ private Bson parseQuery(Query query, Document extraQuery, String user) } } + // If the user doesn't look for a concrete version... + if (!uidVersionQueryFlag && !queryCopy.getBoolean(Constants.ALL_VERSIONS) && !queryCopy.containsKey(QueryParams.VERSION.key())) { + if (queryCopy.containsKey(QueryParams.SNAPSHOT.key())) { + // If the user looks for anything from some release, we will try to find the latest from the release (snapshot) + andBsonList.add(Filters.eq(LAST_OF_RELEASE, true)); + } else { + // Otherwise, we will always look for the latest version + andBsonList.add(Filters.eq(LAST_OF_VERSION, true)); + } + } + if (annotationDocument != null && !annotationDocument.isEmpty()) { andBsonList.add(annotationDocument); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/CohortMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/CohortMongoDBAdaptor.java index 18189eb56d6..6b948a722ba 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/CohortMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/CohortMongoDBAdaptor.java @@ -75,7 +75,7 @@ public class CohortMongoDBAdaptor extends AnnotationMongoDBAdaptor imple private CohortConverter cohortConverter; public CohortMongoDBAdaptor(MongoDBCollection cohortCollection, MongoDBCollection deletedCohortCollection, Configuration configuration, - MongoDBAdaptorFactory dbAdaptorFactory) { + OrganizationMongoDBAdaptorFactory dbAdaptorFactory) { super(configuration, LoggerFactory.getLogger(CohortMongoDBAdaptor.class)); this.dbAdaptorFactory = dbAdaptorFactory; this.cohortCollection = cohortCollection; @@ -96,7 +96,7 @@ public OpenCGAResult nativeInsert(Map cohort, String userId) thr @Override public OpenCGAResult insert(long studyId, Cohort cohort, List variableSetList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + throws CatalogException { try { return runTransaction(clientSession -> { long startTime = startQuery(); @@ -114,7 +114,7 @@ long insert(ClientSession clientSession, long studyId, Cohort cohort, List try { result.append(runTransaction(clientSession -> transactionalUpdate(clientSession, cohort, parameters, variableSetList, queryOptions))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not update cohort {}: {}", cohort.getId(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, cohort.getId(), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -545,7 +545,7 @@ public OpenCGAResult delete(Cohort cohort) throws CatalogDBException, CatalogPar throw new CatalogDBException("Could not find cohort " + cohort.getId() + " with uid " + cohort.getUid()); } return runTransaction(clientSession -> privateDelete(clientSession, result.first())); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete cohort {}: {}", cohort.getId(), e.getMessage(), e); throw new CatalogDBException("Could not delete cohort '" + cohort.getId() + "': " + e.getMessage(), e.getCause()); } @@ -561,7 +561,7 @@ public OpenCGAResult delete(Query query) throws CatalogDBException, CatalogParam String cohortId = cohort.getString(QueryParams.ID.key()); try { result.append(runTransaction(clientSession -> privateDelete(clientSession, cohort))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete cohort {}: {}", cohortId, e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, cohortId, e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -733,7 +733,7 @@ DBIterator iterator(ClientSession clientSession, long studyUid, Query qu query.put(PRIVATE_STUDY_UID, studyUid); MongoDBIterator mongoCursor = getMongoCursor(clientSession, query, options, user); Document studyDocument = getStudyDocument(clientSession, studyUid); - UnaryOperator iteratorFilter = (d) -> filterAnnotationSets(studyDocument, d, user, + UnaryOperator iteratorFilter = (d) -> filterAnnotationSets(dbAdaptorFactory.getOrganizationId(), studyDocument, d, user, StudyPermissions.Permissions.VIEW_COHORT_ANNOTATIONS.name(), CohortPermissions.VIEW_ANNOTATIONS.name()); return new CohortCatalogMongoDBIterator<>(mongoCursor, clientSession, cohortConverter, iteratorFilter, @@ -749,7 +749,7 @@ public DBIterator nativeIterator(long studyUid, Query query, QueryOptions option query.put(PRIVATE_STUDY_UID, studyUid); MongoDBIterator mongoCursor = getMongoCursor(null, query, queryOptions, user); Document studyDocument = getStudyDocument(null, studyUid); - UnaryOperator iteratorFilter = (d) -> filterAnnotationSets(studyDocument, d, user, + UnaryOperator iteratorFilter = (d) -> filterAnnotationSets(dbAdaptorFactory.getOrganizationId(), studyDocument, d, user, StudyPermissions.Permissions.VIEW_COHORT_ANNOTATIONS.name(), CohortPermissions.VIEW_ANNOTATIONS.name()); return new CohortCatalogMongoDBIterator(mongoCursor, null, null, iteratorFilter, dbAdaptorFactory.getCatalogSampleDBAdaptor(), @@ -878,18 +878,19 @@ private Bson parseQuery(Query query, Document extraQuery, String user) if (query.containsKey(QueryParams.STUDY_UID.key()) && (StringUtils.isNotEmpty(user) || query.containsKey(ParamConstants.ACL_PARAM))) { + boolean simplifyPermissions = simplifyPermissions(); Document studyDocument = getStudyDocument(null, query.getLong(QueryParams.STUDY_UID.key())); if (query.containsKey(ParamConstants.ACL_PARAM)) { andBsonList.addAll(AuthorizationMongoDBUtils.parseAclQuery(studyDocument, query, Enums.Resource.COHORT, user, - configuration)); + simplifyPermissions)); } else { if (containsAnnotationQuery(query)) { andBsonList.add(getQueryForAuthorisedEntries(studyDocument, user, - CohortPermissions.VIEW_ANNOTATIONS.name(), Enums.Resource.COHORT, configuration)); + CohortPermissions.VIEW_ANNOTATIONS.name(), Enums.Resource.COHORT, simplifyPermissions)); } else { andBsonList.add(getQueryForAuthorisedEntries(studyDocument, user, CohortPermissions.VIEW.name(), - Enums.Resource.COHORT, configuration)); + Enums.Resource.COHORT, simplifyPermissions)); } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FamilyMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FamilyMongoDBAdaptor.java index e2717dbddb9..97bfd92a671 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FamilyMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FamilyMongoDBAdaptor.java @@ -85,18 +85,19 @@ public class FamilyMongoDBAdaptor extends AnnotationMongoDBAdaptor imple private final MongoDBCollection archiveFamilyCollection; private final MongoDBCollection deletedFamilyCollection; private final FamilyConverter familyConverter; - private final VersionedMongoDBAdaptor versionedMongoDBAdaptor; + private final SnapshotVersionedMongoDBAdaptor versionedMongoDBAdaptor; public FamilyMongoDBAdaptor(MongoDBCollection familyCollection, MongoDBCollection archiveFamilyCollection, MongoDBCollection deletedFamilyCollection, Configuration configuration, - MongoDBAdaptorFactory dbAdaptorFactory) { + OrganizationMongoDBAdaptorFactory dbAdaptorFactory) { super(configuration, LoggerFactory.getLogger(FamilyMongoDBAdaptor.class)); this.dbAdaptorFactory = dbAdaptorFactory; this.familyCollection = familyCollection; this.archiveFamilyCollection = archiveFamilyCollection; this.deletedFamilyCollection = deletedFamilyCollection; this.familyConverter = new FamilyConverter(); - this.versionedMongoDBAdaptor = new VersionedMongoDBAdaptor(familyCollection, archiveFamilyCollection, deletedFamilyCollection); + this.versionedMongoDBAdaptor = new SnapshotVersionedMongoDBAdaptor(familyCollection, archiveFamilyCollection, + deletedFamilyCollection); } /** @@ -114,8 +115,7 @@ public OpenCGAResult nativeInsert(Map family, String userId) thr @Override public OpenCGAResult insert(long studyId, Family family, List members, List variableSetList, - QueryOptions options) throws CatalogDBException, CatalogParameterException, - CatalogAuthorizationException { + QueryOptions options) throws CatalogException { try { AtomicReference familyCopy = new AtomicReference<>(); OpenCGAResult result = runTransaction(clientSession -> { @@ -194,7 +194,7 @@ private Family insert(ClientSession clientSession, long studyUid, Family family, } } - long familyUid = getNewUid(); + long familyUid = getNewUid(clientSession); family.setUid(familyUid); family.setStudyUid(studyUid); @@ -332,7 +332,7 @@ public OpenCGAResult update(long familyUid, ObjectMap parameters, List transactionalUpdate(clientSession, familyDataResult.first(), parameters, variableSetList, queryOptions)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update family {}: {}", familyDataResult.first().getId(), e.getMessage(), e); throw new CatalogDBException("Could not update family " + familyDataResult.first().getId() + ": " + e.getMessage(), e.getCause()); @@ -367,7 +367,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, List try { result.append(runTransaction(clientSession -> transactionalUpdate(clientSession, family, parameters, variableSetList, queryOptions))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not update family {}: {}", family.getId(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, family.getId(), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -783,7 +783,7 @@ public OpenCGAResult delete(Family family) throws CatalogDBException, CatalogPar throw new CatalogDBException("Could not find family " + family.getId() + " with uid " + family.getUid()); } return runTransaction(clientSession -> privateDelete(clientSession, result.first())); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete family {}: {}", family.getId(), e.getMessage(), e); throw new CatalogDBException("Could not delete family " + family.getId() + ": " + e.getMessage(), e.getCause()); } @@ -800,7 +800,7 @@ public OpenCGAResult delete(Query query) throws CatalogDBException, CatalogParam String familyId = family.getString(QueryParams.ID.key()); try { result.append(runTransaction(clientSession -> privateDelete(clientSession, family))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete family {}: {}", familyId, e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, familyId, e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -966,7 +966,7 @@ public DBIterator iterator(ClientSession clientSession, long studyUid, Q query.put(PRIVATE_STUDY_UID, studyUid); MongoDBIterator mongoCursor = getMongoCursor(clientSession, query, options, user); Document studyDocument = getStudyDocument(clientSession, studyUid); - UnaryOperator iteratorFilter = (d) -> filterAnnotationSets(studyDocument, d, user, + UnaryOperator iteratorFilter = (d) -> filterAnnotationSets(dbAdaptorFactory.getOrganizationId(), studyDocument, d, user, StudyPermissions.Permissions.VIEW_FAMILY_ANNOTATIONS.name(), FamilyPermissions.VIEW_ANNOTATIONS.name()); return new FamilyCatalogMongoDBIterator<>(mongoCursor, null, familyConverter, iteratorFilter, @@ -987,7 +987,7 @@ DBIterator nativeIterator(ClientSession clientSession, long studyUid, Query quer query.put(PRIVATE_STUDY_UID, studyUid); MongoDBIterator mongoCursor = getMongoCursor(clientSession, query, queryOptions, user); Document studyDocument = getStudyDocument(clientSession, studyUid); - UnaryOperator iteratorFilter = (d) -> filterAnnotationSets(studyDocument, d, user, + UnaryOperator iteratorFilter = (d) -> filterAnnotationSets(dbAdaptorFactory.getOrganizationId(), studyDocument, d, user, StudyPermissions.Permissions.VIEW_FAMILY_ANNOTATIONS.name(), FamilyPermissions.VIEW_ANNOTATIONS.name()); return new FamilyCatalogMongoDBIterator(mongoCursor, clientSession, null, iteratorFilter, @@ -1190,17 +1190,18 @@ protected Bson parseQuery(Query query, Document extraQuery, String user) if (query.containsKey(QueryParams.STUDY_UID.key()) && (StringUtils.isNotEmpty(user) || query.containsKey(ParamConstants.ACL_PARAM))) { Document studyDocument = getStudyDocument(null, query.getLong(QueryParams.STUDY_UID.key())); + boolean simplifyPermissions = simplifyPermissions(); if (query.containsKey(ParamConstants.ACL_PARAM)) { andBsonList.addAll(AuthorizationMongoDBUtils.parseAclQuery(studyDocument, query, Enums.Resource.FAMILY, user, - configuration)); + simplifyPermissions)); } else { if (containsAnnotationQuery(query)) { andBsonList.add(getQueryForAuthorisedEntries(studyDocument, user, - FamilyPermissions.VIEW_ANNOTATIONS.name(), Enums.Resource.FAMILY, configuration)); + FamilyPermissions.VIEW_ANNOTATIONS.name(), Enums.Resource.FAMILY, simplifyPermissions)); } else { andBsonList.add(getQueryForAuthorisedEntries(studyDocument, user, FamilyPermissions.VIEW.name(), - Enums.Resource.FAMILY, configuration)); + Enums.Resource.FAMILY, simplifyPermissions)); } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java index f256a7e9cec..0f8383a9905 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java @@ -105,7 +105,7 @@ private enum UpdateAttributeParams { * @param ioManagerFactory IOManagerFactory. */ public FileMongoDBAdaptor(MongoDBCollection fileCollection, MongoDBCollection deletedFileCollection, Configuration configuration, - MongoDBAdaptorFactory dbAdaptorFactory, IOManagerFactory ioManagerFactory) { + OrganizationMongoDBAdaptorFactory dbAdaptorFactory, IOManagerFactory ioManagerFactory) { super(configuration, LoggerFactory.getLogger(FileMongoDBAdaptor.class)); this.dbAdaptorFactory = dbAdaptorFactory; this.fileCollection = fileCollection; @@ -127,8 +127,7 @@ public OpenCGAResult nativeInsert(Map file, String userId) throw @Override public OpenCGAResult insert(long studyId, File file, List existingSamples, List nonExistingSamples, - List variableSetList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + List variableSetList, QueryOptions options) throws CatalogException { return runTransaction( (clientSession) -> { long tmpStartTime = startQuery(); @@ -144,7 +143,7 @@ public OpenCGAResult insert(long studyId, File file, List existingSample @Override public OpenCGAResult insertWithVirtualFile(long studyId, File file, File virtualFile, List existingSamples, List nonExistingSamples, List variableSetList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + throws CatalogException { return runTransaction( (clientSession) -> { long tmpStartTime = startQuery(); @@ -247,7 +246,7 @@ long insert(ClientSession clientSession, long studyId, File file, List e } //new file uid - long fileUid = getNewUid(); + long fileUid = getNewUid(clientSession); file.setUid(fileUid); file.setStudyUid(studyId); if (StringUtils.isEmpty(file.getUuid())) { @@ -344,7 +343,7 @@ public OpenCGAResult update(long fileUid, ObjectMap parameters, List transactionalUpdate(clientSession, fileDataResult.first(), parameters, variableSetList, queryOptions)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update file {}: {}", fileDataResult.first().getPath(), e.getMessage(), e); throw new CatalogDBException("Could not update file " + fileDataResult.first().getPath() + ": " + e.getMessage(), e.getCause()); } @@ -370,7 +369,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, List try { result.append(runTransaction(clientSession -> transactionalUpdate(clientSession, file, parameters, variableSetList, queryOptions))); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update file {}: {}", file.getPath(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, file.getPath(), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -931,7 +930,7 @@ public OpenCGAResult delete(File file, String status) try { return runTransaction(clientSession -> privateDelete(clientSession, fileDocument, status)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete file {}: {}", file.getPath(), e.getMessage(), e); throw new CatalogDBException("Could not delete file " + file.getPath() + ": " + e.getMessage(), e.getCause()); } @@ -962,7 +961,7 @@ public OpenCGAResult delete(Query query, String status) Document fileDocument = iterator.next(); try { result.append(runTransaction(clientSession -> privateDelete(clientSession, fileDocument, status))); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete file {}: {}", fileDocument.getString(QueryParams.PATH.key()), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, fileDocument.getString(QueryParams.ID.key()), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -1253,7 +1252,7 @@ public DBIterator iterator(long studyUid, Query query, QueryOptions option MongoDBIterator mongoCursor = getMongoCursor(null, query, options, user); Document studyDocument = getStudyDocument(null, studyUid); - UnaryOperator iteratorFilter = (d) -> filterAnnotationSets(studyDocument, d, user, + UnaryOperator iteratorFilter = (d) -> filterAnnotationSets(dbAdaptorFactory.getOrganizationId(), studyDocument, d, user, StudyPermissions.Permissions.VIEW_FILE_ANNOTATIONS.name(), FilePermissions.VIEW_ANNOTATIONS.name()); @@ -1276,7 +1275,7 @@ public DBIterator nativeIterator(ClientSession clientSession, long stu MongoDBIterator mongoCursor = getMongoCursor(clientSession, query, queryOptions, user); Document studyDocument = getStudyDocument(clientSession, studyUid); - UnaryOperator iteratorFilter = (d) -> filterAnnotationSets(studyDocument, d, user, + UnaryOperator iteratorFilter = (d) -> filterAnnotationSets(dbAdaptorFactory.getOrganizationId(), studyDocument, d, user, StudyPermissions.Permissions.VIEW_FILE_ANNOTATIONS.name(), FilePermissions.VIEW_ANNOTATIONS.name()); @@ -1315,7 +1314,7 @@ private QueryOptions fixQueryOptions(QueryOptions qOptions) { fixAclProjection(options); // type must always be there when relatedFiles is included - options = filterQueryOptions(options, Collections.singletonList(QueryParams.TYPE.key())); + options = filterQueryOptionsToIncludeKeys(options, Collections.singletonList(QueryParams.TYPE.key())); return options; } @@ -1409,16 +1408,18 @@ private Bson parseQuery(Query query, Document extraQuery, String user) if (query.containsKey(QueryParams.STUDY_UID.key()) && (StringUtils.isNotEmpty(user) || query.containsKey(ParamConstants.ACL_PARAM))) { Document studyDocument = getStudyDocument(null, query.getLong(QueryParams.STUDY_UID.key())); + boolean simplifyPermissions = simplifyPermissions(); if (query.containsKey(ParamConstants.ACL_PARAM)) { - andBsonList.addAll(AuthorizationMongoDBUtils.parseAclQuery(studyDocument, query, Enums.Resource.FILE, user, configuration)); + andBsonList.addAll(AuthorizationMongoDBUtils.parseAclQuery(studyDocument, query, Enums.Resource.FILE, user, + simplifyPermissions)); } else { if (containsAnnotationQuery(query)) { andBsonList.add(getQueryForAuthorisedEntries(studyDocument, user, FilePermissions.VIEW_ANNOTATIONS.name(), - Enums.Resource.FILE, configuration)); + Enums.Resource.FILE, simplifyPermissions)); } else { andBsonList.add(getQueryForAuthorisedEntries(studyDocument, user, FilePermissions.VIEW.name(), - Enums.Resource.FILE, configuration)); + Enums.Resource.FILE, simplifyPermissions)); } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptor.java index f325a1cf6e8..03a6704af17 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptor.java @@ -83,20 +83,20 @@ public class IndividualMongoDBAdaptor extends AnnotationMongoDBAdaptor individual, String userId) @Override public OpenCGAResult insert(long studyId, Individual individual, List variableSetList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + throws CatalogException { try { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); @@ -174,7 +174,7 @@ Individual insert(ClientSession clientSession, long studyId, Individual individu throw CatalogDBException.idNotFound("Individual", individual.getMother().getId()); } - long individualUid = getNewUid(); + long individualUid = getNewUid(clientSession); individual.setUid(individualUid); individual.setStudyUid(studyId); @@ -329,7 +329,7 @@ public OpenCGAResult update(long individualUid, ObjectMap parameters, List transactionalUpdate(clientSession, individualUid, parameters, variableSetList, queryOptions)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { throw new CatalogDBException("Could not update individual: " + e.getMessage(), e.getCause()); } } @@ -363,7 +363,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, List try { result.append(runTransaction(clientSession -> transactionalUpdate(clientSession, individual, parameters, variableSetList, queryOptions))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not update individual {}: {}", individual.getId(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, individual.getId(), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -1051,7 +1051,7 @@ public OpenCGAResult delete(Individual individual) throws CatalogDBException, Ca throw new CatalogDBException("Could not find individual " + individual.getId() + " with uid " + individual.getUid()); } return runTransaction(clientSession -> privateDelete(clientSession, result.first())); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete individual {}: {}", individual.getId(), e.getMessage(), e); throw new CatalogDBException("Could not delete individual " + individual.getId() + ": " + e.getMessage(), e); } @@ -1068,7 +1068,7 @@ public OpenCGAResult delete(Query query) throws CatalogDBException, CatalogParam String individualId = individual.getString(QueryParams.ID.key()); try { result.append(runTransaction(clientSession -> privateDelete(clientSession, individual))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete individual {}: {}", individualId, e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, individualId, e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -1258,7 +1258,7 @@ DBIterator iterator(ClientSession clientSession, long studyUid, Quer query.put(PRIVATE_STUDY_UID, studyUid); MongoDBIterator mongoCursor = getMongoCursor(clientSession, query, options, user); Document studyDocument = getStudyDocument(clientSession, studyUid); - UnaryOperator iteratorFilter = (d) -> filterAnnotationSets(studyDocument, d, user, + UnaryOperator iteratorFilter = (d) -> filterAnnotationSets(dbAdaptorFactory.getOrganizationId(), studyDocument, d, user, StudyPermissions.Permissions.VIEW_INDIVIDUAL_ANNOTATIONS.name(), IndividualPermissions.VIEW_ANNOTATIONS.name()); @@ -1280,7 +1280,7 @@ DBIterator nativeIterator(ClientSession clientSession, long studyUid, Query quer query.put(PRIVATE_STUDY_UID, studyUid); MongoDBIterator mongoCursor = getMongoCursor(clientSession, query, queryOptions, user); Document studyDocument = getStudyDocument(clientSession, studyUid); - UnaryOperator iteratorFilter = (d) -> filterAnnotationSets(studyDocument, d, user, + UnaryOperator iteratorFilter = (d) -> filterAnnotationSets(dbAdaptorFactory.getOrganizationId(), studyDocument, d, user, StudyPermissions.Permissions.VIEW_INDIVIDUAL_ANNOTATIONS.name(), IndividualPermissions.VIEW_ANNOTATIONS.name()); @@ -1405,17 +1405,18 @@ private Bson parseQuery(Query query, Document extraQuery, String user) if (query.containsKey(QueryParams.STUDY_UID.key()) && (StringUtils.isNotEmpty(user) || query.containsKey(ParamConstants.ACL_PARAM))) { Document studyDocument = getStudyDocument(null, query.getLong(QueryParams.STUDY_UID.key())); + boolean simplifyPermissions = simplifyPermissions(); if (query.containsKey(ParamConstants.ACL_PARAM)) { andBsonList.addAll(AuthorizationMongoDBUtils.parseAclQuery(studyDocument, query, Enums.Resource.INDIVIDUAL, user, - configuration)); + simplifyPermissions)); } else { if (containsAnnotationQuery(query)) { andBsonList.add(getQueryForAuthorisedEntries(studyDocument, user, - IndividualPermissions.VIEW_ANNOTATIONS.name(), Enums.Resource.INDIVIDUAL, configuration)); + IndividualPermissions.VIEW_ANNOTATIONS.name(), Enums.Resource.INDIVIDUAL, simplifyPermissions)); } else { andBsonList.add(getQueryForAuthorisedEntries(studyDocument, user, IndividualPermissions.VIEW.name(), - Enums.Resource.INDIVIDUAL, configuration)); + Enums.Resource.INDIVIDUAL, simplifyPermissions)); } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/InterpretationMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/InterpretationMongoDBAdaptor.java index af0cc61e95a..0c833de52ed 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/InterpretationMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/InterpretationMongoDBAdaptor.java @@ -40,6 +40,7 @@ import org.opencb.opencga.catalog.db.mongodb.iterators.InterpretationCatalogMongoDBIterator; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.managers.ClinicalAnalysisManager; import org.opencb.opencga.catalog.utils.Constants; @@ -74,11 +75,11 @@ public class InterpretationMongoDBAdaptor extends CatalogMongoDBAdaptor implemen private final MongoDBCollection deleteInterpretationCollection; private final ClinicalAnalysisMongoDBAdaptor clinicalDBAdaptor; private final InterpretationConverter interpretationConverter; - private final VersionedMongoDBAdaptor versionedMongoDBAdaptor; + private final SnapshotVersionedMongoDBAdaptor versionedMongoDBAdaptor; public InterpretationMongoDBAdaptor(MongoDBCollection interpretationCollection, MongoDBCollection archiveInterpretationCollection, MongoDBCollection deleteInterpretationCollection, Configuration configuration, - MongoDBAdaptorFactory dbAdaptorFactory) { + OrganizationMongoDBAdaptorFactory dbAdaptorFactory) { super(configuration, LoggerFactory.getLogger(InterpretationMongoDBAdaptor.class)); this.dbAdaptorFactory = dbAdaptorFactory; this.clinicalDBAdaptor = dbAdaptorFactory.getClinicalAnalysisDBAdaptor(); @@ -86,7 +87,7 @@ public InterpretationMongoDBAdaptor(MongoDBCollection interpretationCollection, this.archiveInterpretationCollection = archiveInterpretationCollection; this.deleteInterpretationCollection = deleteInterpretationCollection; this.interpretationConverter = new InterpretationConverter(); - this.versionedMongoDBAdaptor = new VersionedMongoDBAdaptor(interpretationCollection, archiveInterpretationCollection, + this.versionedMongoDBAdaptor = new SnapshotVersionedMongoDBAdaptor(interpretationCollection, archiveInterpretationCollection, deleteInterpretationCollection); } @@ -108,6 +109,10 @@ public MongoDBCollection getInterpretationCollection() { return interpretationCollection; } + public MongoDBCollection getArchiveInterpretationCollection() { + return archiveInterpretationCollection; + } + @Override public OpenCGAResult nativeInsert(Map interpretation, String userId) throws CatalogDBException { Document document = getMongoDBDocument(interpretation, "clinicalAnalysis"); @@ -116,8 +121,7 @@ public OpenCGAResult nativeInsert(Map interpretation, String use @Override public OpenCGAResult insert(long studyId, Interpretation interpretation, ParamUtils.SaveInterpretationAs action, - List clinicalAuditList) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + List clinicalAuditList) throws CatalogException { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); logger.debug("Starting interpretation insert transaction for interpretation id '{}'", interpretation.getId()); @@ -236,7 +240,7 @@ Interpretation insert(ClientSession clientSession, long studyId, Interpretation + interpretation.getId() + "'} already exists."); } - long interpretationUid = getNewUid(); + long interpretationUid = getNewUid(clientSession); interpretation.setUid(interpretationUid); interpretation.setStudyUid(studyId); if (StringUtils.isEmpty(interpretation.getUuid())) { @@ -429,7 +433,7 @@ private UpdateDocument parseAndValidateUpdateParams(ClientSession clientSession, String[] booleanParams = {LOCKED.key()}; filterBooleanParams(parameters, document.getSet(), booleanParams); - String[] acceptedParams = {QueryParams.DESCRIPTION.key()}; + String[] acceptedParams = {QueryParams.NAME.key(), QueryParams.DESCRIPTION.key()}; filterStringParams(parameters, document.getSet(), acceptedParams); if (StringUtils.isNotEmpty(parameters.getString(QueryParams.CREATION_DATE.key()))) { @@ -631,7 +635,7 @@ public OpenCGAResult update(long uid, ObjectMap parameters, List try { return runTransaction(clientSession -> update(clientSession, interpretation.first(), parameters, clinicalAuditList, action, queryOptions)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update interpretation {}: {}", interpretationId, e.getMessage(), e); throw new CatalogDBException("Could not update interpretation " + interpretationId + ": " + e.getMessage(), e.getCause()); } @@ -650,7 +654,7 @@ public OpenCGAResult revert(long id, int previousVersion, List c return delete(clientSession, interpretation, clinicalAuditList, clinicalResult.first()); }); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete interpretation {}: {}", interpretationId, e.getMessage(), e); throw new CatalogDBException("Could not delete interpretation " + interpretation.getId() + ": " + e.getMessage(), e.getCause()); } @@ -848,7 +852,7 @@ public OpenCGAResult delete(Query query, List cli return delete(clientSession, interpretation, clinicalAuditList, clinicalResult.first()); })); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete interpretation {}: {}", interpretationId, e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, interpretationId, e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -942,8 +946,8 @@ private MongoDBIterator getMongoCursor(ClientSession clientSession, Qu qOptions = new QueryOptions(); } - qOptions = filterQueryOptions(qOptions, Arrays.asList(QueryParams.ID.key(), QueryParams.UUID.key(), QueryParams.UID.key(), - QueryParams.VERSION.key(), QueryParams.CLINICAL_ANALYSIS_ID.key())); + qOptions = filterQueryOptionsToIncludeKeys(qOptions, Arrays.asList(QueryParams.ID.key(), QueryParams.UUID.key(), + QueryParams.UID.key(), QueryParams.VERSION.key(), QueryParams.CLINICAL_ANALYSIS_ID.key())); logger.debug("Interpretation query : {}", bson.toBsonDocument()); MongoDBCollection collection = getQueryCollection(query, interpretationCollection, archiveInterpretationCollection, @@ -1132,6 +1136,7 @@ protected Bson parseQuery(Query query) throws CatalogDBException { break; // Other parameter that can be queried. case ID: + case NAME: case UUID: case PANELS_UID: case RELEASE: diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java index 9f201fa9cad..d89e0020f93 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java @@ -73,7 +73,7 @@ public class JobMongoDBAdaptor extends CatalogMongoDBAdaptor implements JobDBAda private JobConverter jobConverter; public JobMongoDBAdaptor(MongoDBCollection jobCollection, MongoDBCollection deletedJobCollection, Configuration configuration, - MongoDBAdaptorFactory dbAdaptorFactory) { + OrganizationMongoDBAdaptorFactory dbAdaptorFactory) { super(configuration, LoggerFactory.getLogger(JobMongoDBAdaptor.class)); this.dbAdaptorFactory = dbAdaptorFactory; this.jobCollection = jobCollection; @@ -100,8 +100,7 @@ public OpenCGAResult nativeInsert(Map job, String userId) throws } @Override - public OpenCGAResult insert(long studyId, Job job, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult insert(long studyId, Job job, QueryOptions options) throws CatalogException { try { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); @@ -129,7 +128,7 @@ long insert(ClientSession clientSession, long studyId, Job job) throws CatalogDB throw new CatalogDBException("Job { id: '" + job.getId() + "'} already exists."); } - long jobUid = getNewUid(); + long jobUid = getNewUid(clientSession); job.setUid(jobUid); job.setStudyUid(studyId); if (StringUtils.isEmpty(job.getUuid())) { @@ -228,7 +227,7 @@ public OpenCGAResult update(long jobUid, ObjectMap parameters, QueryOptions quer try { return runTransaction(session -> privateUpdate(session, dataResult.first(), parameters, queryOptions)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update job {}: {}", dataResult.first().getId(), e.getMessage(), e); throw new CatalogDBException("Could not update job " + dataResult.first().getId() + ": " + e.getMessage(), e.getCause()); } @@ -254,7 +253,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, QueryOptions quer Job job = iterator.next(); try { result.append(runTransaction(session -> privateUpdate(session, job, parameters, queryOptions))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not update job {}: {}", job.getId(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, job.getId(), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -306,7 +305,7 @@ public OpenCGAResult delete(Job job) throws CatalogDBException, CatalogParameter throw new CatalogDBException("Could not find job " + job.getId() + " with uid " + job.getUid()); } return runTransaction(clientSession -> privateDelete(clientSession, result.first())); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete job {}: {}", job.getId(), e.getMessage(), e); throw new CatalogDBException("Could not delete job " + job.getId() + ": " + e.getMessage(), e.getCause()); } @@ -322,7 +321,7 @@ public OpenCGAResult delete(Query query) throws CatalogDBException, CatalogParam String jobId = job.getString(QueryParams.ID.key()); try { result.append(runTransaction(clientSession -> privateDelete(clientSession, job))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete job {}: {}", jobId, e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, jobId, e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -373,7 +372,7 @@ private UpdateDocument parseAndValidateUpdateParams(ObjectMap parameters, QueryO String[] acceptedParams = {QueryParams.USER_ID.key(), QueryParams.DESCRIPTION.key(), QueryParams.COMMAND_LINE.key()}; filterStringParams(parameters, document.getSet(), acceptedParams); - String[] acceptedBooleanParams = {QueryParams.VISITED.key()}; + String[] acceptedBooleanParams = {QueryParams.VISITED.key(), QueryParams.INTERNAL_KILL_JOB_REQUESTED.key()}; filterBooleanParams(parameters, document.getSet(), acceptedBooleanParams); String[] acceptedStringListParams = {QueryParams.TAGS.key()}; @@ -814,13 +813,15 @@ private Bson parseQuery(Query query, Document extraQuery, QueryOptions options, if (query.containsKey(QueryParams.STUDY_UID.key()) && (StringUtils.isNotEmpty(user) || query.containsKey(ParamConstants.ACL_PARAM))) { Document studyDocument = getStudyDocument(null, query.getLong(QueryParams.STUDY_UID.key())); + boolean simplifyPermissions = simplifyPermissions(); if (query.containsKey(ParamConstants.ACL_PARAM)) { - andBsonList.addAll(AuthorizationMongoDBUtils.parseAclQuery(studyDocument, query, Enums.Resource.JOB, user, configuration)); + andBsonList.addAll(AuthorizationMongoDBUtils.parseAclQuery(studyDocument, query, Enums.Resource.JOB, user, + simplifyPermissions)); } else { // Get the document query needed to check the permissions as well andBsonList.add(getQueryForAuthorisedEntries(studyDocument, user, JobPermissions.VIEW.name(), - Enums.Resource.JOB, configuration)); + Enums.Resource.JOB, simplifyPermissions)); } query.remove(ParamConstants.ACL_PARAM); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MetaMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MetaMongoDBAdaptor.java index 295a84f2093..b2aba25019e 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MetaMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MetaMongoDBAdaptor.java @@ -52,13 +52,13 @@ public class MetaMongoDBAdaptor extends MongoDBAdaptor implements MetaDBAdaptor private static final String OLD_ID = "_id"; public static final Bson METADATA_QUERY = Filters.or( - Filters.eq(ID, MongoDBAdaptorFactory.METADATA_OBJECT_ID), - Filters.eq(OLD_ID, MongoDBAdaptorFactory.METADATA_OBJECT_ID)); + Filters.eq(ID, OrganizationMongoDBAdaptorFactory.METADATA_OBJECT_ID), + Filters.eq(OLD_ID, OrganizationMongoDBAdaptorFactory.METADATA_OBJECT_ID)); private final MongoDBCollection metaCollection; private static final String VERSION = GitRepositoryState.getInstance().getBuildVersion(); public MetaMongoDBAdaptor(MongoDBCollection metaMongoDBCollection, Configuration configuration, - MongoDBAdaptorFactory dbAdaptorFactory) { + OrganizationMongoDBAdaptorFactory dbAdaptorFactory) { super(configuration, LoggerFactory.getLogger(MetaMongoDBAdaptor.class)); this.dbAdaptorFactory = dbAdaptorFactory; this.metaCollection = metaMongoDBCollection; @@ -112,25 +112,25 @@ public void createIndexes() { throw new UncheckedIOException(e); } - createIndexes(MongoDBAdaptorFactory.USER_COLLECTION, indexes); - createIndexes(MongoDBAdaptorFactory.STUDY_COLLECTION, indexes); - createIndexes(MongoDBAdaptorFactory.FILE_COLLECTION, indexes); - createIndexes(MongoDBAdaptorFactory.COHORT_COLLECTION, indexes); - createIndexes(MongoDBAdaptorFactory.JOB_COLLECTION, indexes); - createIndexes(MongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, indexes); - createIndexes(MongoDBAdaptorFactory.AUDIT_COLLECTION, indexes); + createIndexes(OrganizationMongoDBAdaptorFactory.USER_COLLECTION, indexes); + createIndexes(OrganizationMongoDBAdaptorFactory.STUDY_COLLECTION, indexes); + createIndexes(OrganizationMongoDBAdaptorFactory.FILE_COLLECTION, indexes); + createIndexes(OrganizationMongoDBAdaptorFactory.COHORT_COLLECTION, indexes); + createIndexes(OrganizationMongoDBAdaptorFactory.JOB_COLLECTION, indexes); + createIndexes(OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, indexes); + createIndexes(OrganizationMongoDBAdaptorFactory.AUDIT_COLLECTION, indexes); // Versioned collections - createIndexes(MongoDBAdaptorFactory.SAMPLE_COLLECTION, indexes); - createIndexes(MongoDBAdaptorFactory.SAMPLE_ARCHIVE_COLLECTION, indexes); - createIndexes(MongoDBAdaptorFactory.INDIVIDUAL_COLLECTION, indexes); - createIndexes(MongoDBAdaptorFactory.INDIVIDUAL_ARCHIVE_COLLECTION, indexes); - createIndexes(MongoDBAdaptorFactory.FAMILY_COLLECTION, indexes); - createIndexes(MongoDBAdaptorFactory.FAMILY_ARCHIVE_COLLECTION, indexes); - createIndexes(MongoDBAdaptorFactory.PANEL_COLLECTION, indexes); - createIndexes(MongoDBAdaptorFactory.PANEL_ARCHIVE_COLLECTION, indexes); - createIndexes(MongoDBAdaptorFactory.INTERPRETATION_COLLECTION, indexes); - createIndexes(MongoDBAdaptorFactory.INTERPRETATION_ARCHIVE_COLLECTION, indexes); + createIndexes(OrganizationMongoDBAdaptorFactory.SAMPLE_COLLECTION, indexes); + createIndexes(OrganizationMongoDBAdaptorFactory.SAMPLE_ARCHIVE_COLLECTION, indexes); + createIndexes(OrganizationMongoDBAdaptorFactory.INDIVIDUAL_COLLECTION, indexes); + createIndexes(OrganizationMongoDBAdaptorFactory.INDIVIDUAL_ARCHIVE_COLLECTION, indexes); + createIndexes(OrganizationMongoDBAdaptorFactory.FAMILY_COLLECTION, indexes); + createIndexes(OrganizationMongoDBAdaptorFactory.FAMILY_ARCHIVE_COLLECTION, indexes); + createIndexes(OrganizationMongoDBAdaptorFactory.PANEL_COLLECTION, indexes); + createIndexes(OrganizationMongoDBAdaptorFactory.PANEL_ARCHIVE_COLLECTION, indexes); + createIndexes(OrganizationMongoDBAdaptorFactory.INTERPRETATION_COLLECTION, indexes); + createIndexes(OrganizationMongoDBAdaptorFactory.INTERPRETATION_ARCHIVE_COLLECTION, indexes); } private void createIndexes(String collection, Map>> indexCollectionMap) { @@ -188,7 +188,7 @@ public void initializeMetaCollection(Admin admin) throws CatalogException { } Document metadataObject = getMongoDBDocument(metadata, "Metadata"); - metadataObject.put(ID, MongoDBAdaptorFactory.METADATA_OBJECT_ID); + metadataObject.put(ID, OrganizationMongoDBAdaptorFactory.METADATA_OBJECT_ID); Document adminDocument = getMongoDBDocument(admin, "Admin"); metadataObject.put("admin", adminDocument); metadataObject.put("_fullVersion", new Document() diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MigrationMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MigrationMongoDBAdaptor.java index 440107d7b57..a73955c91fc 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MigrationMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MigrationMongoDBAdaptor.java @@ -10,7 +10,7 @@ import org.opencb.opencga.catalog.db.api.MigrationDBAdaptor; import org.opencb.opencga.catalog.db.mongodb.converters.MigrationConverter; import org.opencb.opencga.catalog.exceptions.CatalogDBException; -import org.opencb.opencga.catalog.migration.MigrationRun; +import org.opencb.opencga.core.models.migration.MigrationRun; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.response.OpenCGAResult; import org.slf4j.LoggerFactory; @@ -25,7 +25,7 @@ public class MigrationMongoDBAdaptor extends MongoDBAdaptor implements Migration private final MigrationConverter migrationConverter; public MigrationMongoDBAdaptor(MongoDBCollection migrationCollection, Configuration configuration, - MongoDBAdaptorFactory dbAdaptorFactory) { + OrganizationMongoDBAdaptorFactory dbAdaptorFactory) { super(configuration, LoggerFactory.getLogger(MigrationMongoDBAdaptor.class)); this.dbAdaptorFactory = dbAdaptorFactory; this.migrationCollection = migrationCollection; @@ -52,6 +52,10 @@ public OpenCGAResult get(Query query) throws CatalogDBException { return new OpenCGAResult<>(migrationCollection.find(bsonQuery, migrationConverter, QueryOptions.empty())); } + public OpenCGAResult nativeGet() { + return new OpenCGAResult<>(migrationCollection.find(new Document(), QueryOptions.empty())); + } + @Override public OpenCGAResult get(List migrationRunIds) throws CatalogDBException { Query query = new Query(QueryParams.ID.key(), migrationRunIds); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptor.java index 0c577c0a9da..32c10c362f7 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptor.java @@ -28,10 +28,12 @@ import org.opencb.opencga.catalog.db.AbstractDBAdaptor; import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; import org.opencb.opencga.catalog.exceptions.*; +import org.opencb.opencga.catalog.managers.OrganizationManager; import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.config.Configuration; +import org.opencb.opencga.core.models.organizations.Organization; import org.opencb.opencga.core.response.OpenCGAResult; import org.slf4j.Logger; @@ -56,9 +58,9 @@ public abstract class MongoDBAdaptor extends AbstractDBAdaptor { static final String PRIVATE_PROJECT_ID = PRIVATE_PROJECT + '.' + ID; static final String PRIVATE_PROJECT_UID = PRIVATE_PROJECT + '.' + PRIVATE_UID; static final String PRIVATE_PROJECT_UUID = PRIVATE_PROJECT + '.' + PRIVATE_UUID; - static final String PRIVATE_OWNER_ID = "_ownerId"; public static final String PRIVATE_STUDY_UID = "studyUid"; public static final String VERSION = "version"; + public static final String RELEASE = "release"; static final String FILTER_ROUTE_STUDIES = "projects.studies."; static final String FILTER_ROUTE_COHORTS = "projects.studies.cohorts."; @@ -81,7 +83,7 @@ public abstract class MongoDBAdaptor extends AbstractDBAdaptor { // TEST PURPOSES ONLY public static final boolean MOCK_TRANSIENT_TRANSACTION_ERRORS = false; - protected MongoDBAdaptorFactory dbAdaptorFactory; + protected OrganizationMongoDBAdaptorFactory dbAdaptorFactory; protected Configuration configuration; protected static final QueryOptions EXCLUDE_MONGO_ID = new QueryOptions(QueryOptions.EXCLUDE, PRIVATE_MONGO_ID); @@ -92,16 +94,15 @@ public MongoDBAdaptor(Configuration configuration, Logger logger) { } public interface TransactionBodyWithException { - T execute(ClientSession session) throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException; + T execute(ClientSession session) throws CatalogException; } - protected T runTransaction(TransactionBodyWithException body) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + protected T runTransaction(TransactionBodyWithException body) throws CatalogException { return runTransaction(body, null); } protected T runTransaction(TransactionBodyWithException inputBody, Consumer onException) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + throws CatalogException { ClientSession session = dbAdaptorFactory.getMongoDataStore().startSession(); try { TransactionBodyWithException body; @@ -123,7 +124,7 @@ protected T runTransaction(TransactionBodyWithException inputBody, Consum return session.withTransaction(() -> { try { return body.execute(session); - } catch (CatalogDBException | CatalogAuthorizationException | CatalogParameterException e) { + } catch (CatalogException e) { throw new CatalogDBRuntimeException(e); } }); @@ -146,6 +147,12 @@ protected T runTransaction(TransactionBodyWithException inputBody, Consum onException.accept(cause); } throw cause; + } else if (e.getCause() instanceof CatalogAuthenticationException) { + CatalogAuthenticationException cause = (CatalogAuthenticationException) e.getCause(); + if (onException != null) { + onException.accept(cause); + } + throw cause; } else { throw e; } @@ -176,13 +183,11 @@ protected MongoDBCollection getQueryCollection(Query query, MongoDBCollection co } protected long getNewUid() { -// return CatalogMongoDBUtils.getNewAutoIncrementId(metaCollection); - return dbAdaptorFactory.getCatalogMetaDBAdaptor().getNewAutoIncrementId(); + return dbAdaptorFactory.getCatalogOrganizationDBAdaptor().getNewAutoIncrementId(); } protected long getNewUid(ClientSession clientSession) { -// return CatalogMongoDBUtils.getNewAutoIncrementId(metaCollection); - return dbAdaptorFactory.getCatalogMetaDBAdaptor().getNewAutoIncrementId(clientSession); + return dbAdaptorFactory.getCatalogOrganizationDBAdaptor().getNewAutoIncrementId(clientSession); } @Deprecated @@ -418,7 +423,7 @@ public static QueryOptions addPrefixInOptions(QueryOptions options, String prefi * @param keys Keys that always need to be included in the response. * @return A new QueryOptions object containing the mandatory fields. */ - public static QueryOptions filterQueryOptions(QueryOptions options, List keys) { + public static QueryOptions filterQueryOptionsToIncludeKeys(QueryOptions options, List keys) { if (options == null) { return null; } @@ -439,6 +444,34 @@ public static QueryOptions filterQueryOptions(QueryOptions options, List return queryOptions; } + /** + * Filter QueryOptions object to ensure the keys provided are always included. + * + * @param options QueryOptions object. + * @param keys Keys that always need to be included in the response. + * @return A new QueryOptions object containing the mandatory fields. + */ + public static QueryOptions filterQueryOptionsToExcludeKeys(QueryOptions options, List keys) { + if (options == null) { + return null; + } + + QueryOptions queryOptions = new QueryOptions(options); + + if (queryOptions.containsKey(QueryOptions.INCLUDE)) { + Set includeList = new HashSet<>(queryOptions.getAsStringList(QueryOptions.INCLUDE)); + includeList.removeAll(keys); + queryOptions.put(QueryOptions.INCLUDE, new ArrayList<>(includeList)); + } + if (queryOptions.containsKey(QueryOptions.EXCLUDE)) { + Set excludeList = new HashSet<>(queryOptions.getAsStringList(QueryOptions.EXCLUDE)); + excludeList.addAll(keys); + queryOptions.put(QueryOptions.EXCLUDE, new ArrayList<>(excludeList)); + } + + return queryOptions; + } + /** * Create a date projection if included in the includeGroupByFields, removes the date fields from includeGroupByFields and @@ -678,6 +711,24 @@ protected Document getStudyDocument(ClientSession clientSession, long studyUid) return dataResult.first(); } + /** + * Method to obtain whether permissions should be simplified or not. + * + * @return true if permissions should be simplified, false otherwise. + * @throws CatalogDBException if there is any error obtaining the organization configuration. + */ + protected boolean simplifyPermissions() throws CatalogDBException { + Organization organization = dbAdaptorFactory.getCatalogOrganizationDBAdaptor() + .get(OrganizationManager.INCLUDE_ORGANIZATION_CONFIGURATION).first(); + if (organization.getConfiguration().getOptimizations() != null) { + return organization.getConfiguration().getOptimizations().isSimplifyPermissions(); + } else { + logger.warn("Organization '{}' configuration does not contain the 'optimizations.simplifyPermissions' field. Defaulting" + + " to false", organization.getId()); + return false; + } + } + public class NestedArrayUpdateDocument { private Query query; private Document set; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java index 8dff0ee6129..e262c3d3595 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java @@ -17,155 +17,55 @@ package org.opencb.opencga.catalog.db.mongodb; import com.fasterxml.jackson.core.JsonProcessingException; -import com.mongodb.BasicDBObject; +import com.fasterxml.jackson.databind.JsonMappingException; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.time.StopWatch; -import org.bson.Document; import org.opencb.commons.datastore.core.DataStoreServerAddress; -import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.commons.datastore.mongodb.MongoDBCollection; +import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.mongodb.MongoDBConfiguration; import org.opencb.commons.datastore.mongodb.MongoDataStore; import org.opencb.commons.datastore.mongodb.MongoDataStoreManager; import org.opencb.opencga.catalog.db.DBAdaptorFactory; -import org.opencb.opencga.catalog.db.api.MigrationDBAdaptor; +import org.opencb.opencga.catalog.db.api.*; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.io.IOManagerFactory; +import org.opencb.opencga.catalog.managers.NoteManager; +import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.config.Admin; import org.opencb.opencga.core.config.Configuration; +import org.opencb.opencga.core.models.notes.Note; +import org.opencb.opencga.core.models.notes.NoteCreateParams; +import org.opencb.opencga.core.models.organizations.Organization; +import org.opencb.opencga.core.models.organizations.OrganizationSummary; +import org.opencb.opencga.core.response.OpenCGAResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.*; -import java.util.concurrent.TimeUnit; - -import static org.opencb.opencga.core.common.JacksonUtils.getDefaultObjectMapper; /** * Created by pfurio on 08/01/16. */ public class MongoDBAdaptorFactory implements DBAdaptorFactory { - public static final String USER_COLLECTION = "user"; - public static final String STUDY_COLLECTION = "study"; - public static final String FILE_COLLECTION = "file"; - public static final String JOB_COLLECTION = "job"; - public static final String SAMPLE_COLLECTION = "sample"; - public static final String INDIVIDUAL_COLLECTION = "individual"; - public static final String COHORT_COLLECTION = "cohort"; - public static final String FAMILY_COLLECTION = "family"; - public static final String PANEL_COLLECTION = "panel"; - public static final String CLINICAL_ANALYSIS_COLLECTION = "clinical"; - public static final String INTERPRETATION_COLLECTION = "interpretation"; - - public static final String SAMPLE_ARCHIVE_COLLECTION = "sample_archive"; - public static final String INDIVIDUAL_ARCHIVE_COLLECTION = "individual_archive"; - public static final String FAMILY_ARCHIVE_COLLECTION = "family_archive"; - public static final String PANEL_ARCHIVE_COLLECTION = "panel_archive"; - public static final String INTERPRETATION_ARCHIVE_COLLECTION = "interpretation_archive"; - - @Deprecated - public static final String OLD_DELETED_USER_COLLECTION = "deleted_user"; - @Deprecated - public static final String OLD_DELETED_STUDY_COLLECTION = "deleted_study"; - @Deprecated - public static final String OLD_DELETED_FILE_COLLECTION = "deleted_file"; - @Deprecated - public static final String OLD_DELETED_JOB_COLLECTION = "deleted_job"; - @Deprecated - public static final String OLD_DELETED_SAMPLE_COLLECTION = "deleted_sample"; - @Deprecated - public static final String OLD_DELETED_INDIVIDUAL_COLLECTION = "deleted_individual"; - @Deprecated - public static final String OLD_DELETED_COHORT_COLLECTION = "deleted_cohort"; - @Deprecated - public static final String OLD_DELETED_FAMILY_COLLECTION = "deleted_family"; - @Deprecated - public static final String OLD_DELETED_PANEL_COLLECTION = "deleted_panel"; - @Deprecated - public static final String OLD_DELETED_CLINICAL_ANALYSIS_COLLECTION = "deleted_clinical"; - @Deprecated - public static final String OLD_DELETED_INTERPRETATION_COLLECTION = "deleted_interpretation"; - - public static final String DELETED_USER_COLLECTION = "user_deleted"; - public static final String DELETED_STUDY_COLLECTION = "study_deleted"; - public static final String DELETED_FILE_COLLECTION = "file_deleted"; - public static final String DELETED_JOB_COLLECTION = "job_deleted"; - public static final String DELETED_SAMPLE_COLLECTION = "sample_deleted"; - public static final String DELETED_INDIVIDUAL_COLLECTION = "individual_deleted"; - public static final String DELETED_COHORT_COLLECTION = "cohort_deleted"; - public static final String DELETED_FAMILY_COLLECTION = "family_deleted"; - public static final String DELETED_PANEL_COLLECTION = "panel_deleted"; - public static final String DELETED_CLINICAL_ANALYSIS_COLLECTION = "clinical_deleted"; - public static final String DELETED_INTERPRETATION_COLLECTION = "interpretation_deleted"; - - public static final String METADATA_COLLECTION = "metadata"; - public static final String MIGRATION_COLLECTION = "migration"; - public static final String AUDIT_COLLECTION = "audit"; - - public static final List COLLECTIONS_LIST = Arrays.asList( - USER_COLLECTION, - STUDY_COLLECTION, - FILE_COLLECTION, - JOB_COLLECTION, - SAMPLE_COLLECTION, - INDIVIDUAL_COLLECTION, - COHORT_COLLECTION, - PANEL_COLLECTION, - FAMILY_COLLECTION, - CLINICAL_ANALYSIS_COLLECTION, - INTERPRETATION_COLLECTION, - - SAMPLE_ARCHIVE_COLLECTION, - INDIVIDUAL_ARCHIVE_COLLECTION, - FAMILY_ARCHIVE_COLLECTION, - PANEL_ARCHIVE_COLLECTION, - INTERPRETATION_ARCHIVE_COLLECTION, - - DELETED_USER_COLLECTION, - DELETED_STUDY_COLLECTION, - DELETED_FILE_COLLECTION, - DELETED_JOB_COLLECTION, - DELETED_SAMPLE_COLLECTION, - DELETED_INDIVIDUAL_COLLECTION, - DELETED_COHORT_COLLECTION, - DELETED_PANEL_COLLECTION, - DELETED_FAMILY_COLLECTION, - DELETED_CLINICAL_ANALYSIS_COLLECTION, - DELETED_INTERPRETATION_COLLECTION, - - MIGRATION_COLLECTION, - METADATA_COLLECTION, - AUDIT_COLLECTION - ); - - static final String METADATA_OBJECT_ID = "METADATA"; private final IOManagerFactory ioManagerFactory; private final MongoDataStoreManager mongoManager; - private final MongoDBConfiguration configuration; - private final String database; - private MongoDataStore mongoDataStore; - - private MongoDBCollection metaCollection; - private Map collections; - private UserMongoDBAdaptor userDBAdaptor; - private StudyMongoDBAdaptor studyDBAdaptor; - private IndividualMongoDBAdaptor individualDBAdaptor; - private SampleMongoDBAdaptor sampleDBAdaptor; - private FileMongoDBAdaptor fileDBAdaptor; - private JobMongoDBAdaptor jobDBAdaptor; - private ProjectMongoDBAdaptor projectDBAdaptor; - private CohortMongoDBAdaptor cohortDBAdaptor; - private FamilyMongoDBAdaptor familyDBAdaptor; - private PanelMongoDBAdaptor panelDBAdaptor; - private ClinicalAnalysisMongoDBAdaptor clinicalDBAdaptor; - private InterpretationMongoDBAdaptor interpretationDBAdaptor; - private AuditMongoDBAdaptor auditDBAdaptor; - private MetaMongoDBAdaptor metaDBAdaptor; - private MigrationMongoDBAdaptor migrationDBAdaptor; - - private Logger logger; + private final MongoDBConfiguration mongoDbConfiguration; + private final Configuration configuration; + + private static final String ORGANIZATION_PREFIX = "ORG_"; + private enum OrganizationTag { + ACTIVE, + SUSPENDED, // owner action + INACTIVE, // ADMINISTRATOR + DELETED + } + + private Map organizationDBAdaptorMap; + + private final Logger logger; public MongoDBAdaptorFactory(Configuration catalogConfiguration, IOManagerFactory ioManagerFactory) throws CatalogDBException { List dataStoreServerAddresses = new LinkedList<>(); @@ -188,8 +88,8 @@ public MongoDBAdaptorFactory(Configuration catalogConfiguration, IOManagerFactor .build(); this.mongoManager = new MongoDataStoreManager(dataStoreServerAddresses); - this.configuration = mongoDBConfiguration; - this.database = getCatalogDatabase(catalogConfiguration.getDatabasePrefix()); + this.mongoDbConfiguration = mongoDBConfiguration; + this.configuration = catalogConfiguration; this.ioManagerFactory = ioManagerFactory; logger = LoggerFactory.getLogger(this.getClass()); @@ -198,253 +98,311 @@ public MongoDBAdaptorFactory(Configuration catalogConfiguration, IOManagerFactor @Override public void createAllCollections(Configuration configuration) throws CatalogException { - // TODO: Check META object does not exist. Use {@link isCatalogDBReady} - // TODO: Check all collections do not exists, or are empty - // TODO: Catch DuplicatedKeyException while inserting META object + for (OrganizationMongoDBAdaptorFactory orgFactory : organizationDBAdaptorMap.values()) { + createAllCollections(orgFactory); + } + } - MongoDataStore mongoDataStore = mongoManager.get(database, this.configuration); + private void createAllCollections(OrganizationMongoDBAdaptorFactory organizationFactory) throws CatalogDBException { + MongoDataStore mongoDataStore = organizationFactory.getMongoDataStore(); if (!mongoDataStore.getCollectionNames().isEmpty()) { - throw new CatalogException("Database " + database + " already exists with the following collections: " - + StringUtils.join(mongoDataStore.getCollectionNames()) + ".\nPlease, remove the database or choose a different one."); + throw new CatalogDBException("Database " + mongoDataStore.getDatabaseName() + " already exists with the following " + + "collections: " + StringUtils.join(mongoDataStore.getCollectionNames()) + ".\nPlease, remove the database or" + + " choose a different one."); } - COLLECTIONS_LIST.forEach(mongoDataStore::createCollection); + OrganizationMongoDBAdaptorFactory.COLLECTIONS_LIST.forEach(mongoDataStore::createCollection); } @Override public void initialiseMetaCollection(Admin admin) throws CatalogException { - metaDBAdaptor.initializeMetaCollection(admin); + for (OrganizationMongoDBAdaptorFactory dbAdaptorFactory : organizationDBAdaptorMap.values()) { + dbAdaptorFactory.initialiseMetaCollection(admin); + } } @Override - public void createIndexes() { - StopWatch stopWatch = StopWatch.createStarted(); - metaDBAdaptor.createIndexes(); - logger.info("Creating all indexes took {} milliseconds", stopWatch.getTime(TimeUnit.MILLISECONDS)); + public boolean getDatabaseStatus() throws CatalogDBException { + return getOrganizationMongoDBAdaptorFactory(ParamConstants.ADMIN_ORGANIZATION).getDatabaseStatus(); } @Override - public boolean getDatabaseStatus() { - Document dbStatus = mongoManager.get(database, this.configuration).getServerStatus(); - try { - ObjectMap map = new ObjectMap(getDefaultObjectMapper().writeValueAsString(dbStatus)); - return map.getInt("ok", 0) > 0; - } catch (JsonProcessingException e) { - logger.error(e.getMessage(), e); + public void deleteCatalogDB() { + for (OrganizationMongoDBAdaptorFactory dbAdaptorFactory : organizationDBAdaptorMap.values()) { + dbAdaptorFactory.deleteCatalogDB(); + } + } + + @Override + public boolean isCatalogDBReady() throws CatalogDBException { + if (organizationDBAdaptorMap.isEmpty()) { return false; } + return getOrganizationMongoDBAdaptorFactory(ParamConstants.ADMIN_ORGANIZATION).isCatalogDBReady(); } @Override - public String getCatalogDatabase(String prefix) { - String database; - if (StringUtils.isNotEmpty(prefix)) { - if (!prefix.endsWith("_")) { - database = prefix + "_catalog"; - } else { - database = prefix + "catalog"; - } - } else { - database = "opencga_catalog"; + public void close() { + for (OrganizationMongoDBAdaptorFactory dbAdaptorFactory : organizationDBAdaptorMap.values()) { + dbAdaptorFactory.close(); } - return database; } @Override - public void deleteCatalogDB() throws CatalogDBException { - mongoManager.drop(database); + public void createIndexes(String organization) throws CatalogDBException { + OrganizationMongoDBAdaptorFactory orgFactory = getOrganizationMongoDBAdaptorFactory(organization); + orgFactory.createIndexes(); } @Override - public boolean isCatalogDBReady() { - return metaCollection.count(new BasicDBObject("id", METADATA_OBJECT_ID)).getNumMatches() == 1; + public List getOrganizationIds() throws CatalogDBException { + // Recheck in case there are new organizations + initOrganizations(configuration); + return new ArrayList<>(organizationDBAdaptorMap.keySet()); + } + + public MongoDataStore getMongoDataStore(String organization) throws CatalogDBException { + return getOrganizationMongoDBAdaptorFactory(organization).getMongoDataStore(); + } + + private void connect(Configuration catalogConfiguration) throws CatalogDBException { + // Init map of organization db adaptor factories + organizationDBAdaptorMap = new HashMap<>(); + initOrganizations(catalogConfiguration); + } + + private void initOrganizations(Configuration catalogConfiguration) throws CatalogDBException { + // Configure admin organization first + OrganizationMongoDBAdaptorFactory adminFactory; + if (organizationDBAdaptorMap.containsKey(ParamConstants.ADMIN_ORGANIZATION)) { + adminFactory = organizationDBAdaptorMap.get(ParamConstants.ADMIN_ORGANIZATION); + } else { + adminFactory = configureOrganizationMongoDBAdaptorFactory(ParamConstants.ADMIN_ORGANIZATION, catalogConfiguration); + organizationDBAdaptorMap.put(ParamConstants.ADMIN_ORGANIZATION, adminFactory); + } + if (adminFactory.isCatalogDBReady()) { + // Read organizations present in the installation + Query query = new Query(NoteDBAdaptor.QueryParams.TAGS.key(), OrganizationTag.ACTIVE.name()); + OpenCGAResult results = adminFactory.getCatalogNotesDBAdaptor().get(query, new QueryOptions()); + + for (Note organizationNote : results.getResults()) { + OrganizationSummary organizationSummary = getOrganizationSummary(organizationNote); + if (!ParamConstants.ADMIN_ORGANIZATION.equals(organizationSummary.getId()) + && (!organizationDBAdaptorMap.containsKey(organizationSummary.getId()))) { + OrganizationMongoDBAdaptorFactory orgFactory = configureOrganizationMongoDBAdaptorFactory(organizationSummary.getId(), + catalogConfiguration); + organizationDBAdaptorMap.put(organizationSummary.getId(), orgFactory); + } + } + } + } + + private OrganizationSummary getOrganizationSummary(Note note) { + try { + String orgSummaryString = JacksonUtils.getDefaultObjectMapper().writeValueAsString(note.getValue()); + return JacksonUtils.getDefaultObjectMapper().readerFor(OrganizationSummary.class).readValue(orgSummaryString); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private OrganizationMongoDBAdaptorFactory configureOrganizationMongoDBAdaptorFactory(String organizationId, Configuration configuration) + throws CatalogDBException { + return new OrganizationMongoDBAdaptorFactory(organizationId, mongoManager, mongoDbConfiguration, configuration, ioManagerFactory); + } + + public OrganizationMongoDBAdaptorFactory getOrganizationMongoDBAdaptorFactory(String organization) throws CatalogDBException { + return getOrganizationMongoDBAdaptorFactory(organization, true); + } + + private OrganizationMongoDBAdaptorFactory getOrganizationMongoDBAdaptorFactory(String organizationId, boolean raiseException) + throws CatalogDBException { + OrganizationMongoDBAdaptorFactory orgFactory = organizationDBAdaptorMap.get(organizationId); + if (orgFactory == null) { + if (!ParamConstants.ADMIN_ORGANIZATION.equals(organizationId)) { + orgFactory = getOrganizationMongoDBAdaptorFactory(ParamConstants.ADMIN_ORGANIZATION); + + // Read organizations present in the installation + Query query = new Query(NoteDBAdaptor.QueryParams.TAGS.key(), OrganizationTag.ACTIVE.name()); + OpenCGAResult results = orgFactory.getCatalogNotesDBAdaptor().get(query, new QueryOptions()); + + for (Note organizationNote : results.getResults()) { + OrganizationSummary organizationSummary = getOrganizationSummary(organizationNote); + if (organizationSummary.getId().equals(organizationId)) { + // Organization is present, so create new OrganizationMongoDBAdaptorFactory for the organization + OrganizationMongoDBAdaptorFactory organizationMongoDBAdaptorFactory = + new OrganizationMongoDBAdaptorFactory(organizationId, mongoManager, mongoDbConfiguration, configuration, + ioManagerFactory); + organizationDBAdaptorMap.put(organizationId, organizationMongoDBAdaptorFactory); + return organizationMongoDBAdaptorFactory; + } + } + } + + if (raiseException) { + throw new CatalogDBException("Could not find database for organization '" + organizationId + "'"); + } else { + return null; + } + } + return orgFactory; } @Override - public void close() { - mongoManager.close(mongoDataStore.getDatabaseName()); + public MigrationDBAdaptor getMigrationDBAdaptor(String organizationId) throws CatalogDBException { + return getOrganizationMongoDBAdaptorFactory(organizationId).getMigrationDBAdaptor(); } @Override - public MetaMongoDBAdaptor getCatalogMetaDBAdaptor() { - return metaDBAdaptor; + public MetaDBAdaptor getCatalogMetaDBAdaptor(String organizationId) throws CatalogDBException { + return getOrganizationMongoDBAdaptorFactory(organizationId).getCatalogMetaDBAdaptor(); } @Override - public UserMongoDBAdaptor getCatalogUserDBAdaptor() { - return userDBAdaptor; + public OpenCGAResult createOrganization(Organization organization, QueryOptions options, String userId) + throws CatalogException { + OrganizationMongoDBAdaptorFactory orgFactory = getOrganizationMongoDBAdaptorFactory(organization.getId(), false); + if (orgFactory != null && orgFactory.isCatalogDBReady()) { + throw new CatalogDBException("Organization '" + organization.getId() + "' already exists."); + } + + try { + // Create organization + OrganizationMongoDBAdaptorFactory organizationDBAdaptorFactory = new OrganizationMongoDBAdaptorFactory(organization.getId(), + mongoManager, mongoDbConfiguration, configuration, ioManagerFactory); + organizationDBAdaptorMap.put(organization.getId(), organizationDBAdaptorFactory); + + OrganizationSummary organizationSummary = new OrganizationSummary(organization.getId(), + organizationDBAdaptorFactory.getMongoDataStore().getDatabaseName(), OrganizationTag.ACTIVE.name(), null); + NoteCreateParams noteCreateParams = new NoteCreateParams(ORGANIZATION_PREFIX + organization.getId(), + Collections.singletonList(OrganizationTag.ACTIVE.name()), Note.Visibility.PRIVATE, Note.Type.OBJECT, null); + try { + String orgSummaryString = JacksonUtils.getDefaultObjectMapper().writeValueAsString(organizationSummary); + Map value = JacksonUtils.getDefaultObjectMapper().readerFor(Map.class).readValue(orgSummaryString); + noteCreateParams.setValue(value); + } catch (JsonMappingException e) { + throw new RuntimeException(e); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + Note note = noteCreateParams.toNote(Note.Scope.ORGANIZATION, userId); + NoteManager.validateNewNote(note, userId); + + // Create new database and indexes + organizationDBAdaptorFactory.createAllCollections(); + organizationDBAdaptorFactory.createIndexes(); + + // Create organization + OpenCGAResult result = organizationDBAdaptorFactory.getCatalogOrganizationDBAdaptor() + .insert(organization, options); + + // Keep track of current organization in the ADMIN organization + if (StringUtils.isNotEmpty(note.getUserId())) { + // Remove admin organization prefix from userId as it's written in that same organization + note.setUserId(note.getUserId().replace(ParamConstants.ADMIN_ORGANIZATION + ":", "")); + } + getOrganizationMongoDBAdaptorFactory(ParamConstants.ADMIN_ORGANIZATION).getCatalogNotesDBAdaptor().insert(note); + return result; + } catch (Exception e) { + OrganizationMongoDBAdaptorFactory tmpOrgFactory = organizationDBAdaptorMap.remove(organization.getId()); + if (tmpOrgFactory != null) { + tmpOrgFactory.deleteCatalogDB(); + } + // TODO: Delete settings from ADMIN database + + throw e; + } } @Override - public ProjectMongoDBAdaptor getCatalogProjectDbAdaptor() { - return projectDBAdaptor; + public void deleteOrganization(Organization organizationId) throws CatalogDBException { + OrganizationMongoDBAdaptorFactory orgFactory = getOrganizationMongoDBAdaptorFactory(organizationId.getId()); + orgFactory.deleteCatalogDB(); + orgFactory.close(); + organizationDBAdaptorMap.remove(organizationId.getId()); + + // TODO: Remove organization from ADMIN database } @Override - public StudyMongoDBAdaptor getCatalogStudyDBAdaptor() { - return studyDBAdaptor; + public NoteDBAdaptor getCatalogNoteDBAdaptor(String organizationId) throws CatalogDBException { + return getOrganizationMongoDBAdaptorFactory(organizationId).getCatalogNotesDBAdaptor(); } @Override - public SampleMongoDBAdaptor getCatalogSampleDBAdaptor() { - return sampleDBAdaptor; + public OrganizationDBAdaptor getCatalogOrganizationDBAdaptor(String organizationId) throws CatalogDBException { + return getOrganizationMongoDBAdaptorFactory(organizationId).getCatalogOrganizationDBAdaptor(); } @Override - public IndividualMongoDBAdaptor getCatalogIndividualDBAdaptor() { - return individualDBAdaptor; + public UserDBAdaptor getCatalogUserDBAdaptor(String organizationId) throws CatalogDBException { + return getOrganizationMongoDBAdaptorFactory(organizationId).getCatalogUserDBAdaptor(); } @Override - public FileMongoDBAdaptor getCatalogFileDBAdaptor() { - return fileDBAdaptor; + public ProjectDBAdaptor getCatalogProjectDbAdaptor(String organizationId) throws CatalogDBException { + return getOrganizationMongoDBAdaptorFactory(organizationId).getCatalogProjectDBAdaptor(); } @Override - public JobMongoDBAdaptor getCatalogJobDBAdaptor() { - return jobDBAdaptor; + public StudyDBAdaptor getCatalogStudyDBAdaptor(String organizationId) throws CatalogDBException { + return getOrganizationMongoDBAdaptorFactory(organizationId).getCatalogStudyDBAdaptor(); } @Override - public CohortMongoDBAdaptor getCatalogCohortDBAdaptor() { - return cohortDBAdaptor; + public FileDBAdaptor getCatalogFileDBAdaptor(String organizationId) throws CatalogDBException { + return getOrganizationMongoDBAdaptorFactory(organizationId).getCatalogFileDBAdaptor(); } @Override - public PanelMongoDBAdaptor getCatalogPanelDBAdaptor() { - return panelDBAdaptor; + public SampleDBAdaptor getCatalogSampleDBAdaptor(String organizationId) throws CatalogDBException { + return getOrganizationMongoDBAdaptorFactory(organizationId).getCatalogSampleDBAdaptor(); } @Override - public FamilyMongoDBAdaptor getCatalogFamilyDBAdaptor() { - return familyDBAdaptor; + public IndividualDBAdaptor getCatalogIndividualDBAdaptor(String organizationId) throws CatalogDBException { + return getOrganizationMongoDBAdaptorFactory(organizationId).getCatalogIndividualDBAdaptor(); } @Override - public ClinicalAnalysisMongoDBAdaptor getClinicalAnalysisDBAdaptor() { - return clinicalDBAdaptor; + public JobDBAdaptor getCatalogJobDBAdaptor(String organizationId) throws CatalogDBException { + return getOrganizationMongoDBAdaptorFactory(organizationId).getCatalogJobDBAdaptor(); } @Override - public InterpretationMongoDBAdaptor getInterpretationDBAdaptor() { - return interpretationDBAdaptor; + public AuditDBAdaptor getCatalogAuditDbAdaptor(String organizationId) throws CatalogDBException { + return getOrganizationMongoDBAdaptorFactory(organizationId).getCatalogAuditDbAdaptor(); } @Override - public MigrationDBAdaptor getMigrationDBAdaptor() { - return migrationDBAdaptor; + public CohortDBAdaptor getCatalogCohortDBAdaptor(String organizationId) throws CatalogDBException { + return getOrganizationMongoDBAdaptorFactory(organizationId).getCatalogCohortDBAdaptor(); } @Override - public Map getMongoDBCollectionMap() { - return collections; + public PanelDBAdaptor getCatalogPanelDBAdaptor(String organizationId) throws CatalogDBException { + return getOrganizationMongoDBAdaptorFactory(organizationId).getCatalogPanelDBAdaptor(); } @Override - public AuditMongoDBAdaptor getCatalogAuditDbAdaptor() { - return auditDBAdaptor; + public FamilyDBAdaptor getCatalogFamilyDBAdaptor(String organizationId) throws CatalogDBException { + return getOrganizationMongoDBAdaptorFactory(organizationId).getCatalogFamilyDBAdaptor(); } - public MongoDataStore getMongoDataStore() { - return mongoDataStore; + @Override + public ClinicalAnalysisDBAdaptor getClinicalAnalysisDBAdaptor(String organizationId) throws CatalogDBException { + return getOrganizationMongoDBAdaptorFactory(organizationId).getClinicalAnalysisDBAdaptor(); } - private void connect(Configuration catalogConfiguration) throws CatalogDBException { - mongoDataStore = mongoManager.get(database, configuration); - if (mongoDataStore == null) { - throw new CatalogDBException("Unable to connect to MongoDB"); - } + @Override + public InterpretationDBAdaptor getInterpretationDBAdaptor(String organizationId) throws CatalogDBException { + return getOrganizationMongoDBAdaptorFactory(organizationId).getInterpretationDBAdaptor(); + } - metaCollection = mongoDataStore.getCollection(METADATA_COLLECTION); - MongoDBCollection migrationCollection = mongoDataStore.getCollection(MIGRATION_COLLECTION); - - MongoDBCollection userCollection = mongoDataStore.getCollection(USER_COLLECTION); - MongoDBCollection studyCollection = mongoDataStore.getCollection(STUDY_COLLECTION); - MongoDBCollection fileCollection = mongoDataStore.getCollection(FILE_COLLECTION); - MongoDBCollection sampleCollection = mongoDataStore.getCollection(SAMPLE_COLLECTION); - MongoDBCollection individualCollection = mongoDataStore.getCollection(INDIVIDUAL_COLLECTION); - MongoDBCollection jobCollection = mongoDataStore.getCollection(JOB_COLLECTION); - MongoDBCollection cohortCollection = mongoDataStore.getCollection(COHORT_COLLECTION); - MongoDBCollection panelCollection = mongoDataStore.getCollection(PANEL_COLLECTION); - MongoDBCollection familyCollection = mongoDataStore.getCollection(FAMILY_COLLECTION); - MongoDBCollection clinicalCollection = mongoDataStore.getCollection(CLINICAL_ANALYSIS_COLLECTION); - MongoDBCollection interpretationCollection = mongoDataStore.getCollection(INTERPRETATION_COLLECTION); - - MongoDBCollection sampleArchivedCollection = mongoDataStore.getCollection(SAMPLE_ARCHIVE_COLLECTION); - MongoDBCollection individualArchivedCollection = mongoDataStore.getCollection(INDIVIDUAL_ARCHIVE_COLLECTION); - MongoDBCollection familyArchivedCollection = mongoDataStore.getCollection(FAMILY_ARCHIVE_COLLECTION); - MongoDBCollection panelArchivedCollection = mongoDataStore.getCollection(PANEL_ARCHIVE_COLLECTION); - MongoDBCollection interpretationArchivedCollection = mongoDataStore.getCollection(INTERPRETATION_ARCHIVE_COLLECTION); - - MongoDBCollection deletedUserCollection = mongoDataStore.getCollection(DELETED_USER_COLLECTION); - MongoDBCollection deletedStudyCollection = mongoDataStore.getCollection(DELETED_STUDY_COLLECTION); - MongoDBCollection deletedFileCollection = mongoDataStore.getCollection(DELETED_FILE_COLLECTION); - MongoDBCollection deletedSampleCollection = mongoDataStore.getCollection(DELETED_SAMPLE_COLLECTION); - MongoDBCollection deletedIndividualCollection = mongoDataStore.getCollection(DELETED_INDIVIDUAL_COLLECTION); - MongoDBCollection deletedJobCollection = mongoDataStore.getCollection(DELETED_JOB_COLLECTION); - MongoDBCollection deletedCohortCollection = mongoDataStore.getCollection(DELETED_COHORT_COLLECTION); - MongoDBCollection deletedPanelCollection = mongoDataStore.getCollection(DELETED_PANEL_COLLECTION); - MongoDBCollection deletedFamilyCollection = mongoDataStore.getCollection(DELETED_FAMILY_COLLECTION); - MongoDBCollection deletedClinicalCollection = mongoDataStore.getCollection(DELETED_CLINICAL_ANALYSIS_COLLECTION); - MongoDBCollection deletedInterpretationCollection = mongoDataStore.getCollection(DELETED_INTERPRETATION_COLLECTION); - - MongoDBCollection auditCollection = mongoDataStore.getCollection(AUDIT_COLLECTION); - - collections = new HashMap<>(); - collections.put(METADATA_COLLECTION, metaCollection); - collections.put(MIGRATION_COLLECTION, migrationCollection); - - collections.put(USER_COLLECTION, userCollection); - collections.put(STUDY_COLLECTION, studyCollection); - collections.put(FILE_COLLECTION, fileCollection); - collections.put(SAMPLE_COLLECTION, sampleCollection); - collections.put(INDIVIDUAL_COLLECTION, individualCollection); - collections.put(JOB_COLLECTION, jobCollection); - collections.put(COHORT_COLLECTION, cohortCollection); - collections.put(PANEL_COLLECTION, panelCollection); - collections.put(FAMILY_COLLECTION, familyCollection); - collections.put(CLINICAL_ANALYSIS_COLLECTION, clinicalCollection); - collections.put(INTERPRETATION_COLLECTION, interpretationCollection); - - collections.put(SAMPLE_ARCHIVE_COLLECTION, sampleArchivedCollection); - collections.put(INDIVIDUAL_ARCHIVE_COLLECTION, individualArchivedCollection); - collections.put(FAMILY_ARCHIVE_COLLECTION, familyArchivedCollection); - collections.put(PANEL_ARCHIVE_COLLECTION, panelArchivedCollection); - collections.put(INTERPRETATION_ARCHIVE_COLLECTION, interpretationArchivedCollection); - - collections.put(DELETED_USER_COLLECTION, deletedUserCollection); - collections.put(DELETED_STUDY_COLLECTION, deletedStudyCollection); - collections.put(DELETED_FILE_COLLECTION, deletedFileCollection); - collections.put(DELETED_SAMPLE_COLLECTION, deletedSampleCollection); - collections.put(DELETED_INDIVIDUAL_COLLECTION, deletedIndividualCollection); - collections.put(DELETED_JOB_COLLECTION, deletedJobCollection); - collections.put(DELETED_COHORT_COLLECTION, deletedCohortCollection); - collections.put(DELETED_PANEL_COLLECTION, deletedPanelCollection); - collections.put(DELETED_FAMILY_COLLECTION, deletedFamilyCollection); - collections.put(DELETED_CLINICAL_ANALYSIS_COLLECTION, deletedClinicalCollection); - collections.put(DELETED_INTERPRETATION_COLLECTION, deletedInterpretationCollection); - - collections.put(AUDIT_COLLECTION, auditCollection); - - fileDBAdaptor = new FileMongoDBAdaptor(fileCollection, deletedFileCollection, catalogConfiguration, this, ioManagerFactory); - familyDBAdaptor = new FamilyMongoDBAdaptor(familyCollection, familyArchivedCollection, deletedFamilyCollection, - catalogConfiguration, this); - individualDBAdaptor = new IndividualMongoDBAdaptor(individualCollection, individualArchivedCollection, deletedIndividualCollection, - catalogConfiguration, this); - jobDBAdaptor = new JobMongoDBAdaptor(jobCollection, deletedJobCollection, catalogConfiguration, this); - projectDBAdaptor = new ProjectMongoDBAdaptor(userCollection, deletedUserCollection, catalogConfiguration, this); - sampleDBAdaptor = new SampleMongoDBAdaptor(sampleCollection, sampleArchivedCollection, deletedSampleCollection, - catalogConfiguration, this); - studyDBAdaptor = new StudyMongoDBAdaptor(studyCollection, deletedStudyCollection, catalogConfiguration, this); - userDBAdaptor = new UserMongoDBAdaptor(userCollection, deletedUserCollection, catalogConfiguration, this); - cohortDBAdaptor = new CohortMongoDBAdaptor(cohortCollection, deletedCohortCollection, catalogConfiguration, this); - panelDBAdaptor = new PanelMongoDBAdaptor(panelCollection, panelArchivedCollection, deletedPanelCollection, catalogConfiguration, - this); - clinicalDBAdaptor = new ClinicalAnalysisMongoDBAdaptor(clinicalCollection, deletedClinicalCollection, catalogConfiguration, this); - interpretationDBAdaptor = new InterpretationMongoDBAdaptor(interpretationCollection, interpretationArchivedCollection, - deletedInterpretationCollection, catalogConfiguration, this); - metaDBAdaptor = new MetaMongoDBAdaptor(metaCollection, catalogConfiguration, this); - auditDBAdaptor = new AuditMongoDBAdaptor(auditCollection, catalogConfiguration); - migrationDBAdaptor = new MigrationMongoDBAdaptor(migrationCollection, catalogConfiguration, this); + public MongoDataStoreManager getMongoManager() { + return mongoManager; } + public MongoDBConfiguration getMongoDbConfiguration() { + return mongoDbConfiguration; + } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBUtils.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBUtils.java index 424d362e84c..acc75cddcae 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBUtils.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBUtils.java @@ -85,7 +85,7 @@ public class MongoDBUtils { @Deprecated static long getNewAutoIncrementId(String field, MongoDBCollection metaCollection) { - Bson query = Filters.eq(ID, MongoDBAdaptorFactory.METADATA_OBJECT_ID); + Bson query = Filters.eq(ID, OrganizationMongoDBAdaptorFactory.METADATA_OBJECT_ID); Document projection = new Document(field, true); Bson inc = Updates.inc(field, 1); QueryOptions queryOptions = new QueryOptions("returnNew", true); @@ -364,29 +364,26 @@ static void filterMapParams(ObjectMap parameters, Map filteredPa } } - static void filterObjectParams(ObjectMap parameters, Map filteredParams, String[] acceptedMapParams) { + static void filterObjectParams(ObjectMap parameters, Map filteredParams, String[] acceptedMapParams) + throws CatalogDBException { filterObjectParams(parameters, filteredParams, acceptedMapParams, ""); } static void filterObjectParams(ObjectMap parameters, Map filteredParams, String[] acceptedMapParams, - String dbKeyPrefix) { + String dbKeyPrefix) throws CatalogDBException { for (String s : acceptedMapParams) { if (parameters.containsKey(s)) { Document document; - try { - if (parameters.get(s) instanceof List) { - List originalList = parameters.getAsList(s); - List documentList = new ArrayList<>(originalList.size()); - for (Object object : originalList) { - documentList.add(getMongoDBDocument(object, s)); - } - filteredParams.put(dbKeyPrefix + s, documentList); - } else { - document = getMongoDBDocument(parameters.get(s), s); - filteredParams.put(dbKeyPrefix + s, document); + if (parameters.get(s) instanceof List) { + List originalList = parameters.getAsList(s); + List documentList = new ArrayList<>(originalList.size()); + for (Object object : originalList) { + documentList.add(getMongoDBDocument(object, s)); } - } catch (CatalogDBException e) { - logger.warn("Skipping key '" + s + "': " + e.getMessage(), e); + filteredParams.put(dbKeyPrefix + s, documentList); + } else { + document = getMongoDBDocument(parameters.get(s), s); + filteredParams.put(dbKeyPrefix + s, document); } } } @@ -810,7 +807,7 @@ static CatalogDBException ifDuplicateKeyException(Supplier"), null); + } + public static URI getMongoDBUri(DatabaseCredentials credentials, String database) { Map options = credentials.getOptions(); if (options == null) { diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/NoteMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/NoteMongoDBAdaptor.java new file mode 100644 index 00000000000..0b953919713 --- /dev/null +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/NoteMongoDBAdaptor.java @@ -0,0 +1,429 @@ +package org.opencb.opencga.catalog.db.mongodb; + +import com.mongodb.client.ClientSession; +import com.mongodb.client.model.Filters; +import org.apache.commons.lang3.StringUtils; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.opencb.commons.datastore.core.*; +import org.opencb.commons.datastore.mongodb.MongoDBCollection; +import org.opencb.commons.datastore.mongodb.MongoDBIterator; +import org.opencb.opencga.catalog.db.api.DBIterator; +import org.opencb.opencga.catalog.db.api.NoteDBAdaptor; +import org.opencb.opencga.catalog.db.mongodb.converters.NoteConverter; +import org.opencb.opencga.catalog.db.mongodb.iterators.CatalogMongoDBIterator; +import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; +import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.exceptions.CatalogParameterException; +import org.opencb.opencga.catalog.utils.Constants; +import org.opencb.opencga.catalog.utils.ParamUtils; +import org.opencb.opencga.catalog.utils.UuidUtils; +import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.config.Configuration; +import org.opencb.opencga.core.models.notes.Note; +import org.opencb.opencga.core.response.OpenCGAResult; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.function.Consumer; + +import static org.opencb.opencga.catalog.db.mongodb.MongoDBUtils.*; + +public class NoteMongoDBAdaptor extends CatalogMongoDBAdaptor implements NoteDBAdaptor { + + private final MongoDBCollection noteCollection; + private final MongoDBCollection archiveNoteCollection; + private final MongoDBCollection deletedNoteCollection; + private final NoteConverter noteConverter; + private final VersionedMongoDBAdaptor versionedMongoDBAdaptor; + + public NoteMongoDBAdaptor(MongoDBCollection noteCollection, MongoDBCollection archiveNoteCollection, + MongoDBCollection deletedNoteCollection, Configuration configuration, + OrganizationMongoDBAdaptorFactory dbAdaptorFactory) { + super(configuration, LoggerFactory.getLogger(NoteMongoDBAdaptor.class)); + this.dbAdaptorFactory = dbAdaptorFactory; + this.noteCollection = noteCollection; + this.archiveNoteCollection = archiveNoteCollection; + this.deletedNoteCollection = deletedNoteCollection; + this.noteConverter = new NoteConverter(); + this.versionedMongoDBAdaptor = new VersionedMongoDBAdaptor(noteCollection, archiveNoteCollection, + deletedNoteCollection); + } + + @Override + public OpenCGAResult insert(Note note) throws CatalogException { + return runTransaction(clientSession -> { + long tmpStartTime = startQuery(); + logger.debug("Starting note insert transaction for note id '{}'", note.getId()); + + insert(clientSession, note); + return endWrite(tmpStartTime, 1, 1, 0, 0, null); + }, e -> logger.error("Could not create note {}: {}", note.getId(), e.getMessage())); + } + + Note insert(ClientSession clientSession, Note note) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + if (StringUtils.isEmpty(note.getId())) { + throw new CatalogDBException("Missing note id"); + } + + if (note.getStudyUid() <= 0) { + note.setStudyUid(-1L); + } + // Check it doesn't already exist a note with same id and study + Bson bson = Filters.and( + Filters.eq(QueryParams.ID.key(), note.getId()), + Filters.eq(QueryParams.STUDY_UID.key(), note.getStudyUid()) + ); + DataResult count = noteCollection.count(clientSession, bson); + if (count.getNumMatches() > 0) { + throw new CatalogDBException("Note { id: '" + note.getId() + "'} already exists."); + } + long noteUid = getNewUid(clientSession); + note.setUid(noteUid); + if (StringUtils.isEmpty(note.getUuid())) { + note.setUuid(UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.NOTES)); + } + + Document noteObject = noteConverter.convertToStorageType(note); + + // Private parameters + noteObject.put(PRIVATE_CREATION_DATE, + StringUtils.isNotEmpty(note.getCreationDate()) ? TimeUtils.toDate(note.getCreationDate()) : TimeUtils.getDate()); + noteObject.put(PRIVATE_MODIFICATION_DATE, StringUtils.isNotEmpty(note.getModificationDate()) + ? TimeUtils.toDate(note.getModificationDate()) : TimeUtils.getDate()); + + logger.debug("Inserting note '{}'...", note.getId()); + versionedMongoDBAdaptor.insert(clientSession, noteObject); + logger.debug("Note '{}' successfully inserted", note.getId()); + + return note; + } + + @Override + public OpenCGAResult count(Query query) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + return null; + } + + @Override + public OpenCGAResult stats(Query query) { + return null; + } + + @Override + public OpenCGAResult get(Query query, QueryOptions options) throws CatalogDBException { + return get(null, query, options); + } + + OpenCGAResult get(ClientSession clientSession, Query query, QueryOptions options) throws CatalogDBException { + long startTime = startQuery(); + try (DBIterator dbIterator = iterator(clientSession, query, options)) { + return endQuery(startTime, dbIterator); + } + } + + @Override + public OpenCGAResult nativeGet(Query query, QueryOptions options) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + return nativeGet(null, query, options); + } + + + public OpenCGAResult nativeGet(ClientSession clientSession, Query query, QueryOptions options) throws CatalogDBException { + long startTime = startQuery(); + try (DBIterator dbIterator = nativeIterator(clientSession, query, options)) { + return endQuery(startTime, dbIterator); + } + } + + @Override + public OpenCGAResult update(long uid, ObjectMap parameters, QueryOptions queryOptions) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + try { + return runTransaction(clientSession -> privateUpdate(clientSession, uid, parameters, queryOptions)); + } catch (CatalogException e) { + logger.error("Could not update note: {}", e.getMessage(), e); + throw new CatalogDBException("Could not update note: " + e.getMessage(), e.getCause()); + } + } + + @Override + public OpenCGAResult update(Query query, ObjectMap parameters, QueryOptions queryOptions) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + return null; + } + + OpenCGAResult privateUpdate(ClientSession clientSession, long noteUid, ObjectMap parameters, QueryOptions queryOptions) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + long tmpStartTime = startQuery(); + + Query tmpQuery = new Query(QueryParams.UID.key(), noteUid); + Bson bsonQuery = parseQuery(tmpQuery); + List includeFields = Collections.singletonList(QueryParams.VALUE_TYPE.key()); + return versionedMongoDBAdaptor.update(clientSession, bsonQuery, includeFields, (noteList) -> { + Document note = noteList.get(0); + UpdateDocument updateParams = parseAndValidateUpdateParams(note, parameters, queryOptions); + Document noteUpdate = updateParams.toFinalUpdateDocument(); + + if (noteUpdate.isEmpty()) { + if (!parameters.isEmpty()) { + logger.error("Non-processed update parameters: {}", parameters.keySet()); + } + throw new CatalogDBException("Nothing to be updated"); + } + + logger.debug("Note update: query : {}, update: {}", bsonQuery.toBsonDocument(), noteUpdate.toBsonDocument()); + DataResult result = noteCollection.update(clientSession, bsonQuery, noteUpdate, new QueryOptions("multi", true)); + + if (result.getNumMatches() == 0) { + throw new CatalogDBException("Note '" + noteUid + "' not found"); + } + + List events = new ArrayList<>(); + if (result.getNumUpdated() == 0) { + events.add(new Event(Event.Type.WARNING, "", "Note was already updated")); + } + + logger.debug("Note '{}' successfully updated", noteUid); + + return endWrite(tmpStartTime, 1, 1, events); + }); + } + + private UpdateDocument parseAndValidateUpdateParams(Document note, ObjectMap parameters, QueryOptions queryOptions) + throws CatalogDBException { + UpdateDocument document = new UpdateDocument(); + +// if (StringUtils.isNotEmpty(parameters.getString(QueryParams.CREATION_DATE.key()))) { +// String time = parameters.getString(QueryParams.CREATION_DATE.key()); +// Date date = TimeUtils.toDate(time); +// document.getSet().put(QueryParams.CREATION_DATE.key(), time); +// document.getSet().put(PRIVATE_CREATION_DATE, date); +// } +// if (StringUtils.isNotEmpty(parameters.getString(MODIFICATION_DATE.key()))) { +// String time = parameters.getString(QueryParams.MODIFICATION_DATE.key()); +// Date date = TimeUtils.toDate(time); +// document.getSet().put(QueryParams.MODIFICATION_DATE.key(), time); +// document.getSet().put(PRIVATE_MODIFICATION_DATE, date); +// } + + final String[] acceptedStringParams = {QueryParams.USER_ID.key(), QueryParams.VISIBILITY.key()}; + filterStringParams(parameters, document.getSet(), acceptedStringParams); + + Object value = parameters.get(QueryParams.VALUE.key()); + if (value != null) { + String valueTypeStr = note.getString(QueryParams.VALUE_TYPE.key()); + Note.Type type = Note.Type.valueOf(valueTypeStr); + if (type.equals(Note.Type.OBJECT)) { + final String[] acceptedValueParams = {QueryParams.VALUE.key()}; + filterObjectParams(parameters, document.getSet(), acceptedValueParams); + } else { + document.getSet().put(QueryParams.VALUE.key(), value); + } + } + + Map actionMap = queryOptions.getMap(Constants.ACTIONS, new HashMap<>()); + // Tags + if (parameters.containsKey(QueryParams.TAGS.key())) { + ParamUtils.BasicUpdateAction operation = ParamUtils.BasicUpdateAction.from(actionMap, QueryParams.TAGS.key(), + ParamUtils.BasicUpdateAction.ADD); + String[] tagsParams = {QueryParams.TAGS.key()}; + switch (operation) { + case SET: + filterStringListParams(parameters, document.getSet(), tagsParams); + break; + case REMOVE: + filterStringListParams(parameters, document.getPullAll(), tagsParams); + break; + case ADD: + filterStringListParams(parameters, document.getAddToSet(), tagsParams); + break; + default: + throw new IllegalStateException("Unknown operation " + operation); + } + } + + if (!document.toFinalUpdateDocument().isEmpty()) { + String time = TimeUtils.getTime(); + if (StringUtils.isEmpty(parameters.getString(QueryParams.MODIFICATION_DATE.key()))) { + // Update modificationDate param + Date date = TimeUtils.toDate(time); + document.getSet().put(QueryParams.MODIFICATION_DATE.key(), time); + document.getSet().put(PRIVATE_MODIFICATION_DATE, date); + } + } + + return document; + } + + @Override + public OpenCGAResult delete(Note note) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + try { + return runTransaction(clientSession -> privateDelete(clientSession, note)); + } catch (CatalogException e) { + throw new CatalogDBException("Could not delete note " + note.getId() + ": " + e.getMessage(), e); + } + } + + private OpenCGAResult privateDelete(ClientSession clientSession, Note note) + throws CatalogDBException { + long tmpStartTime = startQuery(); + + logger.debug("Deleting note {} ({})", note.getId(), note.getUid()); + + // Delete note + Query noteQuery = new Query(QueryParams.UID.key(), note.getUid()); + Bson bsonQuery = parseQuery(noteQuery); + versionedMongoDBAdaptor.delete(clientSession, bsonQuery); + logger.debug("Note {}({}) deleted", note.getId(), note.getUid()); + return endWrite(tmpStartTime, 1, 0, 0, 1, Collections.emptyList()); + } + + @Override + public OpenCGAResult delete(Query query) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + return null; + } + + @Override + public OpenCGAResult restore(long id, QueryOptions queryOptions) throws CatalogDBException { + return null; + } + + @Override + public OpenCGAResult restore(Query query, QueryOptions queryOptions) throws CatalogDBException { + return null; + } + + @Override + public DBIterator iterator(Query query, QueryOptions options) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + return null; + } + + DBIterator iterator(ClientSession clientSession, Query query, QueryOptions options) throws CatalogDBException { + MongoDBIterator mongoCursor = getMongoCursor(clientSession, query, options); + return new CatalogMongoDBIterator<>(mongoCursor, noteConverter); + } + + @Override + public DBIterator nativeIterator(Query query, QueryOptions options) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + return nativeIterator(null, query, options); + } + + DBIterator nativeIterator(ClientSession clientSession, Query query, QueryOptions options) throws CatalogDBException { + QueryOptions queryOptions = options != null ? new QueryOptions(options) : new QueryOptions(); + queryOptions.put(NATIVE_QUERY, true); + MongoDBIterator mongoCursor = getMongoCursor(clientSession, query, queryOptions); + return new CatalogMongoDBIterator<>(mongoCursor, clientSession, null, null); + } + + private MongoDBIterator getMongoCursor(ClientSession clientSession, Query query, QueryOptions options) + throws CatalogDBException { + Query finalQuery = new Query(query); + + QueryOptions qOptions; + if (options != null) { + qOptions = new QueryOptions(options); + } else { + qOptions = new QueryOptions(); + } + + Bson bson = parseQuery(finalQuery); + MongoDBCollection collection = getQueryCollection(finalQuery, noteCollection, archiveNoteCollection, + deletedNoteCollection); + logger.debug("Note query: {}", bson.toBsonDocument()); + return collection.iterator(clientSession, bson, null, null, qOptions); + } + + @Override + public OpenCGAResult rank(Query query, String field, int numResults, boolean asc) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + return null; + } + + @Override + public OpenCGAResult groupBy(Query query, String field, QueryOptions options) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + return null; + } + + @Override + public OpenCGAResult groupBy(Query query, List fields, QueryOptions options) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + return null; + } + + @Override + public void forEach(Query query, Consumer action, QueryOptions options) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + + } + + private Bson parseQuery(Query query) throws CatalogDBException { + List andBsonList = new ArrayList<>(); + + Query queryCopy = new Query(query); + + if ("all".equalsIgnoreCase(queryCopy.getString(QueryParams.VERSION.key()))) { + queryCopy.put(Constants.ALL_VERSIONS, true); + queryCopy.remove(QueryParams.VERSION.key()); + } + + for (Map.Entry entry : queryCopy.entrySet()) { + String key = entry.getKey().split("\\.")[0]; + QueryParams queryParam = QueryParams.getParam(entry.getKey()) != null ? QueryParams.getParam(entry.getKey()) + : QueryParams.getParam(key); + if (queryParam == null) { + if (Constants.ALL_VERSIONS.equals(entry.getKey())) { + continue; + } + throw new CatalogDBException("Unexpected parameter " + entry.getKey() + ". The parameter does not exist or cannot be " + + "queried for."); + } + try { + switch (queryParam) { + case UID: + addAutoOrQuery(PRIVATE_UID, queryParam.key(), queryCopy, queryParam.type(), andBsonList); + break; + case STUDY_UID: + addAutoOrQuery(PRIVATE_STUDY_UID, queryParam.key(), queryCopy, queryParam.type(), andBsonList); + break; + case CREATION_DATE: + addAutoOrQuery(PRIVATE_CREATION_DATE, queryParam.key(), queryCopy, queryParam.type(), andBsonList); + break; + case MODIFICATION_DATE: + addAutoOrQuery(PRIVATE_MODIFICATION_DATE, queryParam.key(), query, queryParam.type(), andBsonList); + break; + case ID: + case UUID: + case SCOPE: + case USER_ID: + case TAGS: + case VERSION: + case VISIBILITY: + addAutoOrQuery(queryParam.key(), queryParam.key(), queryCopy, queryParam.type(), andBsonList); + break; + default: + throw new CatalogDBException("Cannot query by parameter " + queryParam.key()); + } + } catch (Exception e) { + if (e instanceof CatalogDBException) { + throw e; + } else { + throw new CatalogDBException("Error parsing query : " + queryCopy.toJson(), e); + } + } + } + + if (!andBsonList.isEmpty()) { + return Filters.and(andBsonList); + } else { + return new Document(); + } + } + + +} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptor.java new file mode 100644 index 00000000000..b971f5a5740 --- /dev/null +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptor.java @@ -0,0 +1,496 @@ +package org.opencb.opencga.catalog.db.mongodb; + +import com.mongodb.client.ClientSession; +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Updates; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.opencb.commons.datastore.core.*; +import org.opencb.commons.datastore.mongodb.MongoDBCollection; +import org.opencb.commons.datastore.mongodb.MongoDBIterator; +import org.opencb.opencga.catalog.db.api.DBIterator; +import org.opencb.opencga.catalog.db.api.OrganizationDBAdaptor; +import org.opencb.opencga.catalog.db.api.UserDBAdaptor; +import org.opencb.opencga.catalog.db.mongodb.converters.OrganizationConverter; +import org.opencb.opencga.catalog.db.mongodb.iterators.OrganizationCatalogMongoDBIterator; +import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; +import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.exceptions.CatalogParameterException; +import org.opencb.opencga.catalog.managers.OrganizationManager; +import org.opencb.opencga.catalog.utils.Constants; +import org.opencb.opencga.catalog.utils.ParamUtils; +import org.opencb.opencga.catalog.utils.UuidUtils; +import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.config.AuthenticationOrigin; +import org.opencb.opencga.core.config.Configuration; +import org.opencb.opencga.core.models.organizations.Organization; +import org.opencb.opencga.core.response.OpenCGAResult; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.stream.Collectors; + +import static org.opencb.opencga.catalog.db.api.ClinicalAnalysisDBAdaptor.QueryParams.MODIFICATION_DATE; +import static org.opencb.opencga.catalog.db.mongodb.MongoDBUtils.*; + +public class OrganizationMongoDBAdaptor extends MongoDBAdaptor implements OrganizationDBAdaptor { + + private final MongoDBCollection organizationCollection; + private final OrganizationConverter organizationConverter; + + private final String ID_COUNTER = "_idCounter"; + + public OrganizationMongoDBAdaptor(MongoDBCollection organizationCollection, Configuration configuration, + OrganizationMongoDBAdaptorFactory dbAdaptorFactory) { + super(configuration, LoggerFactory.getLogger(OrganizationMongoDBAdaptor.class)); + this.dbAdaptorFactory = dbAdaptorFactory; + this.organizationCollection = organizationCollection; + this.organizationConverter = new OrganizationConverter(); + } + + @Override + public OpenCGAResult insert(Organization organization, QueryOptions options) throws CatalogException { + return runTransaction(clientSession -> { + long tmpStartTime = startQuery(); + logger.debug("Starting organization insert transaction for organization id '{}'", organization.getId()); + insert(clientSession, organization); + return endWrite(tmpStartTime, 1, 1, 0, 0, null); + }, e -> logger.error("Could not create sample {}: {}", organization.getId(), e.getMessage())); + } + + Organization insert(ClientSession clientSession, Organization organization) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + + if (StringUtils.isEmpty(organization.getId())) { + throw new CatalogDBException("Missing organization id"); + } + + // Check the organization does not exist + List filterList = new ArrayList<>(); + filterList.add(Filters.eq(QueryParams.ID.key(), organization.getId())); + + Bson bson = Filters.and(filterList); + DataResult count = organizationCollection.count(clientSession, bson); + if (count.getNumMatches() > 0) { + throw new CatalogDBException("Organization { id: '" + organization.getId() + "'} already exists."); + } + if (StringUtils.isEmpty(organization.getUuid())) { + organization.setUuid(UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.ORGANIZATION)); + } + + Document organizationObject = organizationConverter.convertToStorageType(organization); + + // Counter private parameter + organizationObject.put(ID_COUNTER, 0L); + + // Versioning private parameters + organizationObject.put(PRIVATE_CREATION_DATE, StringUtils.isNotEmpty(organization.getCreationDate()) + ? TimeUtils.toDate(organization.getCreationDate()) : TimeUtils.getDate()); + organizationObject.put(PRIVATE_MODIFICATION_DATE, StringUtils.isNotEmpty(organization.getModificationDate()) + ? TimeUtils.toDate(organization.getModificationDate()) : TimeUtils.getDate()); + + logger.debug("Inserting organization '{}'...", organization.getId()); + organizationCollection.insert(clientSession, organizationObject, null); + logger.debug("Organization '{}' successfully inserted", organization.getId()); + + return organization; + } + + @Override + public OpenCGAResult get(String user, QueryOptions options) throws CatalogDBException { + return get(null, user, options); + } + + @Override + public OpenCGAResult get(QueryOptions options) throws CatalogDBException { + return get(null, null, options); + } + + OpenCGAResult get(ClientSession clientSession, String user, QueryOptions options) throws CatalogDBException { + long startTime = startQuery(); + try (DBIterator dbIterator = iterator(clientSession, user, options)) { + return endQuery(startTime, dbIterator); + } + } + + + @Override + public OpenCGAResult update(String organizationId, ObjectMap parameters, QueryOptions queryOptions) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + try { + QueryOptions options = queryOptions != null ? new QueryOptions(queryOptions) : QueryOptions.empty(); + return runTransaction(clientSession -> privateUpdate(clientSession, organizationId, parameters, options)); + } catch (CatalogException e) { + logger.error("Could not update organization {}: {}", organizationId, e.getMessage(), e); + throw new CatalogDBException("Could not update organization " + organizationId + ": " + e.getMessage(), + e.getCause()); + } + } + + private OpenCGAResult privateUpdate(ClientSession clientSession, String organizationId, ObjectMap parameters, + QueryOptions queryOptions) throws CatalogParameterException, CatalogDBException { + long tmpStartTime = startQuery(); + + UpdateDocument updateDocument = getValidatedUpdateParams(clientSession, parameters, queryOptions); + Document organizationUpdate = updateDocument.toFinalUpdateDocument(); + + if (organizationUpdate.isEmpty() && CollectionUtils.isEmpty(updateDocument.getNestedUpdateList())) { + if (!parameters.isEmpty()) { + logger.error("Non-processed update parameters: {}", parameters.keySet()); + throw new CatalogDBException("Update could not be performed. Some fields could not be processed."); + } + throw new CatalogDBException("Nothing to be updated"); + } + + // Update study admins need to be executed before the actual update because we need to fetch the previous owner/admins in case + // of an update on these fields. + updateStudyAdmins(clientSession, parameters, queryOptions); + + List events = new ArrayList<>(); + DataResult updateResult; + if (!organizationUpdate.isEmpty()) { + Bson queryBson = Filters.eq(QueryParams.ID.key(), organizationId); + logger.debug("Update organization. Query: {}, Update: {}", queryBson.toBsonDocument(), organizationUpdate.toBsonDocument()); + + updateResult = organizationCollection.update(clientSession, queryBson, organizationUpdate, null); + + if (updateResult.getNumMatches() == 0) { + throw new CatalogDBException("Organization not found"); + } + if (updateResult.getNumUpdated() == 0) { + events.add(new Event(Event.Type.WARNING, organizationId, "Organization was already updated")); + } + } + if (CollectionUtils.isNotEmpty(updateDocument.getNestedUpdateList())) { + for (NestedArrayUpdateDocument nestedDocument : updateDocument.getNestedUpdateList()) { + Bson bsonQuery = new Document(nestedDocument.getQuery()); + logger.debug("Update nested element from Organization. Query: {}, Update: {}", bsonQuery.toBsonDocument(), + nestedDocument.getSet()); + DataResult result = organizationCollection.update(clientSession, bsonQuery, nestedDocument.getSet(), null); + if (result.getNumMatches() == 0) { + throw new CatalogDBException("Couldn't update organization. Nothing could be found for query " + + bsonQuery.toBsonDocument()); + } + } + /* + for (NestedArrayUpdateDocument nestedDocument : updateDocument.getNestedUpdateList()) { + Bson nestedBsonQuery = parseQuery(nestedDocument.getQuery() + .append(QueryParams.UID.key(), interpretation.getUid())); + logger.debug("Update nested element from interpretation. Query: {}, Update: {}", + nestedBsonQuery.toBsonDocument(), nestedDocument.getSet()); + + update = interpretationCollection.update(clientSession, nestedBsonQuery, nestedDocument.getSet(), null); + + if (update.getNumMatches() == 0) { + throw CatalogDBException.uidNotFound("Interpretation", interpretationUid); + } + } + * */ + } + + logger.debug("Organization {} successfully updated", organizationId); + + + return endWrite(tmpStartTime, 1, 1, events); + } + + private void updateStudyAdmins(ClientSession clientSession, ObjectMap parameters, QueryOptions options) throws CatalogDBException { + if (!parameters.containsKey(QueryParams.OWNER.key()) && !parameters.containsKey(QueryParams.ADMINS.key())) { + return; + } + + Organization organization = get(clientSession, null, OrganizationManager.INCLUDE_ORGANIZATION_ADMINS).first(); + + if (parameters.containsKey(QueryParams.OWNER.key())) { + // Owner has changed + String newOwner = parameters.getString(QueryParams.OWNER.key()); + // Only do changes if the owner actually changes + if (!newOwner.equals(organization.getOwner())) { + if (!organization.getAdmins().contains(organization.getOwner())) { + // Remove Owner from all @admins groups + logger.info("Removing old owner '{}' from all '{}' groups", organization.getOwner(), ParamConstants.ADMINS_GROUP); + dbAdaptorFactory.getCatalogStudyDBAdaptor().removeUsersFromAdminsGroup(clientSession, + Collections.singletonList(organization.getOwner())); + } + // Add new owner to @admins group + logger.info("Adding new owner '{}' to all '{}' groups", newOwner, ParamConstants.ADMINS_GROUP); + dbAdaptorFactory.getCatalogStudyDBAdaptor().addUsersToAdminsAndMembersGroup(clientSession, + Collections.singletonList(newOwner)); + } + } + + if (parameters.containsKey(QueryParams.ADMINS.key())) { + List admins = parameters.getAsStringList(QueryParams.ADMINS.key()); + if (CollectionUtils.isNotEmpty(admins)) { + Map actionMap = options.getMap(Constants.ACTIONS, new HashMap<>()); + ParamUtils.AddRemoveAction operation = ParamUtils.AddRemoveAction.from(actionMap, QueryParams.ADMINS.key(), + ParamUtils.AddRemoveAction.ADD); + + switch (operation) { + case ADD: + // Add new admins to @admins group + logger.info("Adding new admins '{}' to all '{}' groups", admins, ParamConstants.ADMINS_GROUP); + dbAdaptorFactory.getCatalogStudyDBAdaptor().addUsersToAdminsAndMembersGroup(clientSession, admins); + break; + case REMOVE: + // Fetch current organization owner + String newOwner = parameters.getString(QueryParams.OWNER.key()); + String owner = StringUtils.isNotEmpty(newOwner) ? newOwner : organization.getOwner(); + + // Remove organization owner in case is one of the removed admins + admins = admins.stream().filter(a -> !owner.equals(a)).collect(Collectors.toList()); + logger.info("Removing old admins '{}' from all '{}' groups", admins, ParamConstants.ADMINS_GROUP); + dbAdaptorFactory.getCatalogStudyDBAdaptor().removeUsersFromAdminsGroup(clientSession, admins); + break; + default: + throw new CatalogDBException("Unexpected " + QueryParams.ADMINS.key() + " action"); + } + } + } + } + + private UpdateDocument getValidatedUpdateParams(ClientSession clientSession, ObjectMap parameters, QueryOptions queryOptions) + throws CatalogParameterException, CatalogDBException { + checkUpdatedParams(parameters, Arrays.asList(QueryParams.NAME.key(), QueryParams.OWNER.key(), + QueryParams.CREATION_DATE.key(), QueryParams.MODIFICATION_DATE.key(), QueryParams.ADMINS.key(), + QueryParams.CONFIGURATION.key(), QueryParams.ATTRIBUTES.key())); + + UpdateDocument document = new UpdateDocument(); + + String[] acceptedParams = { QueryParams.NAME.key() }; + filterStringParams(parameters, document.getSet(), acceptedParams); + + String[] acceptedObjectParams = { QueryParams.CONFIGURATION_OPTIMIZATIONS.key(), QueryParams.CONFIGURATION_TOKEN.key() }; + filterObjectParams(parameters, document.getSet(), acceptedObjectParams); + + // Authentication Origins action + if (parameters.containsKey(QueryParams.CONFIGURATION_AUTHENTICATION_ORIGINS.key())) { + Map actionMap = queryOptions.getMap(Constants.ACTIONS, new HashMap<>()); + ParamUtils.UpdateAction operation = ParamUtils.UpdateAction.from(actionMap, OrganizationDBAdaptor.AUTH_ORIGINS_FIELD); + String[] authOriginsParams = {QueryParams.CONFIGURATION_AUTHENTICATION_ORIGINS.key()}; + switch (operation) { + case SET: + filterObjectParams(parameters, document.getSet(), authOriginsParams); + break; + case REMOVE: + fixAuthOriginsForRemoval(parameters); + filterObjectParams(parameters, document.getPull(), authOriginsParams); + break; + case ADD: + filterObjectParams(parameters, document.getAddToSet(), authOriginsParams); + break; + case REPLACE: + filterReplaceParams(parameters.getAsList(QueryParams.CONFIGURATION_AUTHENTICATION_ORIGINS.key(), Map.class), document, + m -> String.valueOf(m.get("id")), QueryParams.CONFIGURATION_AUTHENTICATION_ORIGINS_ID.key()); + break; + default: + throw new IllegalStateException("Unknown operation " + operation); + } + } + + String owner = parameters.getString(QueryParams.OWNER.key(), null); + if (StringUtils.isNotEmpty(owner)) { + // Check user exists + OpenCGAResult count = dbAdaptorFactory.getCatalogUserDBAdaptor().count(clientSession, + new Query(UserDBAdaptor.QueryParams.ID.key(), owner)); + if (count.getNumMatches() == 0) { + throw new CatalogDBException("Could not update owner. User not found."); + } + // Fetch current owner + Organization organization = get(clientSession, null, OrganizationManager.INCLUDE_ORGANIZATION_ADMINS).first(); + if (owner.equals(organization.getOwner())) { + logger.warn("Organization owner is already '{}'.", owner); + } else { + document.getSet().put(QueryParams.OWNER.key(), owner); + } + } + + if (StringUtils.isNotEmpty(parameters.getString(QueryParams.CREATION_DATE.key()))) { + String time = parameters.getString(QueryParams.CREATION_DATE.key()); + Date date = TimeUtils.toDate(time); + document.getSet().put(QueryParams.CREATION_DATE.key(), time); + document.getSet().put(PRIVATE_CREATION_DATE, date); + } + if (StringUtils.isNotEmpty(parameters.getString(MODIFICATION_DATE.key()))) { + String time = parameters.getString(QueryParams.MODIFICATION_DATE.key()); + Date date = TimeUtils.toDate(time); + document.getSet().put(QueryParams.MODIFICATION_DATE.key(), time); + document.getSet().put(PRIVATE_MODIFICATION_DATE, date); + } + + if (parameters.containsKey(QueryParams.ADMINS.key())) { + List adminList = parameters.getAsStringList(QueryParams.ADMINS.key()).stream().distinct().collect(Collectors.toList()); + + // Check users exist + OpenCGAResult count = dbAdaptorFactory.getCatalogUserDBAdaptor().count(clientSession, + new Query(UserDBAdaptor.QueryParams.ID.key(), adminList)); + if (count.getNumMatches() < adminList.size()) { + throw new CatalogDBException("Could not update admins. Some users were not found."); + } + + Map actionMap = queryOptions.getMap(Constants.ACTIONS, new HashMap<>()); + ParamUtils.AddRemoveAction operation = ParamUtils.AddRemoveAction.from(actionMap, QueryParams.ADMINS.key(), + ParamUtils.AddRemoveAction.ADD); + if (!adminList.isEmpty()) { + switch (operation) { + case REMOVE: + document.getPullAll().put(QueryParams.ADMINS.key(), adminList); + break; + case ADD: + document.getAddToSet().put(QueryParams.ADMINS.key(), adminList); + break; + default: + throw new IllegalArgumentException("Unknown operation " + operation); + } + } + } + + String[] acceptedMapParams = {QueryParams.ATTRIBUTES.key(), }; + filterMapParams(parameters, document.getSet(), acceptedMapParams); + + if (!document.toFinalUpdateDocument().isEmpty()) { + String time = TimeUtils.getTime(); + if (StringUtils.isEmpty(parameters.getString(MODIFICATION_DATE.key()))) { + // Update modificationDate param + Date date = TimeUtils.toDate(time); + document.getSet().put(QueryParams.MODIFICATION_DATE.key(), time); + document.getSet().put(PRIVATE_MODIFICATION_DATE, date); + } + document.getSet().put(INTERNAL_LAST_MODIFIED, time); + } + + return document; + } + + private void fixAuthOriginsForRemoval(ObjectMap parameters) { + if (parameters.get(QueryParams.CONFIGURATION_AUTHENTICATION_ORIGINS.key()) == null) { + return; + } + List authOriginParamList = new LinkedList<>(); + for (Object authOrigin : parameters.getAsList(QueryParams.CONFIGURATION_AUTHENTICATION_ORIGINS.key())) { + if (authOrigin instanceof AuthenticationOrigin) { + authOriginParamList.add(new Document("id", ((AuthenticationOrigin) authOrigin).getId())); + } else { + authOriginParamList.add(new Document("id", ((Map) authOrigin).get("id"))); + } + } + parameters.putNested(QueryParams.CONFIGURATION_AUTHENTICATION_ORIGINS.key(), authOriginParamList, false); + } + + @Override + public OpenCGAResult delete(Organization organization) throws CatalogDBException { + return null; + } + + @Override + public DBIterator iterator(QueryOptions options) throws CatalogDBException { + return iterator(null, null, options); + } + + public DBIterator iterator(ClientSession clientSession, String user, QueryOptions options) throws CatalogDBException { + MongoDBIterator mongoCursor = getMongoCursor(clientSession, options); + return new OrganizationCatalogMongoDBIterator<>(mongoCursor, clientSession, organizationConverter, dbAdaptorFactory, options, user); + } + + private MongoDBIterator getMongoCursor(ClientSession clientSession, QueryOptions options) { + QueryOptions qOptions; + if (options != null) { + qOptions = new QueryOptions(options); + } else { + qOptions = new QueryOptions(); + } + qOptions = filterQueryOptionsToIncludeKeys(qOptions, + OrganizationManager.INCLUDE_ORGANIZATION_IDS.getAsStringList(QueryOptions.INCLUDE)); + + return organizationCollection.iterator(clientSession, new Document(), null, null, qOptions); + } + + public List getOwnerAndAdmins(ClientSession clientSession) throws CatalogDBException { + Organization organization = get(clientSession, null, OrganizationManager.INCLUDE_ORGANIZATION_ADMINS).first(); + List members = new ArrayList<>(organization.getAdmins().size() + 1); + if (StringUtils.isNotEmpty(organization.getOwner())) { + members.add(organization.getOwner()); + } else { + logger.warn("No owner found for organization {}", organization.getId()); + } + members.addAll(organization.getAdmins()); + return members; + } + + public boolean isOwnerOrAdmin(ClientSession clientSession, String userId) throws CatalogDBException { + return getOwnerAndAdmins(clientSession).contains(userId); + } + + public long getNewAutoIncrementId() { + return getNewAutoIncrementId(null, ID_COUNTER); //, metaCollection + } + + public long getNewAutoIncrementId(ClientSession clientSession) { + return getNewAutoIncrementId(clientSession, ID_COUNTER); //, metaCollection + } + + public long getNewAutoIncrementId(ClientSession clientSession, String field) { //, MongoDBCollection metaCollection + Bson query = new Document(); + Document projection = new Document(field, true); + Bson inc = Updates.inc(field, 1L); + QueryOptions queryOptions = new QueryOptions("returnNew", true); + DataResult result = organizationCollection.findAndUpdate(clientSession, query, projection, null, inc, queryOptions); + return result.getResults().get(0).getLong(field); + } + +// private Bson parseQuery(Query query) throws CatalogDBException { +// List andBsonList = new ArrayList<>(); +// +// Query queryCopy = new Query(query); +// queryCopy.remove(ParamConstants.DELETED_PARAM); +// +// for (Map.Entry entry : queryCopy.entrySet()) { +// String key = entry.getKey().split("\\.")[0]; +// QueryParams queryParam = QueryParams.getParam(entry.getKey()) != null ? QueryParams.getParam(entry.getKey()) +// : QueryParams.getParam(key); +// if (queryParam == null) { +// throw new CatalogDBException("Unexpected parameter " + entry.getKey() + ". The parameter does not exist or cannot be " +// + "queried for."); +// } +// try { +// switch (queryParam) { +// case UID: +// addAutoOrQuery(PRIVATE_UID, queryParam.key(), queryCopy, queryParam.type(), andBsonList); +// break; +// case CREATION_DATE: +// addAutoOrQuery(PRIVATE_CREATION_DATE, queryParam.key(), queryCopy, queryParam.type(), andBsonList); +// break; +// case MODIFICATION_DATE: +// addAutoOrQuery(PRIVATE_MODIFICATION_DATE, queryParam.key(), query, queryParam.type(), andBsonList); +// break; +// case ID: +// case UUID: +// case NAME: +// case DOMAIN: +// case OWNER: +// case ADMINS: +// addAutoOrQuery(queryParam.key(), queryParam.key(), queryCopy, queryParam.type(), andBsonList); +// break; +// default: +// throw new CatalogDBException("Cannot query by parameter " + queryParam.key()); +// } +// } catch (Exception e) { +// if (e instanceof CatalogDBException) { +// throw e; +// } else { +// throw new CatalogDBException("Error parsing query : " + queryCopy.toJson(), e); +// } +// } +// } +// +// if (andBsonList.size() > 0) { +// return Filters.and(andBsonList); +// } else { +// return new Document(); +// } +// } +} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptorFactory.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptorFactory.java new file mode 100644 index 00000000000..9884bbccc6c --- /dev/null +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptorFactory.java @@ -0,0 +1,498 @@ +package org.opencb.opencga.catalog.db.mongodb; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.StopWatch; +import org.bson.Document; +import org.opencb.commons.datastore.core.DataResult; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.datastore.mongodb.MongoDBCollection; +import org.opencb.commons.datastore.mongodb.MongoDBConfiguration; +import org.opencb.commons.datastore.mongodb.MongoDataStore; +import org.opencb.commons.datastore.mongodb.MongoDataStoreManager; +import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.io.IOManagerFactory; +import org.opencb.opencga.core.config.Admin; +import org.opencb.opencga.core.config.Configuration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static org.opencb.opencga.core.common.JacksonUtils.getDefaultObjectMapper; + +public class OrganizationMongoDBAdaptorFactory { + + public static final String NOTE_COLLECTION = "note"; + public static final String ORGANIZATION_COLLECTION = "organization"; + public static final String USER_COLLECTION = "user"; + public static final String PROJECT_COLLECTION = "project"; + public static final String STUDY_COLLECTION = "study"; + public static final String FILE_COLLECTION = "file"; + public static final String JOB_COLLECTION = "job"; + public static final String SAMPLE_COLLECTION = "sample"; + public static final String INDIVIDUAL_COLLECTION = "individual"; + public static final String COHORT_COLLECTION = "cohort"; + public static final String FAMILY_COLLECTION = "family"; + public static final String PANEL_COLLECTION = "panel"; + public static final String CLINICAL_ANALYSIS_COLLECTION = "clinical"; + public static final String INTERPRETATION_COLLECTION = "interpretation"; + + public static final String NOTE_ARCHIVE_COLLECTION = "note_archive"; + public static final String SAMPLE_ARCHIVE_COLLECTION = "sample_archive"; + public static final String INDIVIDUAL_ARCHIVE_COLLECTION = "individual_archive"; + public static final String FAMILY_ARCHIVE_COLLECTION = "family_archive"; + public static final String PANEL_ARCHIVE_COLLECTION = "panel_archive"; + public static final String CLINICAL_ANALYSIS_ARCHIVE_COLLECTION = "clinical_archive"; + public static final String INTERPRETATION_ARCHIVE_COLLECTION = "interpretation_archive"; + + public static final String DELETED_NOTE_COLLECTION = "note_deleted"; + public static final String DELETED_USER_COLLECTION = "user_deleted"; + public static final String DELETED_PROJECT_COLLECTION = "project_deleted"; + public static final String DELETED_STUDY_COLLECTION = "study_deleted"; + public static final String DELETED_FILE_COLLECTION = "file_deleted"; + public static final String DELETED_JOB_COLLECTION = "job_deleted"; + public static final String DELETED_SAMPLE_COLLECTION = "sample_deleted"; + public static final String DELETED_INDIVIDUAL_COLLECTION = "individual_deleted"; + public static final String DELETED_COHORT_COLLECTION = "cohort_deleted"; + public static final String DELETED_FAMILY_COLLECTION = "family_deleted"; + public static final String DELETED_PANEL_COLLECTION = "panel_deleted"; + public static final String DELETED_CLINICAL_ANALYSIS_COLLECTION = "clinical_deleted"; + public static final String DELETED_INTERPRETATION_COLLECTION = "interpretation_deleted"; + + public static final String METADATA_COLLECTION = "metadata"; + public static final String MIGRATION_COLLECTION = "migration"; + public static final String AUDIT_COLLECTION = "audit"; + + public static final List COLLECTIONS_LIST = Arrays.asList( + NOTE_COLLECTION, + ORGANIZATION_COLLECTION, + USER_COLLECTION, + PROJECT_COLLECTION, + STUDY_COLLECTION, + FILE_COLLECTION, + JOB_COLLECTION, + SAMPLE_COLLECTION, + INDIVIDUAL_COLLECTION, + COHORT_COLLECTION, + PANEL_COLLECTION, + FAMILY_COLLECTION, + CLINICAL_ANALYSIS_COLLECTION, + INTERPRETATION_COLLECTION, + + NOTE_ARCHIVE_COLLECTION, + SAMPLE_ARCHIVE_COLLECTION, + INDIVIDUAL_ARCHIVE_COLLECTION, + FAMILY_ARCHIVE_COLLECTION, + PANEL_ARCHIVE_COLLECTION, + CLINICAL_ANALYSIS_ARCHIVE_COLLECTION, + INTERPRETATION_ARCHIVE_COLLECTION, + + DELETED_NOTE_COLLECTION, + DELETED_USER_COLLECTION, + DELETED_PROJECT_COLLECTION, + DELETED_STUDY_COLLECTION, + DELETED_FILE_COLLECTION, + DELETED_JOB_COLLECTION, + DELETED_SAMPLE_COLLECTION, + DELETED_INDIVIDUAL_COLLECTION, + DELETED_COHORT_COLLECTION, + DELETED_PANEL_COLLECTION, + DELETED_FAMILY_COLLECTION, + DELETED_CLINICAL_ANALYSIS_COLLECTION, + DELETED_INTERPRETATION_COLLECTION, + + MIGRATION_COLLECTION, + // FIXME metadata collection is unused + METADATA_COLLECTION, + AUDIT_COLLECTION + ); + static final String METADATA_OBJECT_ID = "METADATA"; + + private final MongoDataStoreManager mongoManager; + private final MongoDataStore mongoDataStore; + private final String organizationId; + private final String database; + + private final NoteMongoDBAdaptor notesDBAdaptor; + private final OrganizationMongoDBAdaptor organizationDBAdaptor; + private UserMongoDBAdaptor userDBAdaptor; + private final ProjectMongoDBAdaptor projectDBAdaptor; + private final StudyMongoDBAdaptor studyDBAdaptor; + private final IndividualMongoDBAdaptor individualDBAdaptor; + private final SampleMongoDBAdaptor sampleDBAdaptor; + private final FileMongoDBAdaptor fileDBAdaptor; + private final JobMongoDBAdaptor jobDBAdaptor; + private final CohortMongoDBAdaptor cohortDBAdaptor; + private final FamilyMongoDBAdaptor familyDBAdaptor; + private final PanelMongoDBAdaptor panelDBAdaptor; + private final ClinicalAnalysisMongoDBAdaptor clinicalDBAdaptor; + private final InterpretationMongoDBAdaptor interpretationDBAdaptor; + private final AuditMongoDBAdaptor auditDBAdaptor; +// private final MetaMongoDBAdaptor metaDBAdaptor; + private final MigrationMongoDBAdaptor migrationDBAdaptor; + private final AuthorizationMongoDBAdaptor authorizationMongoDBAdaptor; + + private final MongoDBCollection organizationCollection; + private final Map mongoDBCollectionMap; + + private final Logger logger; + + public OrganizationMongoDBAdaptorFactory(String organizationId, MongoDataStoreManager mongoDataStoreManager, + MongoDBConfiguration mongoDBConfiguration, Configuration configuration, + IOManagerFactory ioManagerFactory) throws CatalogDBException { + logger = LoggerFactory.getLogger(OrganizationMongoDBAdaptorFactory.class); + this.mongoManager = mongoDataStoreManager; + this.organizationId = organizationId; + this.database = getCatalogOrganizationDatabase(configuration.getDatabasePrefix(), organizationId); + + this.mongoDataStore = mongoManager.get(database, mongoDBConfiguration); + if (mongoDataStore == null) { + throw new CatalogDBException("Unable to connect to MongoDB '" + database + "'"); + } + + MongoDBCollection migrationCollection = mongoDataStore.getCollection(MIGRATION_COLLECTION); + + organizationCollection = mongoDataStore.getCollection(ORGANIZATION_COLLECTION); + MongoDBCollection notesCollection = mongoDataStore.getCollection(NOTE_COLLECTION); + MongoDBCollection userCollection = mongoDataStore.getCollection(USER_COLLECTION); + MongoDBCollection projectCollection = mongoDataStore.getCollection(PROJECT_COLLECTION); + MongoDBCollection studyCollection = mongoDataStore.getCollection(STUDY_COLLECTION); + MongoDBCollection fileCollection = mongoDataStore.getCollection(FILE_COLLECTION); + MongoDBCollection sampleCollection = mongoDataStore.getCollection(SAMPLE_COLLECTION); + MongoDBCollection individualCollection = mongoDataStore.getCollection(INDIVIDUAL_COLLECTION); + MongoDBCollection jobCollection = mongoDataStore.getCollection(JOB_COLLECTION); + MongoDBCollection cohortCollection = mongoDataStore.getCollection(COHORT_COLLECTION); + MongoDBCollection panelCollection = mongoDataStore.getCollection(PANEL_COLLECTION); + MongoDBCollection familyCollection = mongoDataStore.getCollection(FAMILY_COLLECTION); + MongoDBCollection clinicalCollection = mongoDataStore.getCollection(CLINICAL_ANALYSIS_COLLECTION); + MongoDBCollection interpretationCollection = mongoDataStore.getCollection(INTERPRETATION_COLLECTION); + + MongoDBCollection notesArchivedCollection = mongoDataStore.getCollection(NOTE_ARCHIVE_COLLECTION); + MongoDBCollection sampleArchivedCollection = mongoDataStore.getCollection(SAMPLE_ARCHIVE_COLLECTION); + MongoDBCollection individualArchivedCollection = mongoDataStore.getCollection(INDIVIDUAL_ARCHIVE_COLLECTION); + MongoDBCollection familyArchivedCollection = mongoDataStore.getCollection(FAMILY_ARCHIVE_COLLECTION); + MongoDBCollection panelArchivedCollection = mongoDataStore.getCollection(PANEL_ARCHIVE_COLLECTION); + MongoDBCollection clinicalArchivedCollection = mongoDataStore.getCollection(CLINICAL_ANALYSIS_ARCHIVE_COLLECTION); + MongoDBCollection interpretationArchivedCollection = mongoDataStore.getCollection(INTERPRETATION_ARCHIVE_COLLECTION); + + MongoDBCollection deletedNotesCollection = mongoDataStore.getCollection(DELETED_NOTE_COLLECTION); + MongoDBCollection deletedUserCollection = mongoDataStore.getCollection(DELETED_USER_COLLECTION); + MongoDBCollection deletedProjectCollection = mongoDataStore.getCollection(DELETED_PROJECT_COLLECTION); + MongoDBCollection deletedStudyCollection = mongoDataStore.getCollection(DELETED_STUDY_COLLECTION); + MongoDBCollection deletedFileCollection = mongoDataStore.getCollection(DELETED_FILE_COLLECTION); + MongoDBCollection deletedSampleCollection = mongoDataStore.getCollection(DELETED_SAMPLE_COLLECTION); + MongoDBCollection deletedIndividualCollection = mongoDataStore.getCollection(DELETED_INDIVIDUAL_COLLECTION); + MongoDBCollection deletedJobCollection = mongoDataStore.getCollection(DELETED_JOB_COLLECTION); + MongoDBCollection deletedCohortCollection = mongoDataStore.getCollection(DELETED_COHORT_COLLECTION); + MongoDBCollection deletedPanelCollection = mongoDataStore.getCollection(DELETED_PANEL_COLLECTION); + MongoDBCollection deletedFamilyCollection = mongoDataStore.getCollection(DELETED_FAMILY_COLLECTION); + MongoDBCollection deletedClinicalCollection = mongoDataStore.getCollection(DELETED_CLINICAL_ANALYSIS_COLLECTION); + MongoDBCollection deletedInterpretationCollection = mongoDataStore.getCollection(DELETED_INTERPRETATION_COLLECTION); + + MongoDBCollection auditCollection = mongoDataStore.getCollection(AUDIT_COLLECTION); + + notesDBAdaptor = new NoteMongoDBAdaptor(notesCollection, notesArchivedCollection, deletedNotesCollection, + configuration, this); + organizationDBAdaptor = new OrganizationMongoDBAdaptor(organizationCollection, configuration, this); + fileDBAdaptor = new FileMongoDBAdaptor(fileCollection, deletedFileCollection, configuration, this, ioManagerFactory); + familyDBAdaptor = new FamilyMongoDBAdaptor(familyCollection, familyArchivedCollection, deletedFamilyCollection, configuration, + this); + individualDBAdaptor = new IndividualMongoDBAdaptor(individualCollection, individualArchivedCollection, deletedIndividualCollection, + configuration, this); + jobDBAdaptor = new JobMongoDBAdaptor(jobCollection, deletedJobCollection, configuration, this); + projectDBAdaptor = new ProjectMongoDBAdaptor(projectCollection, deletedProjectCollection, configuration, this); + + sampleDBAdaptor = new SampleMongoDBAdaptor(sampleCollection, sampleArchivedCollection, deletedSampleCollection, configuration, + this); + studyDBAdaptor = new StudyMongoDBAdaptor(studyCollection, deletedStudyCollection, configuration, this); + userDBAdaptor = new UserMongoDBAdaptor(userCollection, deletedUserCollection, configuration, this); + cohortDBAdaptor = new CohortMongoDBAdaptor(cohortCollection, deletedCohortCollection, configuration, this); + panelDBAdaptor = new PanelMongoDBAdaptor(panelCollection, panelArchivedCollection, deletedPanelCollection, configuration, this); + clinicalDBAdaptor = new ClinicalAnalysisMongoDBAdaptor(clinicalCollection, clinicalArchivedCollection, deletedClinicalCollection, + configuration, this); + interpretationDBAdaptor = new InterpretationMongoDBAdaptor(interpretationCollection, interpretationArchivedCollection, + deletedInterpretationCollection, configuration, this); +// metaDBAdaptor = new MetaMongoDBAdaptor(metaCollection, configuration, this); + migrationDBAdaptor = new MigrationMongoDBAdaptor(migrationCollection, configuration, this); + auditDBAdaptor = new AuditMongoDBAdaptor(auditCollection, configuration); + authorizationMongoDBAdaptor = new AuthorizationMongoDBAdaptor(this, configuration); + + mongoDBCollectionMap = new HashMap<>(); +// mongoDBCollectionMap.put(METADATA_COLLECTION, metaCollection); + mongoDBCollectionMap.put(MIGRATION_COLLECTION, migrationCollection); + + mongoDBCollectionMap.put(NOTE_COLLECTION, notesCollection); + mongoDBCollectionMap.put(ORGANIZATION_COLLECTION, organizationCollection); + mongoDBCollectionMap.put(PROJECT_COLLECTION, projectCollection); + mongoDBCollectionMap.put(USER_COLLECTION, userCollection); + mongoDBCollectionMap.put(STUDY_COLLECTION, studyCollection); + mongoDBCollectionMap.put(FILE_COLLECTION, fileCollection); + mongoDBCollectionMap.put(SAMPLE_COLLECTION, sampleCollection); + mongoDBCollectionMap.put(INDIVIDUAL_COLLECTION, individualCollection); + mongoDBCollectionMap.put(JOB_COLLECTION, jobCollection); + mongoDBCollectionMap.put(COHORT_COLLECTION, cohortCollection); + mongoDBCollectionMap.put(PANEL_COLLECTION, panelCollection); + mongoDBCollectionMap.put(FAMILY_COLLECTION, familyCollection); + mongoDBCollectionMap.put(CLINICAL_ANALYSIS_COLLECTION, clinicalCollection); + mongoDBCollectionMap.put(INTERPRETATION_COLLECTION, interpretationCollection); + + mongoDBCollectionMap.put(NOTE_ARCHIVE_COLLECTION, notesArchivedCollection); + mongoDBCollectionMap.put(SAMPLE_ARCHIVE_COLLECTION, sampleArchivedCollection); + mongoDBCollectionMap.put(INDIVIDUAL_ARCHIVE_COLLECTION, individualArchivedCollection); + mongoDBCollectionMap.put(FAMILY_ARCHIVE_COLLECTION, familyArchivedCollection); + mongoDBCollectionMap.put(PANEL_ARCHIVE_COLLECTION, panelArchivedCollection); + mongoDBCollectionMap.put(INTERPRETATION_ARCHIVE_COLLECTION, interpretationArchivedCollection); + + mongoDBCollectionMap.put(DELETED_NOTE_COLLECTION, deletedNotesCollection); + mongoDBCollectionMap.put(DELETED_USER_COLLECTION, deletedUserCollection); + mongoDBCollectionMap.put(DELETED_STUDY_COLLECTION, deletedStudyCollection); + mongoDBCollectionMap.put(DELETED_FILE_COLLECTION, deletedFileCollection); + mongoDBCollectionMap.put(DELETED_SAMPLE_COLLECTION, deletedSampleCollection); + mongoDBCollectionMap.put(DELETED_INDIVIDUAL_COLLECTION, deletedIndividualCollection); + mongoDBCollectionMap.put(DELETED_JOB_COLLECTION, deletedJobCollection); + mongoDBCollectionMap.put(DELETED_COHORT_COLLECTION, deletedCohortCollection); + mongoDBCollectionMap.put(DELETED_PANEL_COLLECTION, deletedPanelCollection); + mongoDBCollectionMap.put(DELETED_FAMILY_COLLECTION, deletedFamilyCollection); + mongoDBCollectionMap.put(DELETED_CLINICAL_ANALYSIS_COLLECTION, deletedClinicalCollection); + mongoDBCollectionMap.put(DELETED_INTERPRETATION_COLLECTION, deletedInterpretationCollection); + + mongoDBCollectionMap.put(AUDIT_COLLECTION, auditCollection); + } + + public boolean isCatalogDBReady() { + return organizationCollection.count(new Document()).getNumMatches() == 1; + } + +// public void createAllCollections(Configuration configuration) throws CatalogException { +// // TODO: Check META object does not exist. Use {@link isCatalogDBReady} +// // TODO: Check all collections do not exists, or are empty +// // TODO: Catch DuplicatedKeyException while inserting META object +// +// if (!mongoDataStore.getCollectionNames().isEmpty()) { +// throw new CatalogException("Database " + database + " already exists with the following collections: " +// + StringUtils.join(mongoDataStore.getCollectionNames()) + ".\nPlease, remove the database or choose a different one."); +// } +// COLLECTIONS_LIST.forEach(mongoDataStore::createCollection); +// } + + public void initialiseMetaCollection(Admin admin) throws CatalogException { + throw new CatalogException("Initialise meta collection must disappear"); +// metaDBAdaptor.initializeMetaCollection(admin); + } + + public void createAllCollections() throws CatalogDBException { + if (!mongoDataStore.getCollectionNames().isEmpty()) { + throw new CatalogDBException("Database " + mongoDataStore.getDatabaseName() + " already exists with the following " + + "collections: " + StringUtils.join(mongoDataStore.getCollectionNames()) + ".\nPlease, remove the database or" + + " choose a different one."); + } + OrganizationMongoDBAdaptorFactory.COLLECTIONS_LIST.forEach(mongoDataStore::createCollection); + } + + public void createIndexes() throws CatalogDBException { + StopWatch stopWatch = StopWatch.createStarted(); + + InputStream resourceAsStream = getClass().getResourceAsStream("/catalog-indexes.txt"); + ObjectMapper objectMapper = getDefaultObjectMapper(); + BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(resourceAsStream)); + // We store all the indexes that are in the file in the indexes object + Map>> indexes = new HashMap<>(); + bufferedReader.lines().filter(s -> !s.trim().isEmpty()).forEach(s -> { + try { + HashMap hashMap = objectMapper.readValue(s, HashMap.class); + + List collections = (List) hashMap.get("collections"); + for (String collection : collections) { + if (!indexes.containsKey(collection)) { + indexes.put(collection, new ArrayList<>()); + } + Map myIndexes = new HashMap<>(); + myIndexes.put("fields", new ObjectMap((Map) hashMap.get("fields"))); + myIndexes.put("options", new ObjectMap((Map) hashMap.getOrDefault("options", Collections.emptyMap()))); + indexes.get(collection).add(myIndexes); + } + } catch (IOException e) { + e.printStackTrace(); + } + }); + try { + bufferedReader.close(); + } catch (IOException e) { + logger.error("Error closing the buffer reader", e); + throw new UncheckedIOException(e); + } + + createIndexes(OrganizationMongoDBAdaptorFactory.PROJECT_COLLECTION, indexes); + createIndexes(OrganizationMongoDBAdaptorFactory.USER_COLLECTION, indexes); + createIndexes(OrganizationMongoDBAdaptorFactory.STUDY_COLLECTION, indexes); + createIndexes(OrganizationMongoDBAdaptorFactory.FILE_COLLECTION, indexes); + createIndexes(OrganizationMongoDBAdaptorFactory.COHORT_COLLECTION, indexes); + createIndexes(OrganizationMongoDBAdaptorFactory.JOB_COLLECTION, indexes); + createIndexes(OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, indexes); + createIndexes(OrganizationMongoDBAdaptorFactory.AUDIT_COLLECTION, indexes); + + // Versioned collections + createIndexes(OrganizationMongoDBAdaptorFactory.NOTE_COLLECTION, indexes); + createIndexes(OrganizationMongoDBAdaptorFactory.NOTE_ARCHIVE_COLLECTION, indexes); + createIndexes(OrganizationMongoDBAdaptorFactory.SAMPLE_COLLECTION, indexes); + createIndexes(OrganizationMongoDBAdaptorFactory.SAMPLE_ARCHIVE_COLLECTION, indexes); + createIndexes(OrganizationMongoDBAdaptorFactory.INDIVIDUAL_COLLECTION, indexes); + createIndexes(OrganizationMongoDBAdaptorFactory.INDIVIDUAL_ARCHIVE_COLLECTION, indexes); + createIndexes(OrganizationMongoDBAdaptorFactory.FAMILY_COLLECTION, indexes); + createIndexes(OrganizationMongoDBAdaptorFactory.FAMILY_ARCHIVE_COLLECTION, indexes); + createIndexes(OrganizationMongoDBAdaptorFactory.PANEL_COLLECTION, indexes); + createIndexes(OrganizationMongoDBAdaptorFactory.PANEL_ARCHIVE_COLLECTION, indexes); + createIndexes(OrganizationMongoDBAdaptorFactory.INTERPRETATION_COLLECTION, indexes); + createIndexes(OrganizationMongoDBAdaptorFactory.INTERPRETATION_ARCHIVE_COLLECTION, indexes); + + logger.info("Creating all indexes took {} milliseconds", stopWatch.getTime(TimeUnit.MILLISECONDS)); + } + + private void createIndexes(String collection, Map>> indexCollectionMap) { + MongoDBCollection mongoCollection = mongoDBCollectionMap.get(collection); + List> indexes = indexCollectionMap.get(collection); + + DataResult index = mongoCollection.getIndex(); + // We store the existing indexes + Set existingIndexes = index.getResults() + .stream() + .map(document -> (String) document.get("name")) + .collect(Collectors.toSet()); + + if (index.getNumResults() != indexes.size() + 1) { // It is + 1 because mongo always create the _id index by default + for (Map userIndex : indexes) { + String indexName = ""; + Document keys = new Document(); + Iterator fieldsIterator = userIndex.get("fields").entrySet().iterator(); + while (fieldsIterator.hasNext()) { + Map.Entry pair = (Map.Entry) fieldsIterator.next(); + keys.append((String) pair.getKey(), pair.getValue()); + + if (!indexName.isEmpty()) { + indexName += "_"; + } + indexName += pair.getKey() + "_" + pair.getValue(); + } + + if (!existingIndexes.contains(indexName)) { + mongoCollection.createIndex(keys, new ObjectMap(userIndex.get("options"))); + } + } + } + } + + public Map getMongoDBCollectionMap() { + return mongoDBCollectionMap; + } + + public boolean getDatabaseStatus() { + Document dbStatus = mongoDataStore.getServerStatus(); + try { + ObjectMap map = new ObjectMap(getDefaultObjectMapper().writeValueAsString(dbStatus)); + return map.getInt("ok", 0) > 0; + } catch (JsonProcessingException e) { + logger.error(e.getMessage(), e); + return false; + } + } + + public void deleteCatalogDB() { + mongoManager.drop(database); + } + + public void close() { + mongoManager.close(database); + } + + public String getOrganizationId() { + return organizationId; + } + + public MigrationMongoDBAdaptor getMigrationDBAdaptor() { + return migrationDBAdaptor; + } + + public MetaMongoDBAdaptor getCatalogMetaDBAdaptor() { + return null; + } + + public NoteMongoDBAdaptor getCatalogNotesDBAdaptor() { + return notesDBAdaptor; + } + + public OrganizationMongoDBAdaptor getCatalogOrganizationDBAdaptor() { + return organizationDBAdaptor; + } + + public UserMongoDBAdaptor getCatalogUserDBAdaptor() { + return userDBAdaptor; + } + + public ProjectMongoDBAdaptor getCatalogProjectDBAdaptor() { + return projectDBAdaptor; + } + + public StudyMongoDBAdaptor getCatalogStudyDBAdaptor() { + return studyDBAdaptor; + } + + public FileMongoDBAdaptor getCatalogFileDBAdaptor() { + return fileDBAdaptor; + } + + public SampleMongoDBAdaptor getCatalogSampleDBAdaptor() { + return sampleDBAdaptor; + } + + public IndividualMongoDBAdaptor getCatalogIndividualDBAdaptor() { + return individualDBAdaptor; + } + + public JobMongoDBAdaptor getCatalogJobDBAdaptor() { + return jobDBAdaptor; + } + + public AuditMongoDBAdaptor getCatalogAuditDbAdaptor() { + return auditDBAdaptor; + } + + public CohortMongoDBAdaptor getCatalogCohortDBAdaptor() { + return cohortDBAdaptor; + } + + public PanelMongoDBAdaptor getCatalogPanelDBAdaptor() { + return panelDBAdaptor; + } + + public FamilyMongoDBAdaptor getCatalogFamilyDBAdaptor() { + return familyDBAdaptor; + } + + public ClinicalAnalysisMongoDBAdaptor getClinicalAnalysisDBAdaptor() { + return clinicalDBAdaptor; + } + + public InterpretationMongoDBAdaptor getInterpretationDBAdaptor() { + return interpretationDBAdaptor; + } + + public AuthorizationMongoDBAdaptor getAuthorizationDBAdaptor() { + return authorizationMongoDBAdaptor; + } + + public MongoDataStore getMongoDataStore() { + return mongoDataStore; + } + + private String getCatalogOrganizationDatabase(String prefix, String organization) { + String dbPrefix = StringUtils.isEmpty(prefix) ? "opencga" : prefix; + dbPrefix = dbPrefix.endsWith("_") ? dbPrefix : dbPrefix + "_"; + return (dbPrefix + "catalog_" + organization).toLowerCase(); + } +} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/PanelMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/PanelMongoDBAdaptor.java index 9ad5b57fccf..fcc8f32fd6f 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/PanelMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/PanelMongoDBAdaptor.java @@ -33,6 +33,7 @@ import org.opencb.opencga.catalog.db.mongodb.iterators.CatalogMongoDBIterator; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.UuidUtils; @@ -61,18 +62,18 @@ public class PanelMongoDBAdaptor extends CatalogMongoDBAdaptor implements PanelD private final MongoDBCollection panelArchiveCollection; private final MongoDBCollection deletedPanelCollection; private final PanelConverter panelConverter; - private final VersionedMongoDBAdaptor versionedMongoDBAdaptor; + private final SnapshotVersionedMongoDBAdaptor versionedMongoDBAdaptor; public PanelMongoDBAdaptor(MongoDBCollection panelCollection, MongoDBCollection panelArchiveCollection, MongoDBCollection deletedPanelCollection, Configuration configuration, - MongoDBAdaptorFactory dbAdaptorFactory) { + OrganizationMongoDBAdaptorFactory dbAdaptorFactory) { super(configuration, LoggerFactory.getLogger(PanelMongoDBAdaptor.class)); this.dbAdaptorFactory = dbAdaptorFactory; this.panelCollection = panelCollection; this.panelArchiveCollection = panelArchiveCollection; this.deletedPanelCollection = deletedPanelCollection; this.panelConverter = new PanelConverter(); - this.versionedMongoDBAdaptor = new VersionedMongoDBAdaptor(panelCollection, panelArchiveCollection, deletedPanelCollection); + this.versionedMongoDBAdaptor = new SnapshotVersionedMongoDBAdaptor(panelCollection, panelArchiveCollection, deletedPanelCollection); } /** @@ -87,8 +88,7 @@ public MongoDBCollection getPanelArchiveCollection() { } @Override - public OpenCGAResult insert(long studyUid, List panelList) throws CatalogDBException, CatalogParameterException, - CatalogAuthorizationException { + public OpenCGAResult insert(long studyUid, List panelList) throws CatalogException { if (panelList == null || panelList.isEmpty()) { throw new CatalogDBException("Missing panel list"); } @@ -108,8 +108,7 @@ public OpenCGAResult insert(long studyUid, List panelList) throws Catalog } @Override - public OpenCGAResult insert(long studyUid, Panel panel, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult insert(long studyUid, Panel panel, QueryOptions options) throws CatalogException { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); logger.debug("Starting insert transaction of panel id '{}'", panel.getId()); @@ -291,7 +290,7 @@ public OpenCGAResult update(long panelUid, ObjectMap parameters, QueryOptions qu try { return runTransaction(clientSession -> privateUpdate(clientSession, dataResult.first(), parameters, queryOptions)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update panel {}: {}", dataResult.first().getId(), e.getMessage(), e); throw new CatalogDBException("Could not update panel '" + dataResult.first().getId() + "': " + e.getMessage(), e.getCause()); } @@ -317,7 +316,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, QueryOptions quer Panel panel = iterator.next(); try { result.append(runTransaction(clientSession -> privateUpdate(clientSession, panel, parameters, queryOptions))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not update panel {}: {}", panel.getId(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, panel.getId(), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -464,7 +463,7 @@ public OpenCGAResult delete(Panel panel) throws CatalogDBException, CatalogParam throw new CatalogDBException("Could not find panel " + panel.getId() + " with uid " + panel.getUid()); } return runTransaction(clientSession -> privateDelete(clientSession, result.first())); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete panel {}: {}", panel.getId(), e.getMessage(), e); throw new CatalogDBException("Could not delete panel '" + panel.getId() + "': " + e.getMessage(), e.getCause()); } @@ -480,7 +479,7 @@ public OpenCGAResult delete(Query query) throws CatalogDBException, CatalogParam String panelId = panel.getString(QueryParams.ID.key()); try { result.append(runTransaction(clientSession -> privateDelete(clientSession, panel))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete panel {}: {}", panelId, e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, panelId, e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -722,14 +721,15 @@ private Bson parseQuery(Query query, Document extraQuery, String user) if (query.containsKey(QueryParams.STUDY_UID.key()) && (StringUtils.isNotEmpty(user) || query.containsKey(ParamConstants.ACL_PARAM))) { Document studyDocument = getStudyDocument(null, query.getLong(QueryParams.STUDY_UID.key())); + boolean simplifyPermissions = simplifyPermissions(); if (query.containsKey(ParamConstants.ACL_PARAM)) { andBsonList.addAll(AuthorizationMongoDBUtils.parseAclQuery(studyDocument, query, Enums.Resource.DISEASE_PANEL, user, - configuration)); + simplifyPermissions)); } else { // Get the document query needed to check the permissions as well andBsonList.add(getQueryForAuthorisedEntries(studyDocument, user, PanelPermissions.VIEW.name(), - Enums.Resource.DISEASE_PANEL, configuration)); + Enums.Resource.DISEASE_PANEL, simplifyPermissions)); } query.remove(ParamConstants.ACL_PARAM); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptor.java index 85304c18e71..3a91bc26284 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptor.java @@ -17,10 +17,7 @@ package org.opencb.opencga.catalog.db.mongodb; import com.mongodb.client.ClientSession; -import com.mongodb.client.model.Aggregates; import com.mongodb.client.model.Filters; -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.Updates; import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.lang3.StringUtils; import org.bson.Document; @@ -31,21 +28,19 @@ import org.opencb.opencga.catalog.db.api.DBIterator; import org.opencb.opencga.catalog.db.api.ProjectDBAdaptor; import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; -import org.opencb.opencga.catalog.db.api.UserDBAdaptor; import org.opencb.opencga.catalog.db.mongodb.converters.ProjectConverter; import org.opencb.opencga.catalog.db.mongodb.iterators.ProjectCatalogMongoDBIterator; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.utils.FqnUtils; import org.opencb.opencga.catalog.utils.UuidUtils; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.models.common.InternalStatus; -import org.opencb.opencga.core.models.project.DataStore; import org.opencb.opencga.core.models.project.Project; import org.opencb.opencga.core.models.study.StudyPermissions; -import org.opencb.opencga.core.models.user.User; import org.opencb.opencga.core.response.OpenCGAResult; import org.slf4j.LoggerFactory; @@ -62,68 +57,57 @@ */ public class ProjectMongoDBAdaptor extends CatalogMongoDBAdaptor implements ProjectDBAdaptor { - private final MongoDBCollection userCollection; - private final MongoDBCollection deletedUserCollection; + private final MongoDBCollection projectCollection; + private final MongoDBCollection deletedProjectCollection; private ProjectConverter projectConverter; - public ProjectMongoDBAdaptor(MongoDBCollection userCollection, MongoDBCollection deletedUserCollection, Configuration configuration, - MongoDBAdaptorFactory dbAdaptorFactory) { + public ProjectMongoDBAdaptor(MongoDBCollection projectCollection, MongoDBCollection deletedProjectCollection, + Configuration configuration, OrganizationMongoDBAdaptorFactory dbAdaptorFactory) { super(configuration, LoggerFactory.getLogger(ProjectMongoDBAdaptor.class)); this.dbAdaptorFactory = dbAdaptorFactory; - this.userCollection = userCollection; - this.deletedUserCollection = deletedUserCollection; + this.projectCollection = projectCollection; + this.deletedProjectCollection = deletedProjectCollection; this.projectConverter = new ProjectConverter(); } @Override - public boolean exists(long projectId) { - DataResult count = userCollection.count(new Document(UserDBAdaptor.QueryParams.PROJECTS_UID.key(), projectId)); + public boolean exists(long projectUid) { + DataResult count = projectCollection.count(new Document(QueryParams.UID.key(), projectUid)); return count.getNumMatches() != 0; } @Override public OpenCGAResult nativeInsert(Map project, String userId) throws CatalogDBException { - Bson query = Filters.and(Filters.eq(UserDBAdaptor.QueryParams.ID.key(), userId), - Filters.ne(UserDBAdaptor.QueryParams.PROJECTS_ID.key(), project.get(QueryParams.ID.key()))); - Bson update = Updates.push("projects", getMongoDBDocument(project, "project")); - - //Update object - DataResult result = userCollection.update(query, update, null); - if (result.getNumInserted() == 0) { // Check if the project has been inserted - throw new CatalogDBException("Project {" + project.get(QueryParams.ID.key()) + "\"} already exists for this user"); - } - return new OpenCGAResult(result); + Document projectDocument = getMongoDBDocument(project, "project"); + return new OpenCGAResult(projectCollection.insert(projectDocument, null)); } @Override - public OpenCGAResult insert(Project project, String userId, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult insert(Project project, QueryOptions options) throws CatalogException { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); logger.debug("Starting project insert transaction for project id '{}'", project.getId()); - insert(clientSession, project, userId); + insert(clientSession, project); return endWrite(tmpStartTime, 1, 1, 0, 0, null); }, e -> logger.error("Could not create project {}: {}", project.getId(), e.getMessage())); } - Project insert(ClientSession clientSession, Project project, String userId) + Project insert(ClientSession clientSession, Project project) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { if (project.getStudies() != null && !project.getStudies().isEmpty()) { throw new CatalogParameterException("Creating project and studies in a single transaction is forbidden"); } project.setStudies(Collections.emptyList()); - Bson countQuery = Filters.and(Filters.eq(UserDBAdaptor.QueryParams.ID.key(), userId), - Filters.eq(UserDBAdaptor.QueryParams.PROJECTS_ID.key(), project.getId())); - DataResult count = userCollection.count(clientSession, countQuery); + Bson countQuery = Filters.eq(QueryParams.ID.key(), project.getId()); + DataResult count = projectCollection.count(clientSession, countQuery); if (count.getNumMatches() != 0) { - throw new CatalogDBException("Project {id:\"" + project.getId() + "\"} already exists for this user"); + throw new CatalogDBException("Project {id:\"" + project.getId() + "\"} already exists in this organization"); } - long projectUid = getNewUid(); + long projectUid = getNewUid(clientSession); project.setUid(projectUid); - project.setFqn(userId + "@" + project.getId()); if (StringUtils.isEmpty(project.getUuid())) { project.setUuid(UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.PROJECT)); } @@ -134,24 +118,16 @@ Project insert(ClientSession clientSession, Project project, String userId) projectDocument.put(PRIVATE_MODIFICATION_DATE, StringUtils.isNotEmpty(project.getModificationDate()) ? TimeUtils.toDate(project.getModificationDate()) : TimeUtils.getDate()); - Bson update = Updates.push("projects", projectDocument); - - //Update object - Bson query = Filters.eq(UserDBAdaptor.QueryParams.ID.key(), userId); - - logger.debug("Inserting project. Query: {}, update: {}", query.toBsonDocument(), update.toBsonDocument()); - userCollection.update(clientSession, query, update, null); - + projectCollection.insert(clientSession, projectDocument, null); return project; } @Override public OpenCGAResult incrementCurrentRelease(long projectId) throws CatalogDBException { - long startTime = startQuery(); Query query = new Query(QueryParams.UID.key(), projectId); - Bson update = new Document("$inc", new Document("projects.$." + QueryParams.CURRENT_RELEASE.key(), 1)); + Bson update = new Document("$inc", new Document(QueryParams.CURRENT_RELEASE.key(), 1)); - DataResult updateQR = userCollection.update(parseQuery(query), update, null); + DataResult updateQR = projectCollection.update(parseQuery(query), update, null); if (updateQR == null || updateQR.getNumMatches() == 0) { throw new CatalogDBException("Could not increment release number. Project id " + projectId + " not found"); } else if (updateQR.getNumUpdated() == 0) { @@ -160,23 +136,25 @@ public OpenCGAResult incrementCurrentRelease(long projectId) throws CatalogDBExc return new OpenCGAResult(updateQR); } - private void editId(ClientSession clientSession, String owner, long projectUid, String newId) throws CatalogDBException { - if (!exists(projectUid)) { - logger.error("Project {} not found", projectUid); - throw new CatalogDBException("Project not found."); + private void editId(ClientSession clientSession, String organizationId, long projectUid, String newId) throws CatalogDBException { +// // Check new id is not in use + Query query = new Query(QueryParams.ID.key(), newId); + if (count(clientSession, query).getNumMatches() > 0) { + throw new CatalogDBException("Project {id:\"" + newId + "\"} already exists"); } - Bson query = Filters.and( - Filters.eq(UserDBAdaptor.QueryParams.PROJECTS_UID.key(), projectUid), - Filters.ne(UserDBAdaptor.QueryParams.PROJECTS_ID.key(), newId) - ); + Bson bsonQuery = Filters.eq(QueryParams.UID.key(), projectUid); Bson update = new Document("$set", new Document() - .append("projects.$." + QueryParams.ID.key(), newId) - .append("projects.$." + QueryParams.FQN.key(), owner + "@" + newId) + .append(QueryParams.ID.key(), newId) + .append(QueryParams.FQN.key(), FqnUtils.buildFqn(organizationId, newId)) ); - DataResult result = userCollection.update(clientSession, query, update, null); + DataResult result = projectCollection.update(clientSession, bsonQuery, update, null); if (result.getNumUpdated() == 0) { //Check if the the project id was modified - throw new CatalogDBException("Project {id:\"" + newId + "\"} already exists"); + if (result.getNumMatches() == 0) { + throw new CatalogDBException("Project {uid:\"" + projectUid + "\"} not found."); + } else { + throw new CatalogDBException("Project {id:\"" + newId + "\"} already exists"); + } } // Update all the internal project ids stored in the study documents @@ -184,50 +162,14 @@ private void editId(ClientSession clientSession, String owner, long projectUid, } @Override - public long getId(final String userId, final String projectIdStr) throws CatalogDBException { - - String projectId = projectIdStr; - - if (projectId.contains("@")) { - projectId = projectId.split("@", 2)[1]; - } - - Bson filter = Filters.and( - Filters.eq(UserDBAdaptor.QueryParams.PROJECTS_ID.key(), projectId), - Filters.eq(UserDBAdaptor.QueryParams.ID.key(), userId) - ); - Bson projection = Projections.elemMatch("projects", Filters.eq(QueryParams.ID.key(), projectId)); - DataResult queryResult = userCollection.find(filter, projection, null); - User user = parseUser(queryResult); - if (user == null || user.getProjects().isEmpty()) { - return -1; - } else { - return user.getProjects().get(0).getUid(); - } - } - - @Override - public String getOwnerId(long projectId) throws CatalogDBException { -// DBObject query = new BasicDBObject(UserDBAdaptor.QueryParams.PROJECTS_UID.key(), projectId); - Bson query = Filters.eq(UserDBAdaptor.QueryParams.PROJECTS_UID.key(), projectId); - -// DBObject projection = new BasicDBObject(UserDBAdaptor.QueryParams.ID.key(), "true"); - Bson projection = Projections.include(UserDBAdaptor.QueryParams.ID.key()); - -// DataResult result = userCollection.find(query, projection, null); - DataResult result = userCollection.find(query, projection, null); - - if (result.getResults().isEmpty()) { - throw CatalogDBException.uidNotFound("Project", projectId); - } else { - return result.getResults().get(0).get(UserDBAdaptor.QueryParams.ID.key()).toString(); - } + public OpenCGAResult count(Query query) throws CatalogDBException { + Bson bson = parseQuery(query); + return new OpenCGAResult<>(projectCollection.count(bson)); } - @Override - public OpenCGAResult count(Query query) throws CatalogDBException { + OpenCGAResult count(ClientSession clientSession, Query query) throws CatalogDBException { Bson bson = parseQuery(query); - return new OpenCGAResult<>(userCollection.count(bson)); + return new OpenCGAResult<>(projectCollection.count(clientSession, bson)); } @Override @@ -238,7 +180,7 @@ public OpenCGAResult count(Query query, String user, StudyPermissions.Perm @Override public OpenCGAResult distinct(Query query, String field) throws CatalogDBException { Bson bson = parseQuery(query); - return new OpenCGAResult(userCollection.distinct(field, bson)); + return new OpenCGAResult(projectCollection.distinct(field, bson)); } @Override @@ -259,7 +201,7 @@ public OpenCGAResult update(long projectUid, ObjectMap parameters, QueryOptions try { return runTransaction(clientSession -> privateUpdate(clientSession, projectDataResult.first(), parameters)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update project {}: {}", projectDataResult.first().getId(), e.getMessage(), e); throw new CatalogDBException("Could not update project '" + projectDataResult.first().getId() + "': " + e.getMessage(), e.getCause()); @@ -278,7 +220,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, QueryOptions quer Project project = iterator.next(); try { result.append(runTransaction(clientSession -> privateUpdate(clientSession, project, parameters))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not update project {}: {}", project.getId(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, project.getId(), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -288,30 +230,30 @@ public OpenCGAResult update(Query query, ObjectMap parameters, QueryOptions quer } OpenCGAResult privateUpdate(ClientSession clientSession, Project project, ObjectMap parameters) throws CatalogDBException { - Document updateParams = getDocumentUpdateParams(parameters); + UpdateDocument updateParams = getDocumentUpdateParams(parameters); + Document finalUpdateDocument = updateParams.toFinalUpdateDocument(); - if (updateParams.isEmpty() && !parameters.containsKey(QueryParams.ID.key())) { + if (finalUpdateDocument.isEmpty() && !parameters.containsKey(QueryParams.ID.key())) { if (!parameters.isEmpty()) { logger.error("Non-processed update parameters: {}", parameters.keySet()); } throw new CatalogDBException("Nothing to be updated"); } - Document updates = new Document("$set", updateParams); - long tmpStartTime = startQuery(); if (parameters.containsKey(QueryParams.ID.key())) { logger.debug("Update project id '{}'({}) to new id '{}'", project.getId(), project.getUid(), parameters.getString(QueryParams.ID.key())); - editId(clientSession, project.getFqn().split("@")[0], project.getUid(), parameters.getString(QueryParams.ID.key())); + FqnUtils.FQN fqn = FqnUtils.parse(project.getFqn()); + editId(clientSession, fqn.getOrganization(), project.getUid(), parameters.getString(QueryParams.ID.key())); } - if (!updateParams.isEmpty()) { + if (!finalUpdateDocument.isEmpty()) { Query tmpQuery = new Query(QueryParams.UID.key(), project.getUid()); Bson finalQuery = parseQuery(tmpQuery); - logger.debug("Update project. Query: {}, update: {}", finalQuery.toBsonDocument(), updates.toBsonDocument()); - DataResult result = userCollection.update(clientSession, finalQuery, updates, null); + logger.debug("Update project. Query: {}, update: {}", finalQuery.toBsonDocument(), finalUpdateDocument.toBsonDocument()); + DataResult result = projectCollection.update(clientSession, finalQuery, finalUpdateDocument, null); if (result.getNumMatches() == 0) { throw new CatalogDBException("Project " + project.getId() + " not found"); @@ -326,77 +268,86 @@ OpenCGAResult privateUpdate(ClientSession clientSession, Project project return endWrite(tmpStartTime, 1, 1, null); } - Document getDocumentUpdateParams(ObjectMap parameters) throws CatalogDBException { - Document projectParameters = new Document(); + UpdateDocument getDocumentUpdateParams(ObjectMap parameters) throws CatalogDBException { + UpdateDocument document = new UpdateDocument(); String[] acceptedParams = {QueryParams.NAME.key(), QueryParams.DESCRIPTION.key(), QueryParams.ORGANISM_SCIENTIFIC_NAME.key(), QueryParams.ORGANISM_COMMON_NAME.key(), QueryParams.ORGANISM_ASSEMBLY.key()}; - for (String s : acceptedParams) { - if (parameters.containsKey(s)) { - projectParameters.put("projects.$." + s, parameters.getString(s)); - } - } + filterStringParams(parameters, document.getSet(), acceptedParams); if (StringUtils.isNotEmpty(parameters.getString(QueryParams.CREATION_DATE.key()))) { String time = parameters.getString(QueryParams.CREATION_DATE.key()); Date date = TimeUtils.toDate(time); - projectParameters.put("projects.$." + QueryParams.CREATION_DATE.key(), time); - projectParameters.put("projects.$." + PRIVATE_CREATION_DATE, date); + document.getSet().put(QueryParams.CREATION_DATE.key(), time); + document.getSet().put(PRIVATE_CREATION_DATE, date); } if (StringUtils.isNotEmpty(parameters.getString(MODIFICATION_DATE.key()))) { String time = parameters.getString(QueryParams.MODIFICATION_DATE.key()); Date date = TimeUtils.toDate(time); - projectParameters.put("projects.$." + QueryParams.MODIFICATION_DATE.key(), time); - projectParameters.put("projects.$." + PRIVATE_MODIFICATION_DATE, date); + document.getSet().put(QueryParams.MODIFICATION_DATE.key(), time); + document.getSet().put(PRIVATE_MODIFICATION_DATE, date); } - Map attributes = parameters.getMap(QueryParams.ATTRIBUTES.key()); - if (attributes != null) { - for (Map.Entry entry : attributes.entrySet()) { - projectParameters.put("projects.$.attributes." + entry.getKey(), entry.getValue()); - } - } - final String[] acceptedObjectParams = {QueryParams.CELLBASE.key()}; - for (String param : acceptedObjectParams) { - if (parameters.containsKey(param)) { - Object o = getMongoDBDocument(parameters.get(param), param); - projectParameters.put("projects.$." + param, o); - } - } + final String[] acceptedMapParams = {QueryParams.ATTRIBUTES.key()}; + filterMapParams(parameters, document.getSet(), acceptedMapParams); - Object datastores = parameters.get(QueryParams.INTERNAL_DATASTORES_VARIANT.key()); - if (datastores != null) { - if (datastores instanceof DataStore) { - datastores = getMongoDBDocument(datastores, "Datastore"); - } - projectParameters.put("projects.$." + QueryParams.INTERNAL_DATASTORES_VARIANT.key(), datastores); - } else { - datastores = parameters.get(QueryParams.INTERNAL_DATASTORES.key()); - if (datastores instanceof DataStore) { - datastores = getMongoDBDocument(datastores, "Datastore"); - } - if (datastores != null) { - projectParameters.put("projects.$." + QueryParams.INTERNAL_DATASTORES.key(), datastores); + final String[] acceptedObjectParams = {QueryParams.INTERNAL_STATUS.key(), QueryParams.CELLBASE.key(), + QueryParams.INTERNAL_DATASTORES_VARIANT.key(), QueryParams.INTERNAL_DATASTORES.key()}; + filterObjectParams(parameters, document.getSet(), acceptedObjectParams); + + if (!document.toFinalUpdateDocument().isEmpty()) { + String time = TimeUtils.getTime(); + if (StringUtils.isEmpty(parameters.getString(QueryParams.MODIFICATION_DATE.key()))) { + // Update modificationDate param + Date date = TimeUtils.toDate(time); + document.getSet().put(QueryParams.MODIFICATION_DATE.key(), time); + document.getSet().put(PRIVATE_MODIFICATION_DATE, date); } + document.getSet().put(INTERNAL_LAST_MODIFIED, time); } - if (parameters.containsKey(QueryParams.INTERNAL_STATUS.key())) { - projectParameters.put("projects.$." + QueryParams.INTERNAL_STATUS.key(), - getMongoDBDocument(parameters.get(QueryParams.INTERNAL_STATUS.key()), "InternalStatus")); + return document; + } + + UpdateDocument parseAndValidateUpdateParams(ObjectMap parameters) throws CatalogDBException { + UpdateDocument document = new UpdateDocument(); + + String[] acceptedParams = {QueryParams.NAME.key(), QueryParams.DESCRIPTION.key(), + QueryParams.ORGANISM_SCIENTIFIC_NAME.key(), QueryParams.ORGANISM_COMMON_NAME.key(), QueryParams.ORGANISM_ASSEMBLY.key()}; + filterStringParams(parameters, document.getSet(), acceptedParams); + + if (StringUtils.isNotEmpty(parameters.getString(QueryParams.CREATION_DATE.key()))) { + String time = parameters.getString(QueryParams.CREATION_DATE.key()); + Date date = TimeUtils.toDate(time); + document.getSet().put(QueryParams.CREATION_DATE.key(), time); + document.getSet().put(PRIVATE_CREATION_DATE, date); } + if (StringUtils.isNotEmpty(parameters.getString(MODIFICATION_DATE.key()))) { + String time = parameters.getString(QueryParams.MODIFICATION_DATE.key()); + Date date = TimeUtils.toDate(time); + document.getSet().put(QueryParams.MODIFICATION_DATE.key(), time); + document.getSet().put(PRIVATE_MODIFICATION_DATE, date); + } + + final String[] acceptedMapParams = {QueryParams.ATTRIBUTES.key()}; + filterMapParams(parameters, document.getSet(), acceptedMapParams); - if (!projectParameters.isEmpty()) { + final String[] acceptedObjectParams = {QueryParams.CELLBASE.key(), QueryParams.INTERNAL_DATASTORES_VARIANT.key(), + QueryParams.INTERNAL_DATASTORES.key(), QueryParams.INTERNAL_STATUS.key()}; + filterObjectParams(parameters, document.getSet(), acceptedObjectParams); + + if (!document.toFinalUpdateDocument().isEmpty()) { String time = TimeUtils.getTime(); - if (StringUtils.isEmpty(parameters.getString(MODIFICATION_DATE.key()))) { + if (StringUtils.isEmpty(parameters.getString(QueryParams.MODIFICATION_DATE.key()))) { // Update modificationDate param Date date = TimeUtils.toDate(time); - projectParameters.put("projects.$." + QueryParams.MODIFICATION_DATE.key(), time); - projectParameters.put("projects.$." + PRIVATE_MODIFICATION_DATE, date); + document.getSet().put(QueryParams.MODIFICATION_DATE.key(), time); + document.getSet().put(PRIVATE_MODIFICATION_DATE, date); } - projectParameters.put("projects.$." + INTERNAL_LAST_MODIFIED, time); + document.getSet().put(INTERNAL_LAST_MODIFIED, time); } - return projectParameters; + return document; } @Deprecated @@ -432,7 +383,7 @@ public OpenCGAResult delete(long id, QueryOptions queryOptions) throws CatalogDB public OpenCGAResult delete(Project project) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { try { return runTransaction(clientSession -> privateDelete(clientSession, project)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete project {}: {}", project.getId(), e.getMessage(), e); throw new CatalogDBException("Could not delete project '" + project.getId() + "': " + e.getMessage(), e.getCause()); } @@ -461,7 +412,7 @@ public OpenCGAResult delete(Query query) throws CatalogDBException { try { result.append(runTransaction(clientSession -> privateDelete(clientSession, project))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete project {}: {}", project.getId(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, project.getId(), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -479,7 +430,7 @@ OpenCGAResult privateDelete(ClientSession clientSession, Project projec // First, we delete the studies Query studyQuery = new Query(StudyDBAdaptor.QueryParams.PROJECT_UID.key(), project.getUid()); - List studyList = studyDBAdaptor.nativeGet(studyQuery, QueryOptions.empty()).getResults(); + List studyList = studyDBAdaptor.nativeGet(clientSession, studyQuery, QueryOptions.empty()).getResults(); if (studyList != null) { for (Document study : studyList) { studyDBAdaptor.privateDelete(clientSession, study); @@ -494,13 +445,7 @@ OpenCGAResult privateDelete(ClientSession clientSession, Project projec .append(QueryParams.INTERNAL_STATUS_DATE.key(), TimeUtils.getTime()) .append(QueryParams.ID.key(), project.getId() + deleteSuffix); - Bson bsonQuery = parseQuery(studyQuery); - Document updateDocument = getDocumentUpdateParams(updateParams); - - logger.debug("Delete project {}: Query: {}, update: {}", project.getId(), bsonQuery.toBsonDocument(), - updateDocument.toBsonDocument()); - DataResult result = userCollection.update(clientSession, bsonQuery, updateDocument, QueryOptions.empty()); - + DataResult result = privateUpdate(clientSession, project, updateParams); if (result.getNumMatches() == 0) { throw new CatalogDBException("Project " + project.getId() + " not found"); } @@ -554,17 +499,10 @@ public OpenCGAResult restore(long id, QueryOptions queryOptions) return setStatus(id, InternalStatus.READY); } - @Override - public OpenCGAResult get(String userId, QueryOptions options) throws CatalogDBException { - long startTime = startQuery(); - Query query = new Query(QueryParams.USER_ID.key(), userId); - return endQuery(startTime, get(query, options)); - } - @Override public OpenCGAResult get(long projectId, QueryOptions options) throws CatalogDBException { checkId(projectId); - Query query = new Query(QueryParams.UID.key(), projectId).append(INTERNAL_STATUS_ID.key(), "!=" + InternalStatus.DELETED); + Query query = new Query(QueryParams.UID.key(), projectId); return get(query, options); // // Fixme: Check the code below // List projects = user.getProjects(); @@ -605,8 +543,12 @@ public OpenCGAResult get(Query query, QueryOptions options, String user @Override public OpenCGAResult nativeGet(Query query, QueryOptions options) throws CatalogDBException { + return nativeGet(null, query, options); + } + + public OpenCGAResult nativeGet(ClientSession clientSession, Query query, QueryOptions options) throws CatalogDBException { long startTime = startQuery(); - try (DBIterator dbIterator = nativeIterator(query, options)) { + try (DBIterator dbIterator = nativeIterator(clientSession, query, options)) { return endQuery(startTime, dbIterator); } } @@ -614,8 +556,13 @@ public OpenCGAResult nativeGet(Query query, QueryOptions options) thro @Override public OpenCGAResult nativeGet(Query query, QueryOptions options, String user) throws CatalogDBException, CatalogAuthorizationException { + return nativeGet(null, query, options, user); + } + + public OpenCGAResult nativeGet(ClientSession clientSession, Query query, QueryOptions options, String user) + throws CatalogDBException, CatalogAuthorizationException { long startTime = startQuery(); - try (DBIterator dbIterator = nativeIterator(query, options, user)) { + try (DBIterator dbIterator = nativeIterator(clientSession, query, options, user)) { return endQuery(startTime, dbIterator); } } @@ -632,11 +579,15 @@ public DBIterator iterator(ClientSession clientSession, Query query, Qu @Override public DBIterator nativeIterator(Query query, QueryOptions options) throws CatalogDBException { + return nativeIterator(null, query, options); + } + + public DBIterator nativeIterator(ClientSession clientSession, Query query, QueryOptions options) throws CatalogDBException { QueryOptions queryOptions = options != null ? new QueryOptions(options) : new QueryOptions(); queryOptions.put(NATIVE_QUERY, true); - MongoDBIterator mongoCursor = getMongoCursor(null, query, queryOptions); - return new ProjectCatalogMongoDBIterator<>(mongoCursor, null, null, dbAdaptorFactory, options, null); + MongoDBIterator mongoCursor = getMongoCursor(clientSession, query, queryOptions); + return new ProjectCatalogMongoDBIterator<>(mongoCursor, clientSession, null, dbAdaptorFactory, options, null); } @Override @@ -649,12 +600,118 @@ public DBIterator iterator(Query query, QueryOptions options, String us @Override public DBIterator nativeIterator(Query query, QueryOptions options, String user) throws CatalogDBException, CatalogAuthorizationException { + return nativeIterator(null, query, options, user); + } + + public DBIterator nativeIterator(ClientSession clientSession, Query query, QueryOptions options, String user) + throws CatalogDBException, CatalogAuthorizationException { QueryOptions queryOptions = options != null ? new QueryOptions(options) : new QueryOptions(); queryOptions.put(NATIVE_QUERY, true); - MongoDBIterator mongoCursor = getMongoCursor(null, query, queryOptions, user); - return new ProjectCatalogMongoDBIterator<>(mongoCursor, null, null, dbAdaptorFactory, options, user); - } + MongoDBIterator mongoCursor = getMongoCursor(clientSession, query, queryOptions, user); + return new ProjectCatalogMongoDBIterator<>(mongoCursor, clientSession, null, dbAdaptorFactory, options, user); + } + +// private MongoDBIterator getMongoCursor(ClientSession clientSession, Query query, QueryOptions options, String user) +// throws CatalogDBException, CatalogAuthorizationException { +// List studyUids = query.getAsLongList(QueryParams.STUDY_UID.key()); +// if (!studyUids.isEmpty()) { +// query.remove(QueryParams.STUDY_UID.key()); +// Query studyQuery = new Query(PRIVATE_UID, studyUids); +// QueryOptions studyOptions = new QueryOptions(QueryOptions.INCLUDE, PRIVATE_PROJECT); +// OpenCGAResult result = dbAdaptorFactory.getCatalogStudyDBAdaptor() +// .nativeGet(clientSession, studyQuery, studyOptions, user); +// if (result.getNumResults() == 0) { +// return new MongoDBIterator<>(MongoDBIterator.EMPTY_MONGO_CURSOR_ITERATOR, 0); +// } +// // Add all project uids to the main query parameter +// query.put(PRIVATE_UID, +// result.getResults() +// .stream() +// .map(x -> x.get(PRIVATE_PROJECT, Document.class)) +// .map(x -> x.get(PRIVATE_UID, Long.class)) +// .distinct() +// .collect(Collectors.toList())); +// } +// +// String requestedUser = query.getString(QueryParams.USER_ID.key()); +// if (StringUtils.isNotEmpty(requestedUser)) { +// if (requestedUser.equals(user)) { +// // My own projects (no permissions check) +// return getMongoCursor(clientSession, query, options); +// } else { +// // Other user projects (permissions check) +// List projectUids = query.getAsLongList(QueryParams.UID.key()); +// Query studyQuery = new Query(); +// if (projectUids != null && projectUids.size() > 0) { +// studyQuery.append(StudyDBAdaptor.QueryParams.PROJECT_UID.key(), projectUids); +// } +// List projectIds = query.getAsStringList(QueryParams.ID.key()); +// if (projectIds != null && projectIds.size() > 0) { +// studyQuery.append(StudyDBAdaptor.QueryParams.PROJECT_ID.key(), projectIds); +// } +// +// studyQuery.putIfNotEmpty(StudyDBAdaptor.QueryParams.UID.key(), query.getString(QueryParams.STUDY_UID.key())); +// studyQuery.putIfNotEmpty(StudyDBAdaptor.QueryParams.ID.key(), query.getString(QueryParams.STUDY_ID.key())); +// studyQuery.putIfNotEmpty(StudyDBAdaptor.QueryParams.OWNER.key(), query.getString(QueryParams.USER_ID.key())); +// OpenCGAResult studiesResult = dbAdaptorFactory.getCatalogStudyDBAdaptor().nativeGet(clientSession, studyQuery, +// new QueryOptions(QueryOptions.INCLUDE, StudyDBAdaptor.QueryParams.FQN.key()), user); +// +// if (studiesResult.getNumResults() > 0) { +// Set projectFqn = new HashSet<>(); +// for (Document study : studiesResult.getResults()) { +// String studyFqn = study.getString(StudyDBAdaptor.QueryParams.FQN.key()); +// projectFqn.add(FqnUtils.toProjectFqn(studyFqn)); +// } +// query.put(QueryParams.FQN.key(), new ArrayList<>(projectFqn)); +// return getMongoCursor(clientSession, query, options); +// } else { +// return new MongoDBIterator<>(MongoDBIterator.EMPTY_MONGO_CURSOR_ITERATOR, 0); +// } +// +// } +// } else { +// // 1. Get all projects matching the query and extract own projects and external projects +// MongoDBIterator mongoCursor = getMongoCursor(clientSession, query, +// new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(QueryParams.FQN.key(), QueryParams.UID.key()))); +// ProjectCatalogMongoDBIterator iterator = new ProjectCatalogMongoDBIterator<>(mongoCursor, clientSession, +// projectConverter, dbAdaptorFactory, options, user); +// List ownerFqns = new ArrayList<>(); // FQNs from projects owned by the user "user" +// List externalUids = new ArrayList<>(); // Project uids from projects owned by a user other than "user" +// while (iterator.hasNext()) { +// Project project = iterator.next(); +// if (FqnUtils.getUser(project.getFqn()).equals(user)) { +// ownerFqns.add(project.getFqn()); +// } else { +// externalUids.add(project.getUid()); +// } +// } +// +// // 2. Extract external projects and check permissions by querying their studies +// List externalFqns = new ArrayList<>(); +// if (!externalUids.isEmpty()) { +// Query studyQuery = new Query(StudyDBAdaptor.QueryParams.PROJECT_UID.key(), externalUids); +// OpenCGAResult studiesResult = dbAdaptorFactory.getCatalogStudyDBAdaptor().nativeGet(clientSession, studyQuery, +// new QueryOptions(QueryOptions.INCLUDE, StudyDBAdaptor.QueryParams.FQN.key()), user); +// if (studiesResult.getNumResults() > 0) { +// Set projectFqn = new HashSet<>(); +// for (Document study : studiesResult.getResults()) { +// String studyFqn = study.getString(StudyDBAdaptor.QueryParams.FQN.key()); +// projectFqn.add(FqnUtils.toProjectFqn(studyFqn)); +// } +// externalFqns.addAll(projectFqn); +// } +// } +// +// // 3. Query based on the final projects the user can see +// List allFqns = new ArrayList<>(ownerFqns.size() + externalFqns.size()); +// allFqns.addAll(ownerFqns); +// allFqns.addAll(externalFqns); +// query.put(QueryParams.FQN.key(), allFqns); +// +// return getMongoCursor(clientSession, query, options); +// } +// } private MongoDBIterator getMongoCursor(ClientSession clientSession, Query query, QueryOptions options, String user) throws CatalogDBException, CatalogAuthorizationException { @@ -678,63 +735,28 @@ private MongoDBIterator getMongoCursor(ClientSession clientSession, Qu .collect(Collectors.toList())); } - String requestedUser = query.getString(QueryParams.USER_ID.key()); - if (StringUtils.isNotEmpty(requestedUser)) { - if (requestedUser.equals(user)) { - // My own projects (no permissions check) - return getMongoCursor(clientSession, query, options); - } else { - // Other user projects (permissions check) - List projectUids = query.getAsLongList(QueryParams.UID.key()); - Query studyQuery = new Query(); - if (projectUids != null && projectUids.size() > 0) { - studyQuery.append(StudyDBAdaptor.QueryParams.PROJECT_UID.key(), projectUids); - } - List projectIds = query.getAsStringList(QueryParams.ID.key()); - if (projectIds != null && projectIds.size() > 0) { - studyQuery.append(StudyDBAdaptor.QueryParams.PROJECT_ID.key(), projectIds); - } - - studyQuery.putIfNotEmpty(StudyDBAdaptor.QueryParams.UID.key(), query.getString(QueryParams.STUDY_UID.key())); - studyQuery.putIfNotEmpty(StudyDBAdaptor.QueryParams.ID.key(), query.getString(QueryParams.STUDY_ID.key())); - studyQuery.putIfNotEmpty(StudyDBAdaptor.QueryParams.OWNER.key(), query.getString(QueryParams.USER_ID.key())); - OpenCGAResult studiesResult = dbAdaptorFactory.getCatalogStudyDBAdaptor().nativeGet(clientSession, studyQuery, - new QueryOptions(QueryOptions.INCLUDE, StudyDBAdaptor.QueryParams.FQN.key()), user); + options = filterQueryOptionsToIncludeKeys(options, Arrays.asList(QueryParams.ID.key(), QueryParams.FQN.key())); - if (studiesResult.getNumResults() > 0) { - Set projectFqn = new HashSet<>(); - for (Document study : studiesResult.getResults()) { - String studyFqn = study.getString(StudyDBAdaptor.QueryParams.FQN.key()); - projectFqn.add(FqnUtils.toProjectFqn(studyFqn)); - } - query.put(QueryParams.FQN.key(), new ArrayList<>(projectFqn)); - return getMongoCursor(clientSession, query, options); - } else { - return new MongoDBIterator<>(MongoDBIterator.EMPTY_MONGO_CURSOR_ITERATOR, 0); - } + // 0. Check if the user is the owner or one of the organization admins + boolean isOwnerOrAdmin = dbAdaptorFactory.getCatalogOrganizationDBAdaptor().isOwnerOrAdmin(clientSession, user); - } + if (isOwnerOrAdmin) { + return getMongoCursor(clientSession, query, options); } else { - // 1. Get all projects matching the query and extract own projects and external projects - MongoDBIterator mongoCursor = getMongoCursor(clientSession, query, - new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(QueryParams.FQN.key(), QueryParams.UID.key()))); - ProjectCatalogMongoDBIterator iterator = new ProjectCatalogMongoDBIterator<>(mongoCursor, clientSession, - projectConverter, dbAdaptorFactory, options, user); - List ownerFqns = new ArrayList<>(); // FQNs from projects owned by the user "user" - List externalUids = new ArrayList<>(); // Project uids from projects owned by a user other than "user" - while (iterator.hasNext()) { - Project project = iterator.next(); - if (FqnUtils.getUser(project.getFqn()).equals(user)) { - ownerFqns.add(project.getFqn()); - } else { - externalUids.add(project.getUid()); + // 1. Get all projects matching the query + List projectUids = new ArrayList<>(); + try (DBIterator iterator = iterator(clientSession, query, + new QueryOptions(QueryOptions.INCLUDE, Collections.singletonList(QueryParams.UID.key())))) { + while (iterator.hasNext()) { + Project project = iterator.next(); + projectUids.add(project.getUid()); } } - // 2. Extract external projects and check permissions by querying their studies - List externalFqns = new ArrayList<>(); - if (!externalUids.isEmpty()) { - Query studyQuery = new Query(StudyDBAdaptor.QueryParams.PROJECT_UID.key(), externalUids); + // 2. Extract project fqns that are allowed to see by the user + List allowedFqns = new ArrayList<>(); + if (!projectUids.isEmpty()) { + Query studyQuery = new Query(StudyDBAdaptor.QueryParams.PROJECT_UID.key(), projectUids); OpenCGAResult studiesResult = dbAdaptorFactory.getCatalogStudyDBAdaptor().nativeGet(clientSession, studyQuery, new QueryOptions(QueryOptions.INCLUDE, StudyDBAdaptor.QueryParams.FQN.key()), user); if (studiesResult.getNumResults() > 0) { @@ -743,64 +765,30 @@ private MongoDBIterator getMongoCursor(ClientSession clientSession, Qu String studyFqn = study.getString(StudyDBAdaptor.QueryParams.FQN.key()); projectFqn.add(FqnUtils.toProjectFqn(studyFqn)); } - externalFqns.addAll(projectFqn); + allowedFqns.addAll(projectFqn); } } // 3. Query based on the final projects the user can see - List allFqns = new ArrayList<>(ownerFqns.size() + externalFqns.size()); - allFqns.addAll(ownerFqns); - allFqns.addAll(externalFqns); - query.put(QueryParams.FQN.key(), allFqns); - - return getMongoCursor(clientSession, query, options); + if (!allowedFqns.isEmpty()) { + query.put(QueryParams.FQN.key(), allowedFqns); + return getMongoCursor(clientSession, query, options); + } else { + return new MongoDBIterator<>(MongoDBIterator.EMPTY_MONGO_CURSOR_ITERATOR, 0); + } } } private MongoDBIterator getMongoCursor(ClientSession clientSession, Query query, QueryOptions options) throws CatalogDBException { - - if (!query.containsKey(INTERNAL_STATUS_ID.key())) { - query.append(INTERNAL_STATUS_ID.key(), "!=" + InternalStatus.DELETED); - } - List aggregates = new ArrayList<>(); - Bson bsonQuery = parseQuery(query); - aggregates.add(Aggregates.match(bsonQuery)); - aggregates.add(Aggregates.unwind("$projects")); - aggregates.add(Aggregates.match(bsonQuery)); // Check include - QueryOptions qOptions = filterQueryOptions(options, Arrays.asList(QueryParams.UID.key(), QueryParams.FQN.key())); - qOptions = addPrefixInOptions(qOptions, "projects."); - - /*List includeList = new ArrayList<>(); - if (options != null && options.get(QueryOptions.INCLUDE) != null) { - List optionsAsStringList = options.getAsStringList(QueryOptions.INCLUDE); - includeList.addAll(optionsAsStringList.stream().collect(Collectors.toList())); - if (!includeList.contains(QueryParams.UID.key())) { - includeList.add(QueryParams.UID.key()); - } - if (!includeList.contains(QueryParams.FQN.key())) { - includeList.add(QueryParams.FQN.key()); - } + QueryOptions qOptions = filterQueryOptionsToIncludeKeys(options, Arrays.asList(QueryParams.UID.key(), QueryParams.FQN.key())); - // Check if they start with projects. - for (int i = 0; i < includeList.size(); i++) { - if (!includeList.get(i).startsWith("projects.")) { - String param = "projects." + includeList.get(i); - includeList.set(i, param); - } - } - qOptions.put(QueryOptions.INCLUDE, includeList); - } - */ - - for (Bson aggregate : aggregates) { - logger.debug("Get project: Aggregate : {}", aggregate.toBsonDocument()); - } - - return userCollection.iterator(clientSession, aggregates, qOptions); + MongoDBCollection collection = getQueryCollection(query, projectCollection, null, deletedProjectCollection); + logger.debug("Project query: {}", bsonQuery.toBsonDocument()); + return collection.iterator(clientSession, bsonQuery, null, null, qOptions); } @Override @@ -844,8 +832,6 @@ private Bson parseQuery(Query query) throws CatalogDBException { List andBsonList = new ArrayList<>(); fixComplexQueryParam(QueryParams.ATTRIBUTES.key(), query); - fixComplexQueryParam(QueryParams.BATTRIBUTES.key(), query); - fixComplexQueryParam(QueryParams.NATTRIBUTES.key(), query); for (Map.Entry entry : query.entrySet()) { String key = entry.getKey().split("\\.")[0]; @@ -858,21 +844,7 @@ private Bson parseQuery(Query query) throws CatalogDBException { try { switch (queryParam) { case UID: - addAutoOrQuery("projects." + queryParam.key(), queryParam.key(), query, queryParam.type(), andBsonList); - break; - case USER_ID: - addAutoOrQuery(ID, queryParam.key(), query, queryParam.type(), andBsonList); - break; - case ATTRIBUTES: - addAutoOrQuery("projects." + entry.getKey(), entry.getKey(), query, queryParam.type(), andBsonList); - break; - case BATTRIBUTES: - String mongoKey = "projects." + entry.getKey().replace(QueryParams.BATTRIBUTES.key(), QueryParams.ATTRIBUTES.key()); - addAutoOrQuery(mongoKey, entry.getKey(), query, queryParam.type(), andBsonList); - break; - case NATTRIBUTES: - mongoKey = "projects." + entry.getKey().replace(QueryParams.NATTRIBUTES.key(), QueryParams.ATTRIBUTES.key()); - addAutoOrQuery(mongoKey, entry.getKey(), query, queryParam.type(), andBsonList); + addAutoOrQuery(queryParam.key(), queryParam.key(), query, queryParam.type(), andBsonList); break; case CREATION_DATE: addAutoOrQuery(PRIVATE_CREATION_DATE, queryParam.key(), query, queryParam.type(), andBsonList); @@ -885,8 +857,7 @@ private Bson parseQuery(Query query) throws CatalogDBException { // Convert the status to a positive status query.put(queryParam.key(), InternalStatus.getPositiveStatus(InternalStatus.STATUS_LIST, query.getString(queryParam.key()))); - addAutoOrQuery("projects." + INTERNAL_STATUS_ID.key(), queryParam.key(), query, INTERNAL_STATUS_ID.type(), - andBsonList); + addAutoOrQuery(INTERNAL_STATUS_ID.key(), queryParam.key(), query, INTERNAL_STATUS_ID.type(), andBsonList); break; case NAME: case UUID: @@ -902,7 +873,7 @@ private Bson parseQuery(Query query) throws CatalogDBException { case INTERNAL_STATUS_DATE: case INTERNAL_DATASTORES: case ACL_USER_ID: - addAutoOrQuery("projects." + queryParam.key(), queryParam.key(), query, queryParam.type(), andBsonList); + addAutoOrQuery(queryParam.key(), queryParam.key(), query, queryParam.type(), andBsonList); break; default: throw new CatalogDBException("Cannot query by parameter " + queryParam.key()); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java index da6dca117ab..d9058193e7d 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java @@ -39,6 +39,7 @@ import org.opencb.opencga.catalog.db.mongodb.iterators.SampleCatalogMongoDBIterator; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.managers.IndividualManager; import org.opencb.opencga.catalog.managers.SampleManager; @@ -81,11 +82,11 @@ public class SampleMongoDBAdaptor extends AnnotationMongoDBAdaptor imple private final MongoDBCollection deletedSampleCollection; private final SampleConverter sampleConverter; private final IndividualMongoDBAdaptor individualDBAdaptor; - private final VersionedMongoDBAdaptor versionedMongoDBAdaptor; + private final SnapshotVersionedMongoDBAdaptor versionedMongoDBAdaptor; public SampleMongoDBAdaptor(MongoDBCollection sampleCollection, MongoDBCollection archiveSampleCollection, MongoDBCollection deletedSampleCollection, Configuration configuration, - MongoDBAdaptorFactory dbAdaptorFactory) { + OrganizationMongoDBAdaptorFactory dbAdaptorFactory) { super(configuration, LoggerFactory.getLogger(SampleMongoDBAdaptor.class)); this.dbAdaptorFactory = dbAdaptorFactory; this.sampleCollection = sampleCollection; @@ -93,7 +94,8 @@ public SampleMongoDBAdaptor(MongoDBCollection sampleCollection, MongoDBCollectio this.deletedSampleCollection = deletedSampleCollection; this.sampleConverter = new SampleConverter(); individualDBAdaptor = dbAdaptorFactory.getCatalogIndividualDBAdaptor(); - this.versionedMongoDBAdaptor = new VersionedMongoDBAdaptor(sampleCollection, archiveSampleCollection, deletedSampleCollection); + this.versionedMongoDBAdaptor = new SnapshotVersionedMongoDBAdaptor(sampleCollection, archiveSampleCollection, + deletedSampleCollection); } @Override @@ -160,7 +162,7 @@ Sample insert(ClientSession clientSession, long studyUid, Sample sample, List variableSetList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult insert(long studyId, Sample sample, List variableSetList, QueryOptions options) + throws CatalogException { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); logger.debug("Starting sample insert transaction for sample id '{}'", sample.getId()); @@ -261,7 +263,7 @@ public OpenCGAResult update(long uid, ObjectMap parameters, List va try { return runTransaction(clientSession -> privateUpdate(clientSession, documentResult.first(), parameters, variableSetList, queryOptions)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update sample {}: {}", sampleId, e.getMessage(), e); throw new CatalogDBException("Could not update sample " + sampleId + ": " + e.getMessage(), e.getCause()); } @@ -296,7 +298,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, List try { result.append(runTransaction(clientSession -> privateUpdate(clientSession, sampleDocument, parameters, variableSetList, queryOptions))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not update sample {}: {}", sampleId, e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, sampleId, e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -856,8 +858,7 @@ public OpenCGAResult unmarkPermissionRule(long studyId, String permissionRuleId) } @Override - public OpenCGAResult setRgaIndexes(long studyUid, List sampleUids, RgaIndex rgaIndex) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult setRgaIndexes(long studyUid, List sampleUids, RgaIndex rgaIndex) throws CatalogException { ObjectMap params; try { params = new ObjectMap(getDefaultObjectMapper().writeValueAsString(rgaIndex)); @@ -942,7 +943,7 @@ public OpenCGAResult delete(Sample sample) throws CatalogDBException, CatalogPar throw new CatalogDBException("Could not find sample " + sample.getId() + " with uid " + sample.getUid()); } return runTransaction(clientSession -> privateDelete(clientSession, result.first())); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete sample {}: {}", sample.getId(), e.getMessage(), e); throw new CatalogDBException("Could not delete sample " + sample.getId() + ": " + e.getMessage(), e); } @@ -959,7 +960,7 @@ public OpenCGAResult delete(Query query) throws CatalogDBException { try { result.append(runTransaction(clientSession -> privateDelete(clientSession, sample))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete sample {}: {}", sampleId, e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, sampleId, e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -1122,7 +1123,7 @@ public DBIterator iterator(long studyUid, Query query, QueryOptions opti query.put(PRIVATE_STUDY_UID, studyUid); MongoDBIterator mongoCursor = getMongoCursor(null, query, options, user); Document studyDocument = getStudyDocument(null, studyUid); - UnaryOperator iteratorFilter = (d) -> filterAnnotationSets(studyDocument, d, user, + UnaryOperator iteratorFilter = (d) -> filterAnnotationSets(dbAdaptorFactory.getOrganizationId(), studyDocument, d, user, StudyPermissions.Permissions.VIEW_SAMPLE_ANNOTATIONS.name(), SamplePermissions.VIEW_ANNOTATIONS.name()); return new SampleCatalogMongoDBIterator<>(mongoCursor, null, sampleConverter, iteratorFilter, individualDBAdaptor, studyUid, user, @@ -1143,7 +1144,7 @@ DBIterator nativeIterator(ClientSession clientSession, long studyUid, Query quer query.put(PRIVATE_STUDY_UID, studyUid); MongoDBIterator mongoCursor = getMongoCursor(clientSession, query, queryOptions, user); Document studyDocument = getStudyDocument(clientSession, studyUid); - UnaryOperator iteratorFilter = (d) -> filterAnnotationSets(studyDocument, d, user, + UnaryOperator iteratorFilter = (d) -> filterAnnotationSets(dbAdaptorFactory.getOrganizationId(), studyDocument, d, user, StudyPermissions.Permissions.VIEW_SAMPLE_ANNOTATIONS.name(), SamplePermissions.VIEW_ANNOTATIONS.name()); return new SampleCatalogMongoDBIterator<>(mongoCursor, clientSession, null, iteratorFilter, individualDBAdaptor, studyUid, user, @@ -1198,7 +1199,7 @@ private MongoDBIterator getMongoCursor(ClientSession clientSession, Qu qOptions = new QueryOptions(); } qOptions = removeAnnotationProjectionOptions(qOptions); - qOptions = filterQueryOptions(qOptions, SampleManager.INCLUDE_SAMPLE_IDS.getAsStringList(QueryOptions.INCLUDE)); + qOptions = filterQueryOptionsToIncludeKeys(qOptions, SampleManager.INCLUDE_SAMPLE_IDS.getAsStringList(QueryOptions.INCLUDE)); qOptions = filterOptions(qOptions, FILTER_ROUTE_SAMPLES); fixAclProjection(qOptions); @@ -1326,17 +1327,18 @@ private Bson parseQuery(Query query, Document extraQuery, String user) if (query.containsKey(QueryParams.STUDY_UID.key()) && (StringUtils.isNotEmpty(user) || query.containsKey(ParamConstants.ACL_PARAM))) { Document studyDocument = getStudyDocument(null, query.getLong(QueryParams.STUDY_UID.key())); + boolean simplifyPermissions = simplifyPermissions(); if (query.containsKey(ParamConstants.ACL_PARAM)) { andBsonList.addAll(AuthorizationMongoDBUtils.parseAclQuery(studyDocument, query, Enums.Resource.SAMPLE, user, - configuration)); + simplifyPermissions)); } else { if (containsAnnotationQuery(query)) { andBsonList.add(getQueryForAuthorisedEntries(studyDocument, user, - SamplePermissions.VIEW_ANNOTATIONS.name(), Enums.Resource.SAMPLE, configuration)); + SamplePermissions.VIEW_ANNOTATIONS.name(), Enums.Resource.SAMPLE, simplifyPermissions)); } else { andBsonList.add(getQueryForAuthorisedEntries(studyDocument, user, SamplePermissions.VIEW.name(), - Enums.Resource.SAMPLE, configuration)); + Enums.Resource.SAMPLE, simplifyPermissions)); } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SnapshotVersionedMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SnapshotVersionedMongoDBAdaptor.java new file mode 100644 index 00000000000..c54c80df169 --- /dev/null +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SnapshotVersionedMongoDBAdaptor.java @@ -0,0 +1,424 @@ +package org.opencb.opencga.catalog.db.mongodb; + +import com.mongodb.client.ClientSession; +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Updates; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.opencb.commons.datastore.core.DataResult; +import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.commons.datastore.mongodb.GenericDocumentComplexConverter; +import org.opencb.commons.datastore.mongodb.MongoDBCollection; +import org.opencb.commons.datastore.mongodb.MongoDBIterator; +import org.opencb.opencga.catalog.db.api.DBIterator; +import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; +import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogParameterException; +import org.opencb.opencga.core.models.common.InternalStatus; +import org.opencb.opencga.core.response.OpenCGAResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; + +import static org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptor.*; +import static org.opencb.opencga.catalog.db.mongodb.MongoDBUtils.getMongoDBDocument; + +public class SnapshotVersionedMongoDBAdaptor { + + private final Logger logger; + private final MongoDBCollection collection; + private final MongoDBCollection archiveCollection; + private final MongoDBCollection deletedCollection; + + private static final String PRIVATE_TRANSACTION_ID = "_transactionId"; + + public SnapshotVersionedMongoDBAdaptor(MongoDBCollection collection, MongoDBCollection archiveCollection, + MongoDBCollection deletedCollection) { + this.collection = collection; + this.archiveCollection = archiveCollection; + this.deletedCollection = deletedCollection; + logger = LoggerFactory.getLogger(SnapshotVersionedMongoDBAdaptor.class); + } + + + /** + * Generate complex query where [{id - version}, {id2 - version2}] pairs will be queried. + * + * @param query Query object. + * @param bsonQueryList Final bson query object. + * @return a boolean indicating whether the complex query was generated or not. + * @throws CatalogDBException If the size of the array of ids does not match the size of the array of version. + */ + boolean generateUidVersionQuery(Query query, List bsonQueryList) throws CatalogDBException { + if (!query.containsKey(VERSION) || query.getAsIntegerList(VERSION).size() == 1) { + return false; + } + if (!query.containsKey(PRIVATE_UID) && !query.containsKey(ID) && !query.containsKey(PRIVATE_UUID)) { + return false; + } + int numIds = 0; + numIds += query.containsKey(ID) ? 1 : 0; + numIds += query.containsKey(PRIVATE_UID) ? 1 : 0; + numIds += query.containsKey(PRIVATE_UUID) ? 1 : 0; + + if (numIds > 1) { + List versionList = query.getAsIntegerList(VERSION); + if (versionList.size() > 1) { + throw new CatalogDBException("Cannot query by more than one version when more than one id type is being queried"); + } + return false; + } + + String idQueried = PRIVATE_UID; + idQueried = query.containsKey(ID) ? ID : idQueried; + idQueried = query.containsKey(PRIVATE_UUID) ? PRIVATE_UUID : idQueried; + + List idList; + if (PRIVATE_UID.equals(idQueried)) { + idList = query.getAsLongList(PRIVATE_UID); + } else { + idList = query.getAsStringList(idQueried); + } + List versionList = query.getAsIntegerList(VERSION); + + if (versionList.size() > 1 && idList.size() > 1 && versionList.size() != idList.size()) { + throw new CatalogDBException("The size of the array of versions should match the size of the array of ids to be queried"); + } + + List bsonQuery = new ArrayList<>(); + for (int i = 0; i < versionList.size(); i++) { + Document docQuery = new Document(VERSION, versionList.get(i)); + if (idList.size() == 1) { + docQuery.put(idQueried, idList.get(0)); + } else { + docQuery.put(idQueried, idList.get(i)); + } + bsonQuery.add(docQuery); + } + + if (!bsonQuery.isEmpty()) { + bsonQueryList.add(Filters.or(bsonQuery)); + + query.remove(idQueried); + query.remove(VERSION); + + return true; + } + + return false; + } + + private String getClientSessionUuid(ClientSession session) { + UUID sessionUUID = session.getServerSession().getIdentifier().getBinary("id").asUuid(); + long transactionNumber = session.getServerSession().getTransactionNumber(); + + // Generate new UUID with sessionId + transactionNumber + return new UUID(sessionUUID.getMostSignificantBits(), sessionUUID.getLeastSignificantBits() + transactionNumber).toString(); + } + + public interface VersionedModelExecution { + T execute(List entries) throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException; + } + + public interface NonVersionedModelExecution { + T execute() throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException; + } + + public interface ReferenceModelExecution { + void execute(DBIterator iterator) throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException; + } + + @FunctionalInterface + public interface PostVersionIncrementIterator { + DBIterator iterator(ClientSession session, Query query, QueryOptions options) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + } + + protected void insert(ClientSession session, Document document) { + // Versioning private parameters + document.put(VERSION, 1); + document.put(RELEASE_FROM_VERSION, Arrays.asList(document.getInteger(RELEASE))); + document.put(LAST_OF_VERSION, true); + document.put(LAST_OF_RELEASE, true); + + String uuid = getClientSessionUuid(session); + document.put(PRIVATE_TRANSACTION_ID, uuid); + collection.insert(session, document, QueryOptions.empty()); + archiveCollection.insert(session, document, QueryOptions.empty()); + } + + protected OpenCGAResult update(ClientSession session, Bson sourceQuery, VersionedModelExecution> update, + PostVersionIncrementIterator postVersionIncrementIterator, + ReferenceModelExecution postVersionIncrementExecution) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + return update(session, sourceQuery, Collections.emptyList(), update, Collections.emptyList(), postVersionIncrementIterator, + postVersionIncrementExecution); + } + + protected OpenCGAResult update(ClientSession session, Bson sourceQuery, List fieldsToInclude, + VersionedModelExecution> update, + PostVersionIncrementIterator postVersionIncrementIterator, + ReferenceModelExecution postVersionIncrementExecution) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + return update(session, sourceQuery, fieldsToInclude, update, Collections.emptyList(), postVersionIncrementIterator, + postVersionIncrementExecution); + } + + protected OpenCGAResult update(ClientSession session, Bson sourceQuery, List fieldsToInclude, + VersionedModelExecution> update, + List postVersionIncrementAdditionalIncludeFields, + PostVersionIncrementIterator dbIterator, + ReferenceModelExecution postVersionIncrementExecution) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + String uuid = getClientSessionUuid(session); + + // 1. Increment version + // 1.1 Only increase version of those documents not already increased by same transaction id + Set includeFields = new HashSet<>(Arrays.asList(PRIVATE_UID, VERSION, RELEASE_FROM_VERSION, PRIVATE_TRANSACTION_ID)); + if (fieldsToInclude != null) { + includeFields.addAll(fieldsToInclude); + } + QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, includeFields); + List entryList = new LinkedList<>(); + List allUids = new LinkedList<>(); + List uidsChanged = new LinkedList<>(); + try (MongoDBIterator iterator = collection.iterator(session, sourceQuery, null, null, options)) { + while (iterator.hasNext()) { + Document result = iterator.next(); + entryList.add(result); + + long uid = result.get(PRIVATE_UID, Number.class).longValue(); + int version = result.get(VERSION, Number.class).intValue(); + String transactionId = result.getString(PRIVATE_TRANSACTION_ID); + allUids.add(uid); + + if (!uuid.equals(transactionId)) { + // If the version hasn't been incremented yet in this transaction + uidsChanged.add(uid); + + Document collectionUpdate = new Document(); + Document archiveCollectionUpdate = new Document(); + processReleaseSnapshotChanges(result, collectionUpdate, archiveCollectionUpdate); + + Bson bsonQuery = Filters.and( + Filters.eq(PRIVATE_UID, uid), + Filters.eq(VERSION, version) + ); + // Update previous version + logger.debug("Updating previous version: query : {}, update: {}", bsonQuery.toBsonDocument(), + archiveCollectionUpdate.toBsonDocument()); + archiveCollection.update(session, bsonQuery, new Document("$set", archiveCollectionUpdate), QueryOptions.empty()); + + // Add current transaction id to the document so we don't enter here twice in the same transaction + collectionUpdate.put(PRIVATE_TRANSACTION_ID, uuid); + // Update current version + logger.debug("Updating current version: query : {}, update: {}", bsonQuery.toBsonDocument(), + collectionUpdate.toBsonDocument()); + collection.update(session, bsonQuery, new Document("$set", collectionUpdate), QueryOptions.empty()); + } + } + } + + // 2. Execute main update + OpenCGAResult executionResult = update.execute(entryList); + + // 3. Fetch document containing update and copy into the archive collection + Bson bsonQuery = Filters.in(PRIVATE_UID, allUids); + options = new QueryOptions(MongoDBCollection.NO_CURSOR_TIMEOUT, true); + QueryOptions upsertOptions = new QueryOptions() + .append(MongoDBCollection.REPLACE, true) + .append(MongoDBCollection.UPSERT, true); + try (MongoDBIterator iterator = collection.iterator(session, bsonQuery, null, null, options)) { + while (iterator.hasNext()) { + Document result = iterator.next(); + result.remove(PRIVATE_MONGO_ID); + result.put(PRIVATE_TRANSACTION_ID, uuid); + + // Some annotations have "." as part of their keys. Mongo does not support that so we call this method to replace them. + Document fixedResult = GenericDocumentComplexConverter.replaceDots(result); + + // Insert/replace in archive collection + Bson tmpBsonQuery = Filters.and( + Filters.eq(PRIVATE_UID, fixedResult.get(PRIVATE_UID)), + Filters.eq(VERSION, fixedResult.get(VERSION)) + ); + logger.debug("Copying current document to archive: query : {}", tmpBsonQuery.toBsonDocument()); + archiveCollection.update(session, tmpBsonQuery, fixedResult, upsertOptions); + } + } + + // 4. Perform any additional reference checks/updates over those that have increased its version in this call + if (!uidsChanged.isEmpty()) { + Query query = new Query(PRIVATE_UID, uidsChanged); + if (postVersionIncrementExecution != null) { + List includeList = new ArrayList<>(Arrays.asList(PRIVATE_UID, PRIVATE_UUID, ID, PRIVATE_STUDY_UID, VERSION)); + if (postVersionIncrementExecution != null) { + includeList.addAll(postVersionIncrementAdditionalIncludeFields); + } + + logger.debug("Executing react code after incrementing version: query : {}={}", PRIVATE_UID, uidsChanged); + options = new QueryOptions() + .append(QueryOptions.INCLUDE, includeList) + .append(MongoDBCollection.NO_CURSOR_TIMEOUT, true); + try (DBIterator iterator = dbIterator.iterator(session, query, options)) { + postVersionIncrementExecution.execute(iterator); + } +// try (MongoDBIterator iterator = collection.iterator(session, query, null, null, options)) { +// postVersionIncrementExecution.execute(iterator); +// } + } + } + + return executionResult; + } + + protected OpenCGAResult updateWithoutVersionIncrement(Bson sourceQuery, NonVersionedModelExecution> update) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + // Execute main update + OpenCGAResult executionResult = update.execute(); + + // Fetch document containing update and copy into the archive collection + QueryOptions options = new QueryOptions(MongoDBCollection.NO_CURSOR_TIMEOUT, true); + QueryOptions upsertOptions = new QueryOptions() + .append(MongoDBCollection.REPLACE, true) + .append(MongoDBCollection.UPSERT, true); + try (MongoDBIterator iterator = collection.iterator(sourceQuery, options)) { + while (iterator.hasNext()) { + Document result = iterator.next(); + result.remove(PRIVATE_MONGO_ID); + + // Insert/replace in archive collection + Bson tmpBsonQuery = Filters.and( + Filters.eq(PRIVATE_UID, result.get(PRIVATE_UID)), + Filters.eq(VERSION, result.get(VERSION)) + ); + archiveCollection.update(tmpBsonQuery, result, upsertOptions); + } + } + + return executionResult; + } + + /** + * Revert to a previous version. + * + * @param clientSession ClientSession for transactional operations. + * @param uid UID of the element to be recovered. + * @param version Version to be recovered. + * @return the new latest document that will be written in the database. + * @throws CatalogDBException in case of any issue. + */ + protected Document revertToVersion(ClientSession clientSession, long uid, int version) throws CatalogDBException { + Bson query = Filters.and( + Filters.eq(PRIVATE_UID, uid), + Filters.eq(VERSION, version) + ); + DataResult result = archiveCollection.find(clientSession, query, EXCLUDE_MONGO_ID); + if (result.getNumResults() == 0) { + throw new CatalogDBException("Could not find version '" + version + "'"); + } + Document document = result.first(); + + QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(RELEASE_FROM_VERSION, VERSION, LAST_OF_RELEASE)); + // Find out latest version available + result = collection.find(clientSession, Filters.eq(PRIVATE_UID, uid), options); + if (result.getNumResults() == 0) { + throw new CatalogDBException("Unexpected error. Could not find 'uid': " + uid); + } + int lastVersion = result.first().getInteger(VERSION); + + // Delete previous version from active collection + collection.remove(clientSession, Filters.eq(PRIVATE_UID, uid), QueryOptions.empty()); + + // Edit previous version from archive collection + query = Filters.and( + Filters.eq(PRIVATE_UID, uid), + Filters.eq(VERSION, lastVersion) + ); + archiveCollection.update(clientSession, query, Updates.combine( + Updates.set(LAST_OF_RELEASE, false), + Updates.set(LAST_OF_VERSION, false) + ), QueryOptions.empty()); + + // Edit private fields from document to be restored + document.put(VERSION, lastVersion + 1); + document.put(RELEASE_FROM_VERSION, result.first().get(RELEASE_FROM_VERSION)); + document.put(LAST_OF_RELEASE, result.first().get(LAST_OF_RELEASE)); + + // Add restored element to main and archive collection + collection.insert(clientSession, document, QueryOptions.empty()); + archiveCollection.insert(clientSession, document, QueryOptions.empty()); + + return document; + } + + protected void delete(ClientSession session, Bson query) throws CatalogDBException { + // Remove any old documents from the "delete" collection matching the criteria + deletedCollection.remove(session, query, QueryOptions.empty()); + + // Remove document from main collection + DataResult remove = collection.remove(session, query, QueryOptions.empty()); + if (remove.getNumDeleted() == 0) { + logger.error("Delete operation for '{}' could not be performed. Num matches: {}", query.toBsonDocument(), + remove.getNumMatches()); + throw new CatalogDBException("Delete operation could not be performed"); + } + + // Add versioned documents to "delete" collection + InternalStatus internalStatus = new InternalStatus(InternalStatus.DELETED); + Document status; + try { + status = getMongoDBDocument(internalStatus, "status"); + } catch (CatalogDBException e) { + status = new Document("id", InternalStatus.DELETED); + } + for (Document document : archiveCollection.find(session, query, QueryOptions.empty()).getResults()) { + Document internal = document.get("internal", Document.class); + internal.put("status", status); + + deletedCollection.insert(session, document, QueryOptions.empty()); + } + + // Remove documents from versioned collection + archiveCollection.remove(session, query, QueryOptions.empty()); + } + + /** + * Given the current document, it puts in collectionUpdate and archiveCollectionUpdate the changes that need to be applied. + * It takes care of the private fields that allow performing queries by snapshot: + * LAST_OF_VERSION, LAST_OF_RELEASE, RELEASE_FROM_VERSION + * + * @param document Current document. + * @param collectionUpdate Empty document where we will put the changes to be applied to the main collection. + * @param archiveCollectionUpdate Empty document where we will put the changes to be applied to the archive collection. + */ + private void processReleaseSnapshotChanges(Document document, Document collectionUpdate, Document archiveCollectionUpdate) { + int version = document.get(VERSION, Number.class).intValue(); + List releaseFromVersion = document.getList(RELEASE_FROM_VERSION, Integer.class); + + // Current release number + int release; + if (releaseFromVersion.size() > 1) { + release = releaseFromVersion.get(releaseFromVersion.size() - 1); + + // If it contains several releases, it means this is the first update on the current release, so we just need to take the + // current release number out + releaseFromVersion.remove(releaseFromVersion.size() - 1); + } else { + release = releaseFromVersion.get(0); + + // If it is 1, it means that the previous version being checked was made on this same release as well, so it won't be the + // last version of the release + archiveCollectionUpdate.put(LAST_OF_RELEASE, false); + } + archiveCollectionUpdate.put(RELEASE_FROM_VERSION, releaseFromVersion); + archiveCollectionUpdate.put(LAST_OF_VERSION, false); + + // We update the information for the new version of the document + collectionUpdate.put(RELEASE_FROM_VERSION, Collections.singletonList(release)); + collectionUpdate.put(VERSION, version + 1); + } + +} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java index b0d67dc923a..c91fd4c3665 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java @@ -29,26 +29,28 @@ import org.opencb.commons.datastore.core.*; import org.opencb.commons.datastore.mongodb.MongoDBCollection; import org.opencb.commons.datastore.mongodb.MongoDBIterator; -import org.opencb.opencga.catalog.db.api.*; +import org.opencb.opencga.catalog.db.api.DBIterator; +import org.opencb.opencga.catalog.db.api.ProjectDBAdaptor; +import org.opencb.opencga.catalog.db.api.SampleDBAdaptor; +import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; import org.opencb.opencga.catalog.db.mongodb.converters.StudyConverter; import org.opencb.opencga.catalog.db.mongodb.converters.VariableSetConverter; import org.opencb.opencga.catalog.db.mongodb.iterators.StudyCatalogMongoDBIterator; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.utils.Constants; +import org.opencb.opencga.catalog.utils.FqnUtils; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.catalog.utils.UuidUtils; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.Configuration; -import org.opencb.opencga.core.models.cohort.Cohort; import org.opencb.opencga.core.models.common.Annotable; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.common.InternalStatus; -import org.opencb.opencga.core.models.family.Family; import org.opencb.opencga.core.models.file.File; -import org.opencb.opencga.core.models.job.Job; import org.opencb.opencga.core.models.project.Project; import org.opencb.opencga.core.models.study.*; import org.opencb.opencga.core.response.OpenCGAResult; @@ -79,7 +81,7 @@ public class StudyMongoDBAdaptor extends CatalogMongoDBAdaptor implements StudyD private VariableSetConverter variableSetConverter; public StudyMongoDBAdaptor(MongoDBCollection studyCollection, MongoDBCollection deletedStudyCollection, Configuration configuration, - MongoDBAdaptorFactory dbAdaptorFactory) { + OrganizationMongoDBAdaptorFactory dbAdaptorFactory) { super(configuration, LoggerFactory.getLogger(StudyMongoDBAdaptor.class)); this.dbAdaptorFactory = dbAdaptorFactory; this.studyCollection = studyCollection; @@ -88,7 +90,7 @@ public StudyMongoDBAdaptor(MongoDBCollection studyCollection, MongoDBCollection this.variableSetConverter = new VariableSetConverter(); } - static Document getDocumentUpdateParams(ObjectMap parameters) { + static Document getDocumentUpdateParams(ObjectMap parameters) throws CatalogDBException { Document studyParameters = new Document(); String[] acceptedParams = {QueryParams.ALIAS.key(), QueryParams.NAME.key(), QueryParams.DESCRIPTION.key()}; @@ -168,9 +170,8 @@ private boolean studyIdExists(ClientSession clientSession, long projectId, Strin } @Override - public OpenCGAResult nativeInsert(Map study, String userId) throws CatalogDBException { + public OpenCGAResult nativeInsert(Map study) throws CatalogDBException { Document studyDocument = getMongoDBDocument(study, "study"); - studyDocument.put(PRIVATE_OWNER_ID, userId); return new OpenCGAResult<>(studyCollection.insert(studyDocument, null)); } @@ -198,13 +199,13 @@ public OpenCGAResult nativeInsert(Map study, String userI // } @Override - public OpenCGAResult insert(Project project, Study study, QueryOptions options) throws CatalogDBException { + public OpenCGAResult insert(Project project, Study study, List files, QueryOptions options) throws CatalogDBException { try { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); logger.debug("Starting study insert transaction for study id '{}'", study.getId()); - insert(clientSession, project, study); + insert(clientSession, project, study, files); return endWrite(tmpStartTime, 1, 1, 0, 0, null); }); } catch (Exception e) { @@ -213,7 +214,7 @@ public OpenCGAResult insert(Project project, Study study, QueryOptions op } } - Study insert(ClientSession clientSession, Project project, Study study) + Study insert(ClientSession clientSession, Project project, Study study, List files) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { if (project.getUid() < 0) { throw CatalogDBException.uidNotFound("Project", project.getUid()); @@ -228,29 +229,13 @@ Study insert(ClientSession clientSession, Project project, Study study) } //Set new ID - long studyUid = getNewUid(); + long studyUid = getNewUid(clientSession); study.setUid(studyUid); if (StringUtils.isEmpty(study.getUuid())) { study.setUuid(UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.STUDY)); } - //Empty nested fields - List files = study.getFiles(); - study.setFiles(Collections.emptyList()); - - List jobs = study.getJobs(); - study.setJobs(Collections.emptyList()); - - List cohorts = study.getCohorts(); - study.setCohorts(Collections.emptyList()); - - List panels = study.getPanels(); - study.setPanels(Collections.emptyList()); - - List families = study.getFamilies(); - study.setFamilies(Collections.emptyList()); - study.setFqn(project.getFqn() + ":" + study.getId()); List variableSets = study.getVariableSets(); @@ -274,7 +259,6 @@ Study insert(ClientSession clientSession, Project project, Study study) .append(PRIVATE_UID, project.getUid()) .append(PRIVATE_UUID, project.getUuid()) ); - studyObject.put(PRIVATE_OWNER_ID, StringUtils.split(project.getFqn(), "@")[0]); studyObject.put(PRIVATE_CREATION_DATE, StringUtils.isNotEmpty(study.getCreationDate()) ? TimeUtils.toDate(study.getCreationDate()) : TimeUtils.getDate()); @@ -290,37 +274,13 @@ Study insert(ClientSession clientSession, Project project, Study study) } } - if (jobs != null) { - for (Job job : jobs) { - dbAdaptorFactory.getCatalogJobDBAdaptor().insert(clientSession, study.getUid(), job); - } - } - - if (cohorts != null) { - for (Cohort cohort : cohorts) { - dbAdaptorFactory.getCatalogCohortDBAdaptor().insert(clientSession, study.getUid(), cohort, Collections.emptyList()); - } - } - - if (panels != null) { - for (org.opencb.opencga.core.models.panel.Panel panel : panels) { - dbAdaptorFactory.getCatalogPanelDBAdaptor().insert(clientSession, study.getUid(), panel); - } - } - - if (families != null) { - for (Family family : families) { - dbAdaptorFactory.getCatalogFamilyDBAdaptor().insert(clientSession, study.getUid(), family, Collections.emptyList()); - } - } - return study; } @Override public OpenCGAResult getAllStudiesInProject(long projectUid, QueryOptions options) throws CatalogDBException { long startTime = startQuery(); - dbAdaptorFactory.getCatalogProjectDbAdaptor().checkId(projectUid); + dbAdaptorFactory.getCatalogProjectDBAdaptor().checkId(projectUid); Query query = new Query(QueryParams.PROJECT_UID.key(), projectUid); return endQuery(startTime, get(query, options)); } @@ -333,7 +293,7 @@ public boolean hasStudyPermission(long studyId, String user, StudyPermissions.Pe throw new CatalogDBException("Study " + studyId + " not found"); } - return checkStudyPermission((Document) queryResult.first(), user, permission.name()); + return checkStudyPermission(dbAdaptorFactory.getOrganizationId(), (Document) queryResult.first(), user, permission.name()); } @Override @@ -345,19 +305,6 @@ public long getId(long projectId, String studyAlias) throws CatalogDBException { return studies == null || studies.isEmpty() ? -1 : studies.get(0).getUid(); } - @Override - public long getProjectUidByStudyUid(long studyUid) throws CatalogDBException { - Document privateProjet = getPrivateProject(studyUid); - Object id = privateProjet.get(PRIVATE_UID); - return id instanceof Number ? ((Number) id).longValue() : Long.parseLong(id.toString()); - } - - @Override - public String getProjectIdByStudyUid(long studyUid) throws CatalogDBException { - Document privateProjet = getPrivateProject(studyUid); - return privateProjet.getString(ID); - } - int getCurrentRelease(ClientSession clientSession, long studyUid) throws CatalogDBException { Query query = new Query(QueryParams.UID.key(), studyUid); QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, QueryParams.FQN.key()); @@ -367,17 +314,13 @@ int getCurrentRelease(ClientSession clientSession, long studyUid) throws Catalog throw new CatalogDBException("Study uid '" + studyUid + "' not found."); } - String[] split = StringUtils.split(StringUtils.split(studyResult.first().getFqn(), ":")[0], "@"); - String userId = split[0]; - String projectId = split[1]; + String projectFqn = FqnUtils.parse(studyResult.first().getFqn()).getProjectFqn(); - query = new Query() - .append(ProjectDBAdaptor.QueryParams.USER_ID.key(), userId) - .append(ProjectDBAdaptor.QueryParams.ID.key(), projectId); + query = new Query(ProjectDBAdaptor.QueryParams.FQN.key(), projectFqn); options = new QueryOptions(QueryOptions.INCLUDE, ProjectDBAdaptor.QueryParams.CURRENT_RELEASE.key()); - OpenCGAResult projectResult = dbAdaptorFactory.getCatalogProjectDbAdaptor().get(clientSession, query, options); + OpenCGAResult projectResult = dbAdaptorFactory.getCatalogProjectDBAdaptor().get(clientSession, query, options); if (projectResult.getNumResults() == 0) { - throw new CatalogDBException("Project id '" + projectId + "' from user '" + userId + "' not found."); + throw new CatalogDBException("Project '" + projectFqn + "' not found."); } return projectResult.first().getCurrentRelease(); @@ -398,17 +341,6 @@ private Document getPrivateProject(long studyUid) throws CatalogDBException { return privateProjet; } - @Override - public String getOwnerId(long studyId) throws CatalogDBException { - Query query = new Query(QueryParams.UID.key(), studyId); - QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, PRIVATE_OWNER_ID); - OpenCGAResult documentDataResult = nativeGet(query, options); - if (documentDataResult.getNumResults() == 0) { - throw CatalogDBException.uidNotFound("Study", studyId); - } - return documentDataResult.first().getString(PRIVATE_OWNER_ID); - } - @Override public OpenCGAResult createGroup(long studyId, Group group) throws CatalogDBException { Document query = new Document() @@ -516,9 +448,22 @@ public OpenCGAResult addUsersToGroup(long studyId, String groupId, List(result); } + void addUsersToAdminsAndMembersGroup(ClientSession clientSession, List members) throws CatalogDBException { + if (CollectionUtils.isEmpty(members)) { + throw new CatalogDBException("List of 'members' is missing or empty."); + } + + Document query = new Document(QueryParams.GROUP_ID.key(), ParamConstants.ADMINS_GROUP); + Document update = new Document("$addToSet", new Document("groups.$.userIds", new Document("$each", members))); + studyCollection.update(clientSession, query, update, new QueryOptions(MongoDBCollection.MULTI, true)); + + query = new Document(QueryParams.GROUP_ID.key(), ParamConstants.MEMBERS_GROUP); + studyCollection.update(clientSession, query, update, new QueryOptions(MongoDBCollection.MULTI, true)); + } + @Override public OpenCGAResult removeUsersFromGroup(long studyId, String groupId, List members) throws CatalogDBException { - if (members == null || members.size() == 0) { + if (CollectionUtils.isEmpty(members)) { throw new CatalogDBException("Unable to remove members from group. List of members is empty"); } @@ -533,9 +478,20 @@ public OpenCGAResult removeUsersFromGroup(long studyId, String groupId, L return new OpenCGAResult<>(update); } + OpenCGAResult removeUsersFromAdminsGroup(ClientSession clientSession, List members) throws CatalogDBException { + if (CollectionUtils.isEmpty(members)) { + throw new CatalogDBException("Unable to remove members from group. List of members is empty."); + } + + Document query = new Document() + .append(QueryParams.GROUP_ID.key(), ParamConstants.ADMINS_GROUP); + Bson pull = Updates.pullAll("groups.$.userIds", members); + DataResult update = studyCollection.update(clientSession, query, pull, new QueryOptions(MongoDBCollection.MULTI, true)); + return new OpenCGAResult<>(update); + } + @Override - public OpenCGAResult removeUsersFromAllGroups(long studyId, List users) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult removeUsersFromAllGroups(long studyId, List users) throws CatalogException { if (users == null || users.size() == 0) { throw new CatalogDBException("Unable to remove users from groups. List of users is empty"); } @@ -590,8 +546,7 @@ public OpenCGAResult syncGroup(long studyId, String groupId, Group.Sync s } @Override - public OpenCGAResult resyncUserWithSyncedGroups(String user, List groupList, String authOrigin) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult resyncUserWithSyncedGroups(String user, List groupList, String authOrigin) throws CatalogException { if (StringUtils.isEmpty(user)) { throw new CatalogDBException("Missing user field"); } @@ -644,8 +599,7 @@ public OpenCGAResult resyncUserWithSyncedGroups(String user, List @Override public OpenCGAResult updateUserFromGroups(String user, List studyUids, List groupList, - ParamUtils.AddRemoveAction action) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + ParamUtils.AddRemoveAction action) throws CatalogException { if (StringUtils.isEmpty(user)) { throw new CatalogParameterException("Missing user parameter"); @@ -864,7 +818,7 @@ public OpenCGAResult createVariableSet(long studyId, VariableSet va @Override public OpenCGAResult addFieldToVariableSet(long studyUid, long variableSetId, Variable variable, String user) - throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException { + throws CatalogException { OpenCGAResult variableSet = getVariableSet(variableSetId, new QueryOptions(), user); checkVariableNotInVariableSet(variableSet.first(), variable.getId()); @@ -941,9 +895,7 @@ public OpenCGAResult renameFieldVariableSet(long variableSetId, Str @Override public OpenCGAResult removeFieldFromVariableSet(long studyUid, long variableSetId, String name, String user) - throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException { - long startTime = startQuery(); - + throws CatalogException { OpenCGAResult variableSet = getVariableSet(variableSetId, new QueryOptions(), user); checkVariableInVariableSet(variableSet.first(), name); @@ -1030,13 +982,13 @@ public OpenCGAResult getVariableSet(long variableSetId, QueryOption long startTime = startQuery(); Bson query = new Document("variableSets", new Document("$elemMatch", new Document(PRIVATE_UID, variableSetId))); - QueryOptions qOptions = new QueryOptions(QueryOptions.INCLUDE, "variableSets.$,_ownerId,groups,_acl"); + QueryOptions qOptions = new QueryOptions(QueryOptions.INCLUDE, "variableSets.$,groups,_acl"); DataResult studyDataResult = studyCollection.find(query, qOptions); if (studyDataResult.getNumResults() == 0) { throw new CatalogDBException("Variable set not found."); } - if (!checkCanViewStudy(studyDataResult.first(), user)) { + if (!checkCanViewStudy(dbAdaptorFactory.getOrganizationId(), studyDataResult.first(), user)) { throw CatalogAuthorizationException.deny(user, "view", "VariableSet", variableSetId, ""); } Study study = studyConverter.convertToDataModelType(studyDataResult.first()); @@ -1045,7 +997,7 @@ public OpenCGAResult getVariableSet(long variableSetId, QueryOption } // Check if it is confidential if (study.getVariableSets().get(0).isConfidential()) { - if (!checkStudyPermission(studyDataResult.first(), user, + if (!checkStudyPermission(dbAdaptorFactory.getOrganizationId(), studyDataResult.first(), user, StudyPermissions.Permissions.CONFIDENTIAL_VARIABLE_SET_ACCESS.toString())) { throw CatalogAuthorizationException.deny(user, StudyPermissions.Permissions.CONFIDENTIAL_VARIABLE_SET_ACCESS.toString(), "VariableSet", variableSetId, ""); @@ -1192,11 +1144,11 @@ public OpenCGAResult getVariableSets(Query query, QueryOptions quer return endQuery(startTime, Collections.emptyList()); } - if (!checkCanViewStudy(queryResult.first(), user)) { + if (!checkCanViewStudy(dbAdaptorFactory.getOrganizationId(), queryResult.first(), user)) { throw new CatalogAuthorizationException("Permission denied: " + user + " cannot see any variable set"); } - boolean hasConfidentialPermission = checkStudyPermission(queryResult.first(), user, + boolean hasConfidentialPermission = checkStudyPermission(dbAdaptorFactory.getOrganizationId(), queryResult.first(), user, StudyPermissions.Permissions.CONFIDENTIAL_VARIABLE_SET_ACCESS.toString()); List variableSets = new ArrayList<>(); for (Document studyDocument : queryResult.getResults()) { @@ -1211,8 +1163,7 @@ public OpenCGAResult getVariableSets(Query query, QueryOptions quer } @Override - public OpenCGAResult deleteVariableSet(long studyUid, VariableSet variableSet, boolean force) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult deleteVariableSet(long studyUid, VariableSet variableSet, boolean force) throws CatalogException { try { return runTransaction(clientSession -> { if (force) { @@ -1320,99 +1271,6 @@ private void checkVariableSetInUse(VariableSet variableSet) } } - @Override - public long getStudyIdByVariableSetId(long variableSetId) throws CatalogDBException { -// DBObject query = new BasicDBObject("variableSets.id", variableSetId); - Bson query = Filters.eq("variableSets." + PRIVATE_UID, variableSetId); - Bson projection = Projections.include(PRIVATE_UID); - -// DataResult queryResult = studyCollection.find(query, new BasicDBObject(PRIVATE_UID, true), null); - DataResult queryResult = studyCollection.find(query, projection, null); - - if (!queryResult.getResults().isEmpty()) { - Object id = queryResult.getResults().get(0).get(PRIVATE_UID); - return id instanceof Number ? ((Number) id).intValue() : (int) Double.parseDouble(id.toString()); - } else { - throw CatalogDBException.uidNotFound("VariableSet", variableSetId); - } - } - - /* - * Helper methods - ********************/ - - @Override - public OpenCGAResult getStudiesFromUser(String userId, QueryOptions queryOptions) throws CatalogDBException { - OpenCGAResult result = OpenCGAResult.empty(); - - OpenCGAResult allProjects = dbAdaptorFactory.getCatalogProjectDbAdaptor().get(userId, new QueryOptions()); - if (allProjects.getNumResults() == 0) { - return result; - } - - for (Project project : allProjects.getResults()) { - OpenCGAResult allStudiesInProject = getAllStudiesInProject(project.getUid(), queryOptions); - if (allStudiesInProject.getNumResults() > 0) { - result.getResults().addAll(allStudiesInProject.getResults()); - result.setTime(result.getTime() + allStudiesInProject.getTime()); - } - } - - result.setNumMatches(result.getResults().size()); - result.setNumResults(result.getResults().size()); - - return result; - } - - private void joinFields(Study study, QueryOptions options) throws CatalogDBException { - try { - joinFields(study, options, null); - } catch (CatalogAuthorizationException | CatalogParameterException e) { - throw new CatalogDBException(e); - } - } - - private void joinFields(Study study, QueryOptions options, String user) - throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException { - long studyId = study.getUid(); - if (studyId <= 0 || options == null) { - return; - } - - if (options.getBoolean("includeFiles")) { - if (StringUtils.isEmpty(user)) { - study.setFiles(dbAdaptorFactory.getCatalogFileDBAdaptor().getAllInStudy(studyId, options).getResults()); - } else { - Query query = new Query(FileDBAdaptor.QueryParams.STUDY_UID.key(), studyId); - study.setFiles(dbAdaptorFactory.getCatalogFileDBAdaptor().get(studyId, query, options, user).getResults()); - } - } - if (options.getBoolean("includeJobs")) { - if (StringUtils.isEmpty(user)) { - study.setJobs(dbAdaptorFactory.getCatalogJobDBAdaptor().getAllInStudy(studyId, options).getResults()); - } else { - Query query = new Query(JobDBAdaptor.QueryParams.STUDY_UID.key(), studyId); - study.setJobs(dbAdaptorFactory.getCatalogJobDBAdaptor().get(studyId, query, options, user).getResults()); - } - } - if (options.getBoolean("includeSamples")) { - if (StringUtils.isEmpty(user)) { - study.setSamples(dbAdaptorFactory.getCatalogSampleDBAdaptor().getAllInStudy(studyId, options).getResults()); - } else { - Query query = new Query(SampleDBAdaptor.QueryParams.STUDY_UID.key(), studyId); - study.setSamples(dbAdaptorFactory.getCatalogSampleDBAdaptor().get(studyId, query, options, user).getResults()); - } - } - if (options.getBoolean("includeIndividuals")) { - Query query = new Query(IndividualDBAdaptor.QueryParams.STUDY_UID.key(), studyId); - if (StringUtils.isEmpty(user)) { - study.setIndividuals(dbAdaptorFactory.getCatalogIndividualDBAdaptor().get(query, options).getResults()); - } else { - study.setIndividuals(dbAdaptorFactory.getCatalogIndividualDBAdaptor().get(studyId, query, options, user).getResults()); - } - } - } - @Override public OpenCGAResult count(Query query) throws CatalogDBException { return count(null, query); @@ -1448,9 +1306,8 @@ void updateProjectId(ClientSession clientSession, long projectUid, String newPro while (studyIterator.hasNext()) { Study study = studyIterator.next(); - String[] split = study.getFqn().split("@"); - String[] split1 = split[1].split(":"); - String newFqn = split[0] + "@" + newProjectId + ":" + split1[1]; + FqnUtils.FQN oldFqn = FqnUtils.parse(study.getFqn()); + String newFqn = FqnUtils.buildFqn(oldFqn.getOrganization(), newProjectId, oldFqn.getStudy()); // Update the internal project id and fqn Bson update = new Document("$set", new Document() @@ -1479,7 +1336,7 @@ public OpenCGAResult update(long studyUid, ObjectMap parameters, QueryOptions qu try { return runTransaction(clientSession -> privateUpdate(clientSession, studyResult.first(), parameters)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update study {}: {}", studyId, e.getMessage(), e); throw new CatalogDBException("Could not update study '" + studyId + "': " + e.getMessage(), e.getCause()); } @@ -1502,7 +1359,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, QueryOptions quer Study study = iterator.next(); try { result.append(runTransaction(clientSession -> privateUpdate(clientSession, study, parameters))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not update study {}: {}", study.getId(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, study.getId(), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -1616,7 +1473,7 @@ public OpenCGAResult delete(Study study) throws CatalogDBException, CatalogParam throw new CatalogDBException("Could not find study " + study.getId() + " with uid " + study.getUid()); } return runTransaction(clientSession -> privateDelete(clientSession, result.first())); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete study {}: {}", study.getId(), e.getMessage(), e); throw new CatalogDBException("Could not delete study " + study.getId() + ": " + e.getMessage(), e.getCause()); } @@ -1632,7 +1489,7 @@ public OpenCGAResult delete(Query query) throws CatalogDBException { String studyId = study.getString(QueryParams.ID.key()); try { result.append(runTransaction(clientSession -> privateDelete(clientSession, study))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete study {}: {}", studyId, e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, studyId, e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -1750,28 +1607,18 @@ public OpenCGAResult get(Query query, QueryOptions options) throws Catalo OpenCGAResult get(ClientSession clientSession, Query query, QueryOptions options) throws CatalogDBException { long startTime = startQuery(); - OpenCGAResult studyDataResult; try (DBIterator dbIterator = iterator(clientSession, query, options)) { - studyDataResult = endQuery(startTime, dbIterator); - } - for (Study study : studyDataResult.getResults()) { - joinFields(study, options); + return endQuery(startTime, dbIterator); } - return studyDataResult; } @Override public OpenCGAResult get(Query query, QueryOptions options, String user) throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException { long startTime = startQuery(); - OpenCGAResult studyDataResult; try (DBIterator dbIterator = iterator(query, options, user)) { - studyDataResult = endQuery(startTime, dbIterator); - } - for (Study study : studyDataResult.getResults()) { - joinFields(study, options, user); + return endQuery(startTime, dbIterator); } - return studyDataResult; } @Override @@ -1807,7 +1654,7 @@ public DBIterator iterator(Query query, QueryOptions options) throws Cata private DBIterator iterator(ClientSession clientSession, Query query, QueryOptions options) throws CatalogDBException { MongoDBIterator mongoCursor = getMongoCursor(clientSession, query, options); - return new StudyCatalogMongoDBIterator<>(mongoCursor, options, studyConverter); + return new StudyCatalogMongoDBIterator<>(mongoCursor, clientSession, dbAdaptorFactory, options, studyConverter, null, null); } @Override @@ -1819,15 +1666,15 @@ DBIterator nativeIterator(ClientSession clientSession, Query query, QueryOptions QueryOptions queryOptions = options != null ? new QueryOptions(options) : new QueryOptions(); queryOptions.put(NATIVE_QUERY, true); MongoDBIterator mongoCursor = getMongoCursor(clientSession, query, queryOptions); - return new StudyCatalogMongoDBIterator<>(mongoCursor, options); + return new StudyCatalogMongoDBIterator<>(mongoCursor, clientSession, dbAdaptorFactory, options, null, null, null); } @Override public DBIterator iterator(Query query, QueryOptions options, String user) throws CatalogDBException, CatalogAuthorizationException { MongoDBIterator mongoCursor = getMongoCursor(null, query, options); - Function iteratorFilter = (d) -> checkCanViewStudy(d, user); - return new StudyCatalogMongoDBIterator<>(mongoCursor, options, studyConverter, iteratorFilter); + Function iteratorFilter = (d) -> checkCanViewStudy(dbAdaptorFactory.getOrganizationId(), d, user); + return new StudyCatalogMongoDBIterator<>(mongoCursor, null, dbAdaptorFactory, options, studyConverter, iteratorFilter, user); } @Override @@ -1842,21 +1689,16 @@ public DBIterator nativeIterator(ClientSession clientSession, Query query, Query QueryOptions queryOptions = options != null ? new QueryOptions(options) : new QueryOptions(); queryOptions.put(NATIVE_QUERY, true); MongoDBIterator mongoCursor = getMongoCursor(clientSession, query, queryOptions); - Function iteratorFilter = (d) -> checkCanViewStudy(d, user); - return new StudyCatalogMongoDBIterator(mongoCursor, options, iteratorFilter); + Function iteratorFilter = (d) -> checkCanViewStudy(dbAdaptorFactory.getOrganizationId(), d, user); + return new StudyCatalogMongoDBIterator(mongoCursor, null, dbAdaptorFactory, options, null, iteratorFilter, user); } private MongoDBIterator getMongoCursor(ClientSession clientSession, Query query, QueryOptions options) throws CatalogDBException { options = ParamUtils.defaultObject(options, QueryOptions::new); QueryOptions qOptions = new QueryOptions(options); - if (qOptions.containsKey(QueryOptions.INCLUDE)) { - List includeList = new ArrayList<>(qOptions.getAsStringList(QueryOptions.INCLUDE)); - includeList.add("_ownerId"); - includeList.add("_acl"); - includeList.add(QueryParams.GROUPS.key()); - qOptions.put(QueryOptions.INCLUDE, includeList); - } + qOptions = filterQueryOptionsToIncludeKeys(qOptions, + Arrays.asList(AuthorizationMongoDBUtils.PRIVATE_ACL, QueryParams.GROUPS.key())); qOptions = filterOptions(qOptions, FILTER_ROUTE_STUDIES); fixAclProjection(qOptions); @@ -1917,8 +1759,6 @@ private Bson parseQuery(Query query) throws CatalogDBException { queryCopy.remove(QueryParams.DELETED.key()); fixComplexQueryParam(QueryParams.ATTRIBUTES.key(), queryCopy); - fixComplexQueryParam(QueryParams.BATTRIBUTES.key(), queryCopy); - fixComplexQueryParam(QueryParams.NATTRIBUTES.key(), queryCopy); // Flag indicating whether and OR between ID and ALIAS has been performed and already added to the andBsonList object boolean idOrAliasFlag = false; @@ -1948,14 +1788,6 @@ private Bson parseQuery(Query query) throws CatalogDBException { case ATTRIBUTES: addAutoOrQuery(entry.getKey(), entry.getKey(), queryCopy, queryParam.type(), andBsonList); break; - case BATTRIBUTES: - String mongoKey = entry.getKey().replace(QueryParams.BATTRIBUTES.key(), QueryParams.ATTRIBUTES.key()); - addAutoOrQuery(mongoKey, entry.getKey(), queryCopy, queryParam.type(), andBsonList); - break; - case NATTRIBUTES: - mongoKey = entry.getKey().replace(QueryParams.NATTRIBUTES.key(), QueryParams.ATTRIBUTES.key()); - addAutoOrQuery(mongoKey, entry.getKey(), queryCopy, queryParam.type(), andBsonList); - break; case CREATION_DATE: addAutoOrQuery(PRIVATE_CREATION_DATE, queryParam.key(), queryCopy, queryParam.type(), andBsonList); break; @@ -2005,13 +1837,11 @@ private Bson parseQuery(Query query) throws CatalogDBException { case GROUP_ID: case GROUP_USER_IDS: case RELEASE: - case COHORTS: case VARIABLE_SET: case VARIABLE_SET_UID: case VARIABLE_SET_ID: case VARIABLE_SET_NAME: case VARIABLE_SET_DESCRIPTION: - case OWNER: addAutoOrQuery(queryParam.key(), queryParam.key(), queryCopy, queryParam.type(), andBsonList); break; default: @@ -2042,6 +1872,7 @@ public MongoDBCollection getStudyCollection() { * will be > 0 to increment the size field in the study collection or < 0 to decrement it. * @throws CatalogDBException An exception is launched when the update crashes. */ + @Override public void updateDiskUsage(ClientSession clientSession, long studyId, long size) throws CatalogDBException { Bson query = new Document(QueryParams.UID.key(), studyId); Bson update = Updates.inc(QueryParams.SIZE.key(), size); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java index 0ba4f454192..24c71463e32 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java @@ -18,7 +18,9 @@ import com.mongodb.client.ClientSession; import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; import com.mongodb.client.model.Updates; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.lang3.StringUtils; import org.bson.Document; @@ -36,13 +38,13 @@ import org.opencb.opencga.catalog.db.api.UserDBAdaptor; import org.opencb.opencga.catalog.db.mongodb.converters.UserConverter; import org.opencb.opencga.catalog.db.mongodb.iterators.CatalogMongoDBIterator; -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; -import org.opencb.opencga.catalog.exceptions.CatalogDBException; -import org.opencb.opencga.catalog.exceptions.CatalogParameterException; +import org.opencb.opencga.catalog.exceptions.*; import org.opencb.opencga.catalog.managers.StudyManager; import org.opencb.opencga.catalog.utils.ParamUtils; +import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.common.PasswordUtils; import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.config.AuthenticationOrigin; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.models.common.InternalStatus; import org.opencb.opencga.core.models.project.Project; @@ -54,6 +56,7 @@ import org.opencb.opencga.core.response.OpenCGAResult; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; import java.security.NoSuchAlgorithmException; import java.util.*; import java.util.function.Consumer; @@ -71,10 +74,25 @@ public class UserMongoDBAdaptor extends CatalogMongoDBAdaptor implements UserDBA private final MongoDBCollection deletedUserCollection; private UserConverter userConverter; - private static final String PRIVATE_PASSWORD = "_password"; + // --- Password constants --- + public static final String HASH = "hash"; + public static final String SALT = "salt"; + + public static final String PRIVATE_PASSWORD = "_password"; + + public static final String CURRENT = "current"; + private static final String PRIVATE_PASSWORD_CURRENT = "_password." + CURRENT; + private static final String PRIVATE_PASSWORD_CURRENT_HASH = PRIVATE_PASSWORD_CURRENT + "." + HASH; + private static final String PRIVATE_PASSWORD_CURRENT_SALT = PRIVATE_PASSWORD_CURRENT + "." + SALT; + + public static final String ARCHIVE = "archive"; + public static final String PRIVATE_PASSWORD_ARCHIVE = "_password." + ARCHIVE; + private static final String PRIVATE_PASSWORD_ARCHIVE_HASH = PRIVATE_PASSWORD_ARCHIVE + "." + HASH; + private static final String PRIVATE_PASSWORD_ARCHIVE_SALT = PRIVATE_PASSWORD_ARCHIVE + "." + SALT; + // -------------------------- public UserMongoDBAdaptor(MongoDBCollection userCollection, MongoDBCollection deletedUserCollection, Configuration configuration, - MongoDBAdaptorFactory dbAdaptorFactory) { + OrganizationMongoDBAdaptorFactory dbAdaptorFactory) { super(configuration, LoggerFactory.getLogger(UserMongoDBAdaptor.class)); this.dbAdaptorFactory = dbAdaptorFactory; this.userCollection = userCollection; @@ -93,8 +111,7 @@ boolean exists(ClientSession clientSession, String userId) throws CatalogDBExcep } @Override - public OpenCGAResult insert(User user, String password, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult insert(User user, String password, QueryOptions options) throws CatalogException { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); @@ -117,7 +134,18 @@ private void insert(ClientSession clientSession, User user, String password) thr Document userDocument = userConverter.convertToStorageType(user); userDocument.append(ID, user.getId()); - userDocument.append(PRIVATE_PASSWORD, encryptPassword(password)); + + Document privatePassword = new Document(); + if (StringUtils.isNotEmpty(password)) { + String salt = PasswordUtils.getStrongRandomSalt(); + String hash = encryptPassword(password, salt); + Document passwordDoc = new Document() + .append(HASH, hash) + .append(SALT, salt); + privatePassword.put(CURRENT, passwordDoc); + privatePassword.put(ARCHIVE, Collections.singletonList(passwordDoc)); + } + userDocument.put(PRIVATE_PASSWORD, privatePassword); userCollection.insert(clientSession, userDocument, null); } @@ -130,47 +158,133 @@ public OpenCGAResult get(String userId, QueryOptions options) } @Override - public OpenCGAResult changePassword(String userId, String oldPassword, String newPassword) - throws CatalogDBException, CatalogAuthenticationException { - Document bson = new Document(ID, userId) - .append(PRIVATE_PASSWORD, encryptPassword(oldPassword)); - Bson set = Updates.set(PRIVATE_PASSWORD, encryptPassword(newPassword)); - - DataResult result = userCollection.update(bson, set, null); - if (result.getNumUpdated() == 0) { //0 query matches. - throw CatalogAuthenticationException.incorrectUserOrPassword("Internal"); - } - return new OpenCGAResult(result); + public OpenCGAResult changePassword(String userId, String oldPassword, String newPassword) throws CatalogException { + return setPassword(userId, oldPassword, newPassword); } @Override - public void authenticate(String userId, String password) throws CatalogAuthenticationException { - Document bson; - try { - bson = new Document() - .append(ID, userId) - .append(PRIVATE_PASSWORD, encryptPassword(password)); - } catch (CatalogDBException e) { - throw new CatalogAuthenticationException("Could not encrypt password: " + e.getMessage(), e); + public void authenticate(String userId, String password) throws CatalogDBException, CatalogAuthenticationException { + Bson query = Filters.and( + Filters.eq(QueryParams.ID.key(), userId), + // TODO: Deprecated. Remove Filters.or using the deprecated account authentication id + Filters.or( + Filters.eq(DEPRECATED_ACCOUNT_AUTHENTICATION_ID.key(), AuthenticationOrigin.AuthenticationType.OPENCGA), + Filters.eq(INTERNAL_ACCOUNT_AUTHENTICATION_ID.key(), AuthenticationOrigin.AuthenticationType.OPENCGA) + ) + ); + Bson projection = Projections.include(PRIVATE_PASSWORD); + DataResult dataResult = userCollection.find(query, projection, QueryOptions.empty()); + if (dataResult.getNumResults() == 0) { + throw new CatalogDBException("User " + userId + " not found"); + } + Document userDocument = dataResult.first(); + Object rootPasswordObject = userDocument.get(PRIVATE_PASSWORD); + Document rootPasswordDoc; + // TODO: Remove this block of code in the future when all users have been migrated + if (rootPasswordObject instanceof String) { + if (ParamConstants.OPENCGA_USER_ID.equals(userId)) { + logger.warn("User {} is using the deprecated password format. Please, migrate your code as soon as possible.", userId); + if (!encryptPassword(password, "").equals(rootPasswordObject)) { + throw CatalogAuthenticationException.incorrectUserOrPassword(AuthenticationOrigin.AuthenticationType.OPENCGA.name()); + } + return; + } else { + throw new CatalogDBException("User '" + userId + "' is using the deprecated password format. Please, ask your" + + " administrator to run the pending migrations to fix this issue."); + } + } else { + rootPasswordDoc = (Document) rootPasswordObject; + } + // TODO: End of block of code to remove (and replace using commented code below) +// Document rootPasswordDoc = userDocument.get(PRIVATE_PASSWORD, Document.class); + if (rootPasswordDoc == null) { + throw new CatalogDBException("Critical error. User '" + userId + "' does not have any password set. Please, contact" + + " with the developers."); } - if (userCollection.count(bson).getNumMatches() == 0) { - throw CatalogAuthenticationException.incorrectUserOrPassword("Internal"); + Document passwordDoc = rootPasswordDoc.get(CURRENT, Document.class); + if (passwordDoc == null) { + throw new CatalogDBException("Critical error. User '" + userId + "' does not have any password set. Please, contact" + + " with the developers."); + } + + String salt = passwordDoc.getString(SALT); + String hash = encryptPassword(password, salt); + if (!hash.equals(passwordDoc.getString(HASH))) { + throw CatalogAuthenticationException.incorrectUserOrPassword(AuthenticationOrigin.AuthenticationType.OPENCGA.name()); } } @Override - public OpenCGAResult resetPassword(String userId, String email, String newPassword) throws CatalogDBException { - Query query = new Query(QueryParams.ID.key(), userId); - query.append(QueryParams.EMAIL.key(), email); - Bson bson = parseQuery(query); + public OpenCGAResult resetPassword(String userId, String email, String newPassword) throws CatalogException { + return setPassword(userId, null, newPassword); + } - Bson set = Updates.set(PRIVATE_PASSWORD, encryptPassword(newPassword)); + public OpenCGAResult setPassword(String userId, @Nullable String oldPassword, String newPassword) throws CatalogException { + String prefixErrorMsg = "Could not update the password. "; + return runTransaction(clientSession -> { + // 1. Obtain archived passwords + Bson query = Filters.eq(QueryParams.ID.key(), userId); + Bson projection = Projections.include(PRIVATE_PASSWORD); + DataResult userQueryResult = userCollection.find(clientSession, query, projection, QueryOptions.empty()); + if (userQueryResult.getNumResults() == 0) { + throw new CatalogDBException(prefixErrorMsg + "User " + userId + " not found."); + } + Document userDoc = userQueryResult.first(); + Document passwordDoc = userDoc.get(PRIVATE_PASSWORD, Document.class); + + // 1.1. Check oldPassword + if (StringUtils.isNotEmpty(oldPassword)) { + Document currentPasswordDoc = passwordDoc.get(CURRENT, Document.class); + String currentSalt = currentPasswordDoc.getString(SALT); + String currentHash = encryptPassword(oldPassword, currentSalt); + if (!currentHash.equals(currentPasswordDoc.getString(HASH))) { + throw new CatalogAuthenticationException(prefixErrorMsg + "Please, verify that the current password is correct."); + } + } - DataResult result = userCollection.update(bson, set, null); - if (result.getNumUpdated() == 0) { //0 query matches. - throw new CatalogDBException("Bad user or email"); - } - return new OpenCGAResult(result); + // 2. Check new password has not been used before + for (Document document : passwordDoc.getList(ARCHIVE, Document.class)) { + String hashValue = document.getString(HASH); + String saltValue = document.getString(SALT); + String encryptedPassword = encryptPassword(newPassword, saltValue); + if (encryptedPassword.equals(hashValue)) { + throw new CatalogAuthenticationException(prefixErrorMsg + "The new password has already been used." + + " Please, use a different one."); + } + } + + // 3. Generate new salt for current password + String newSalt = PasswordUtils.getStrongRandomSalt(); + String newHash = encryptPassword(newPassword, newSalt); + + // 4. Generate update document + UpdateDocument updateDocument = new UpdateDocument(); + // add to current + updateDocument.getSet().put(PRIVATE_PASSWORD_CURRENT_HASH, newHash); + updateDocument.getSet().put(PRIVATE_PASSWORD_CURRENT_SALT, newSalt); + + // add to archive + Document document = new Document() + .append(HASH, newHash) + .append(SALT, newSalt); + updateDocument.getPush().put(PRIVATE_PASSWORD_ARCHIVE, document); + + updateDocument.getSet().put(INTERNAL_ACCOUNT_PASSWORD_LAST_MODIFIED.key(), TimeUtils.getTime()); + if (configuration.getAccount().getPasswordExpirationDays() > 0) { + Date date = TimeUtils.addDaysToCurrentDate(configuration.getAccount().getPasswordExpirationDays()); + String stringDate = TimeUtils.getTime(date); + updateDocument.getSet().put(INTERNAL_ACCOUNT_PASSWORD_EXPIRATION_DATE.key(), stringDate); + } + Document update = updateDocument.toFinalUpdateDocument(); + + logger.debug("Change password: query '{}'; update: '{}'", query.toBsonDocument(), update); + DataResult result = userCollection.update(clientSession, query, update, null); + if (result.getNumUpdated() == 0) { + throw new CatalogAuthenticationException("Could not update the password. Please, verify that the current password is" + + " correct."); + } + return new OpenCGAResult(result); + }, e -> logger.error("User {}: {}", userId, e.getMessage())); } @Override @@ -302,31 +416,14 @@ public OpenCGAResult stats(Query query) { public OpenCGAResult get(Query query, QueryOptions options) throws CatalogDBException { options = ParamUtils.defaultObject(options, QueryOptions::new); Bson bson = parseQuery(query); - QueryOptions userOptions; - if (includeProjects(options)) { - userOptions = filterQueryOptions(options, Arrays.asList(ID, PROJECTS_UID.key())); - } else { - userOptions = filterQueryOptions(options, Collections.singletonList(ID)); - } + QueryOptions userOptions = filterQueryOptionsToIncludeKeys(options, Collections.singletonList(ID)); DataResult userDataResult = userCollection.find(bson, null, userConverter, userOptions); - if (includeStudies(options)) { - for (User user : userDataResult.getResults()) { - if (user.getProjects() != null) { - for (Project project : user.getProjects()) { - Query query1 = new Query(StudyDBAdaptor.QueryParams.PROJECT_UID.key(), project.getUid()); - QueryOptions studyOptions = extractNestedOptions(options, PROJECTS.key() + ".studies."); - OpenCGAResult studyResult = dbAdaptorFactory.getCatalogStudyDBAdaptor().get(query1, studyOptions); - project.setStudies(studyResult.getResults()); - } - } - } - } - - if (includeSharedProjects(options)) { - QueryOptions sharedProjectOptions = extractNestedOptions(options, SHARED_PROJECTS.key()); - sharedProjectOptions = filterQueryOptions(sharedProjectOptions, Arrays.asList(ProjectDBAdaptor.QueryParams.FQN.key(), - "studies." + StudyDBAdaptor.QueryParams.FQN.key(), "studies." + StudyDBAdaptor.QueryParams.GROUPS.key())); + if (includeProjects(options)) { + QueryOptions sharedProjectOptions = extractNestedOptions(options, PROJECTS.key()); + sharedProjectOptions = filterQueryOptionsToIncludeKeys(sharedProjectOptions, + Arrays.asList(ProjectDBAdaptor.QueryParams.FQN.key(), "studies." + StudyDBAdaptor.QueryParams.FQN.key(), + "studies." + StudyDBAdaptor.QueryParams.GROUPS.key())); extractSharedProjects(userDataResult, sharedProjectOptions); } @@ -355,50 +452,6 @@ private boolean includeProjects(QueryOptions options) { return true; } - private boolean includeSharedProjects(QueryOptions options) { - List includeList = options.getAsStringList(QueryOptions.INCLUDE); - List excludeList = options.getAsStringList(QueryOptions.EXCLUDE); - - if (!includeList.isEmpty()) { - for (String includeKey : includeList) { - if (includeKey.startsWith(SHARED_PROJECTS.key() + ".")) { - return true; - } - } - return false; - } else if (!excludeList.isEmpty()) { - for (String excludeKey : excludeList) { - if (excludeKey.equals(SHARED_PROJECTS.key())) { - return false; - } - } - } - - return true; - } - - private boolean includeStudies(QueryOptions options) { - List includeList = options.getAsStringList(QueryOptions.INCLUDE); - List excludeList = options.getAsStringList(QueryOptions.EXCLUDE); - - if (!includeList.isEmpty()) { - for (String includeKey : includeList) { - if (includeKey.startsWith(PROJECTS.key() + ".studies.")) { - return true; - } - } - return false; - } else if (!excludeList.isEmpty()) { - for (String excludeKey : excludeList) { - if (excludeKey.equals(PROJECTS.key() + ".studies")) { - return false; - } - } - } - - return true; - } - private void extractSharedProjects(DataResult userDataResult, QueryOptions options) throws CatalogDBException { Set users = userDataResult.getResults().stream().map(User::getId).collect(Collectors.toSet()); @@ -406,7 +459,7 @@ private void extractSharedProjects(DataResult userDataResult, QueryOptions Map studyMap = new HashMap<>(); Map studyProjectMap = new HashMap<>(); Map> userStudyMap = new HashMap<>(); - OpenCGAResult result = dbAdaptorFactory.getCatalogProjectDbAdaptor().get(new Query(), options); + OpenCGAResult result = dbAdaptorFactory.getCatalogProjectDBAdaptor().get(new Query(), options); for (Project project : result.getResults()) { projectMap.put(project.getFqn(), project); if (project.getStudies() != null) { @@ -414,21 +467,16 @@ private void extractSharedProjects(DataResult userDataResult, QueryOptions studyMap.put(study.getFqn(), study); studyProjectMap.put(study.getFqn(), project.getFqn()); - String owner = study.getFqn().split("@")[0]; - if (study.getGroups() != null) { for (Group group : study.getGroups()) { if (StudyManager.MEMBERS.equals(group.getId())) { // Add all the users that should be able to see the study to the map for (String userId : group.getUserIds()) { - // Exclude owner of the project - if (!owner.equals(userId)) { - if (users.contains(userId)) { - if (!userStudyMap.containsKey(userId)) { - userStudyMap.put(userId, new ArrayList<>()); - } - userStudyMap.get(userId).add(study.getFqn()); + if (users.contains(userId)) { + if (!userStudyMap.containsKey(userId)) { + userStudyMap.put(userId, new ArrayList<>()); } + userStudyMap.get(userId).add(study.getFqn()); } } break; @@ -439,7 +487,7 @@ private void extractSharedProjects(DataResult userDataResult, QueryOptions } } - // Add SharedProject information + // Add project information for (User user : userDataResult.getResults()) { if (userStudyMap.containsKey(user.getId())) { Map> projectStudyMap = new HashMap<>(); @@ -460,7 +508,7 @@ private void extractSharedProjects(DataResult userDataResult, QueryOptions projectList.add(project); } - user.setSharedProjects(projectList); + user.setProjects(projectList); } } } @@ -475,12 +523,12 @@ public OpenCGAResult nativeGet(Query query, QueryOptions options) throws Catalog for (Document user : queryResult.getResults()) { ArrayList projects = (ArrayList) user.get("projects"); - if (projects.size() > 0) { + if (CollectionUtils.isNotEmpty(projects)) { List projectsTmp = new ArrayList<>(projects.size()); for (Document project : projects) { Query query1 = new Query(ProjectDBAdaptor.QueryParams.UID.key(), project.get(ProjectDBAdaptor .QueryParams.UID.key())); - OpenCGAResult queryResult1 = dbAdaptorFactory.getCatalogProjectDbAdaptor().nativeGet(query1, options); + OpenCGAResult queryResult1 = dbAdaptorFactory.getCatalogProjectDBAdaptor().nativeGet(query1, options); projectsTmp.add(queryResult1.first()); } user.remove("projects"); @@ -493,27 +541,35 @@ public OpenCGAResult nativeGet(Query query, QueryOptions options) throws Catalog @Override public OpenCGAResult update(Query query, ObjectMap parameters, QueryOptions queryOptions) throws CatalogDBException { - Map userParameters = new HashMap<>(); + UpdateDocument document = new UpdateDocument(); - final String[] acceptedParams = {QueryParams.NAME.key(), QueryParams.EMAIL.key(), QueryParams.ORGANIZATION.key()}; - filterStringParams(parameters, userParameters, acceptedParams); + final String[] acceptedParams = {QueryParams.NAME.key(), QueryParams.EMAIL.key(), INTERNAL_ACCOUNT_EXPIRATION_DATE.key()}; + filterStringParams(parameters, document.getSet(), acceptedParams); if (parameters.containsKey(QueryParams.INTERNAL_STATUS_ID.key())) { - userParameters.put(QueryParams.INTERNAL_STATUS_ID.key(), parameters.get(QueryParams.INTERNAL_STATUS_ID.key())); - userParameters.put(QueryParams.INTERNAL_STATUS_DATE.key(), TimeUtils.getTime()); + document.getSet().put(QueryParams.INTERNAL_STATUS_ID.key(), parameters.get(QueryParams.INTERNAL_STATUS_ID.key())); + document.getSet().put(QueryParams.INTERNAL_STATUS_DATE.key(), TimeUtils.getTime()); } - final String[] acceptedLongParams = {QueryParams.QUOTA.key(), QueryParams.SIZE.key()}; - filterLongParams(parameters, userParameters, acceptedLongParams); + final String[] acceptedIntParams = {INTERNAL_ACCOUNT_FAILED_ATTEMPTS.key()}; + filterIntParams(parameters, document.getSet(), acceptedIntParams); + + final String[] acceptedObjectParams = {QueryParams.QUOTA.key()}; + filterObjectParams(parameters, document.getSet(), acceptedObjectParams); final String[] acceptedMapParams = {QueryParams.ATTRIBUTES.key()}; - filterMapParams(parameters, userParameters, acceptedMapParams); + filterMapParams(parameters, document.getSet(), acceptedMapParams); + + if (!document.toFinalUpdateDocument().isEmpty()) { + document.getSet().put(INTERNAL_LAST_MODIFIED, TimeUtils.getTime()); + } - if (!userParameters.isEmpty()) { - return new OpenCGAResult(userCollection.update(parseQuery(query), new Document("$set", userParameters), null)); + Document userUpdate = document.toFinalUpdateDocument(); + if (userUpdate.isEmpty()) { + throw new CatalogDBException("Nothing to be updated."); } - return OpenCGAResult.empty(); + return new OpenCGAResult(userCollection.update(parseQuery(query), userUpdate, null)); } @Override @@ -542,11 +598,7 @@ public OpenCGAResult update(String userId, ObjectMap parameters) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { checkId(userId); Query query = new Query(QueryParams.ID.key(), userId); - OpenCGAResult update = update(query, parameters, QueryOptions.empty()); - if (update.getNumUpdated() != 1) { - throw new CatalogDBException("Could not update user " + userId); - } - return update; + return update(query, parameters, QueryOptions.empty()); } OpenCGAResult setStatus(Query query, String status) throws CatalogDBException { @@ -578,37 +630,10 @@ public OpenCGAResult delete(String id, QueryOptions queryOptions) throw new CatalogDBException("The user {" + id + "} was already " + user.getInternal().getStatus().getId()); } - // If we don't find the force parameter, we check first if the user does not have an active project. - if (!queryOptions.containsKey(FORCE) || !queryOptions.getBoolean(FORCE)) { - checkCanDelete(id); - } - - if (queryOptions.containsKey(FORCE) && queryOptions.getBoolean(FORCE)) { - // Delete the active projects (if any) - query = new Query(ProjectDBAdaptor.QueryParams.USER_ID.key(), id); - dbAdaptorFactory.getCatalogProjectDbAdaptor().delete(query, queryOptions); - } - // Change the status of the user to deleted return setStatus(id, UserStatus.DELETED); } - /** - * Checks whether the userId has any active project. - * - * @param userId user id. - * @throws CatalogDBException when the user has active projects. Projects must be deleted first. - */ - private void checkCanDelete(String userId) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { - checkId(userId); - Query query = new Query(ProjectDBAdaptor.QueryParams.USER_ID.key(), userId) - .append(ProjectDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), InternalStatus.READY); - Long count = dbAdaptorFactory.getCatalogProjectDbAdaptor().count(query).getNumMatches(); - if (count > 0) { - throw new CatalogDBException("The user {" + userId + "} cannot be deleted. The user has " + count + " projects in use."); - } - } - @Override public OpenCGAResult delete(Query query, QueryOptions queryOptions) throws CatalogDBException { throw new UnsupportedOperationException("Remove not yet implemented."); @@ -707,18 +732,10 @@ public void forEach(Query query, Consumer action, QueryOptions o } } - public static void main(String[] args) throws CatalogDBException { - System.out.println(encryptPassword("admin")); - } - - private static String encryptPassword(String password) throws CatalogDBException { + private static String encryptPassword(String password, String salt) throws CatalogDBException { if (StringUtils.isNotEmpty(password)) { - if (password.matches("^[a-fA-F0-9]{40}$")) { - // Password already cyphered - return password; - } try { - return CryptoUtils.sha1(password); + return CryptoUtils.sha1(password + salt); } catch (NoSuchAlgorithmException e) { throw new CatalogDBException("Could not encrypt password", e); } @@ -732,8 +749,6 @@ private Bson parseQuery(Query query) throws CatalogDBException { List andBsonList = new ArrayList<>(); fixComplexQueryParam(QueryParams.ATTRIBUTES.key(), query); - fixComplexQueryParam(QueryParams.BATTRIBUTES.key(), query); - fixComplexQueryParam(QueryParams.NATTRIBUTES.key(), query); for (Map.Entry entry : query.entrySet()) { String key = entry.getKey().split("\\.")[0]; @@ -751,14 +766,6 @@ private Bson parseQuery(Query query) throws CatalogDBException { case ATTRIBUTES: addAutoOrQuery(entry.getKey(), entry.getKey(), query, queryParam.type(), andBsonList); break; - case BATTRIBUTES: - String mongoKey = entry.getKey().replace(QueryParams.BATTRIBUTES.key(), QueryParams.ATTRIBUTES.key()); - addAutoOrQuery(mongoKey, entry.getKey(), query, queryParam.type(), andBsonList); - break; - case NATTRIBUTES: - mongoKey = entry.getKey().replace(QueryParams.NATTRIBUTES.key(), QueryParams.ATTRIBUTES.key()); - addAutoOrQuery(mongoKey, entry.getKey(), query, queryParam.type(), andBsonList); - break; case INTERNAL_STATUS_ID: // Convert the status to a positive status query.put(queryParam.key(), @@ -769,17 +776,8 @@ private Bson parseQuery(Query query) throws CatalogDBException { case EMAIL: case ORGANIZATION: case INTERNAL_STATUS_DATE: - case SIZE: - case QUOTA: - case ACCOUNT_TYPE: - case ACCOUNT_AUTHENTICATION_ID: - case ACCOUNT_CREATION_DATE: - case PROJECTS: - case PROJECTS_UID: - case PROJECT_NAME: - case PROJECTS_ID: - case PROJECT_ORGANIZATION: - case PROJECT_STATUS: + case INTERNAL_ACCOUNT_AUTHENTICATION_ID: + case INTERNAL_ACCOUNT_CREATION_DATE: case TOOL_ID: case TOOL_NAME: case TOOL_ALIAS: diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/VersionedMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/VersionedMongoDBAdaptor.java index 11318fe3fd9..c50446677f7 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/VersionedMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/VersionedMongoDBAdaptor.java @@ -50,36 +50,15 @@ public VersionedMongoDBAdaptor(MongoDBCollection collection, MongoDBCollection a * @return a boolean indicating whether the complex query was generated or not. * @throws CatalogDBException If the size of the array of ids does not match the size of the array of version. */ - boolean generateUidVersionQuery(Query query, List bsonQueryList) throws CatalogDBException { + boolean generateIdVersionQuery(Query query, List bsonQueryList) throws CatalogDBException { if (!query.containsKey(VERSION) || query.getAsIntegerList(VERSION).size() == 1) { return false; } - if (!query.containsKey(PRIVATE_UID) && !query.containsKey(ID) && !query.containsKey(PRIVATE_UUID)) { + if (!query.containsKey(ID)) { return false; } - int numIds = 0; - numIds += query.containsKey(ID) ? 1 : 0; - numIds += query.containsKey(PRIVATE_UID) ? 1 : 0; - numIds += query.containsKey(PRIVATE_UUID) ? 1 : 0; - - if (numIds > 1) { - List versionList = query.getAsIntegerList(VERSION); - if (versionList.size() > 1) { - throw new CatalogDBException("Cannot query by more than one version when more than one id type is being queried"); - } - return false; - } - - String idQueried = PRIVATE_UID; - idQueried = query.containsKey(ID) ? ID : idQueried; - idQueried = query.containsKey(PRIVATE_UUID) ? PRIVATE_UUID : idQueried; - List idList; - if (PRIVATE_UID.equals(idQueried)) { - idList = query.getAsLongList(PRIVATE_UID); - } else { - idList = query.getAsStringList(idQueried); - } + List idList = query.getAsStringList(ID); List versionList = query.getAsIntegerList(VERSION); if (versionList.size() > 1 && idList.size() > 1 && versionList.size() != idList.size()) { @@ -90,9 +69,9 @@ boolean generateUidVersionQuery(Query query, List bsonQueryList) throws Ca for (int i = 0; i < versionList.size(); i++) { Document docQuery = new Document(VERSION, versionList.get(i)); if (idList.size() == 1) { - docQuery.put(idQueried, idList.get(0)); + docQuery.put(ID, idList.get(0)); } else { - docQuery.put(idQueried, idList.get(i)); + docQuery.put(ID, idList.get(i)); } bsonQuery.add(docQuery); } @@ -100,7 +79,7 @@ boolean generateUidVersionQuery(Query query, List bsonQueryList) throws Ca if (!bsonQuery.isEmpty()) { bsonQueryList.add(Filters.or(bsonQuery)); - query.remove(idQueried); + query.remove(ID); query.remove(VERSION); return true; @@ -118,7 +97,7 @@ private String getClientSessionUuid(ClientSession session) { } public interface VersionedModelExecution { - T execute(List entries) throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException; + T execute(List entryList) throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException; } public interface NonVersionedModelExecution { @@ -138,10 +117,23 @@ DBIterator iterator(ClientSession session, Query query, QueryOptions options) protected void insert(ClientSession session, Document document) { String uuid = getClientSessionUuid(session); document.put(PRIVATE_TRANSACTION_ID, uuid); + document.put(VERSION, 1); + document.put(LAST_OF_VERSION, true); collection.insert(session, document, QueryOptions.empty()); archiveCollection.insert(session, document, QueryOptions.empty()); } + protected OpenCGAResult update(ClientSession session, Bson sourceQuery, VersionedModelExecution> update) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + return update(session, sourceQuery, Collections.emptyList(), update, Collections.emptyList(), null, null); + } + + protected OpenCGAResult update(ClientSession session, Bson sourceQuery, List includeFields, + VersionedModelExecution> update) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + return update(session, sourceQuery, includeFields, update, Collections.emptyList(), null, null); + } + protected OpenCGAResult update(ClientSession session, Bson sourceQuery, VersionedModelExecution> update, PostVersionIncrementIterator postVersionIncrementIterator, ReferenceModelExecution postVersionIncrementExecution) @@ -168,7 +160,7 @@ protected OpenCGAResult update(ClientSession session, Bson sourceQuery, L // 1. Increment version // 1.1 Only increase version of those documents not already increased by same transaction id - Set includeFields = new HashSet<>(Arrays.asList(PRIVATE_UID, VERSION, RELEASE_FROM_VERSION, PRIVATE_TRANSACTION_ID)); + Set includeFields = new HashSet<>(Arrays.asList(PRIVATE_UID, VERSION, PRIVATE_TRANSACTION_ID)); if (fieldsToInclude != null) { includeFields.addAll(fieldsToInclude); } @@ -179,6 +171,7 @@ protected OpenCGAResult update(ClientSession session, Bson sourceQuery, L try (MongoDBIterator iterator = collection.iterator(session, sourceQuery, null, null, options)) { while (iterator.hasNext()) { Document result = iterator.next(); + entryList.add(result); entryList.add(result); @@ -193,7 +186,7 @@ protected OpenCGAResult update(ClientSession session, Bson sourceQuery, L Document collectionUpdate = new Document(); Document archiveCollectionUpdate = new Document(); - processReleaseSnapshotChanges(result, collectionUpdate, archiveCollectionUpdate); + processLastOfVersionChanges(result, collectionUpdate, archiveCollectionUpdate); Bson bsonQuery = Filters.and( Filters.eq(PRIVATE_UID, uid), @@ -239,7 +232,7 @@ protected OpenCGAResult update(ClientSession session, Bson sourceQuery, L // Insert/replace in archive collection Bson tmpBsonQuery = Filters.and( - Filters.eq(PRIVATE_UID, fixedResult.get(PRIVATE_UID)), + Filters.eq(ID, fixedResult.get(ID)), Filters.eq(VERSION, fixedResult.get(VERSION)) ); logger.debug("Copying current document to archive: query : {}", tmpBsonQuery.toBsonDocument()); @@ -251,7 +244,7 @@ protected OpenCGAResult update(ClientSession session, Bson sourceQuery, L if (!uidsChanged.isEmpty()) { Query query = new Query(PRIVATE_UID, uidsChanged); if (postVersionIncrementExecution != null) { - List includeList = new ArrayList<>(Arrays.asList(PRIVATE_UID, PRIVATE_UUID, ID, PRIVATE_STUDY_UID, VERSION)); + List includeList = new ArrayList<>(Arrays.asList(ID, VERSION)); if (postVersionIncrementExecution != null) { includeList.addAll(postVersionIncrementAdditionalIncludeFields); } @@ -263,9 +256,6 @@ protected OpenCGAResult update(ClientSession session, Bson sourceQuery, L try (DBIterator iterator = dbIterator.iterator(session, query, options)) { postVersionIncrementExecution.execute(iterator); } -// try (MongoDBIterator iterator = collection.iterator(session, query, null, null, options)) { -// postVersionIncrementExecution.execute(iterator); -// } } } @@ -289,7 +279,7 @@ protected T updateWithoutVersionIncrement(Bson sourceQuery, NonVersionedMode // Insert/replace in archive collection Bson tmpBsonQuery = Filters.and( - Filters.eq(PRIVATE_UID, result.get(PRIVATE_UID)), + Filters.eq(ID, result.get(ID)), Filters.eq(VERSION, result.get(VERSION)) ); archiveCollection.update(tmpBsonQuery, result, upsertOptions); @@ -303,14 +293,14 @@ protected T updateWithoutVersionIncrement(Bson sourceQuery, NonVersionedMode * Revert to a previous version. * * @param clientSession ClientSession for transactional operations. - * @param uid UID of the element to be recovered. + * @param id ID of the element to be recovered. * @param version Version to be recovered. * @return the new latest document that will be written in the database. * @throws CatalogDBException in case of any issue. */ - protected Document revertToVersion(ClientSession clientSession, long uid, int version) throws CatalogDBException { + protected Document revertToVersion(ClientSession clientSession, String id, int version) throws CatalogDBException { Bson query = Filters.and( - Filters.eq(PRIVATE_UID, uid), + Filters.eq(ID, id), Filters.eq(VERSION, version) ); DataResult result = archiveCollection.find(clientSession, query, EXCLUDE_MONGO_ID); @@ -319,28 +309,26 @@ protected Document revertToVersion(ClientSession clientSession, long uid, int ve } Document document = result.first(); - QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(RELEASE_FROM_VERSION, VERSION, LAST_OF_RELEASE)); + QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, VERSION); // Find out latest version available - result = collection.find(clientSession, Filters.eq(PRIVATE_UID, uid), options); + result = collection.find(clientSession, Filters.eq(ID, id), options); if (result.getNumResults() == 0) { - throw new CatalogDBException("Unexpected error. Could not find 'uid': " + uid); + throw new CatalogDBException("Unexpected error. Could not find 'id': " + id); } int lastVersion = result.first().getInteger(VERSION); // Delete previous version from active collection - collection.remove(clientSession, Filters.eq(PRIVATE_UID, uid), QueryOptions.empty()); + collection.remove(clientSession, Filters.eq(ID, id), QueryOptions.empty()); // Edit previous version from archive collection query = Filters.and( - Filters.eq(PRIVATE_UID, uid), + Filters.eq(ID, id), Filters.eq(VERSION, lastVersion) ); - archiveCollection.update(clientSession, query, Updates.set(LAST_OF_RELEASE, false), QueryOptions.empty()); + archiveCollection.update(clientSession, query, Updates.set(LAST_OF_VERSION, false), QueryOptions.empty()); // Edit private fields from document to be restored document.put(VERSION, lastVersion + 1); - document.put(RELEASE_FROM_VERSION, result.first().get(RELEASE_FROM_VERSION)); - document.put(LAST_OF_RELEASE, result.first().get(LAST_OF_RELEASE)); // Add restored element to main and archive collection collection.insert(clientSession, document, QueryOptions.empty()); @@ -366,7 +354,9 @@ protected void delete(ClientSession session, Bson query) { } for (Document document : archiveCollection.find(session, query, QueryOptions.empty()).getResults()) { Document internal = document.get("internal", Document.class); - internal.put("status", status); + if (internal != null) { + internal.put("status", status); + } deletedCollection.insert(session, document, QueryOptions.empty()); } @@ -376,38 +366,20 @@ protected void delete(ClientSession session, Bson query) { } /** - * Given the current document, it puts in collectionUpdate and archiveCollectionUpdate the changes that need to be applied. - * It takes care of the private fields that allow performing queries by snapshot: - * LAST_OF_VERSION, LAST_OF_RELEASE, RELEASE_FROM_VERSION + * Given the current document, it writes the changes that need to be applied in collectionUpdate and archiveCollectionUpdate. + * It takes care of the private fields: LAST_OF_VERSION * * @param document Current document. * @param collectionUpdate Empty document where we will put the changes to be applied to the main collection. * @param archiveCollectionUpdate Empty document where we will put the changes to be applied to the archive collection. */ - private void processReleaseSnapshotChanges(Document document, Document collectionUpdate, Document archiveCollectionUpdate) { + private void processLastOfVersionChanges(Document document, Document collectionUpdate, Document archiveCollectionUpdate) { int version = document.get(VERSION, Number.class).intValue(); - List releaseFromVersion = document.getList(RELEASE_FROM_VERSION, Integer.class); - - // Current release number - int release; - if (releaseFromVersion.size() > 1) { - release = releaseFromVersion.get(releaseFromVersion.size() - 1); - - // If it contains several releases, it means this is the first update on the current release, so we just need to take the - // current release number out - releaseFromVersion.remove(releaseFromVersion.size() - 1); - } else { - release = releaseFromVersion.get(0); - - // If it is 1, it means that the previous version being checked was made on this same release as well, so it won't be the - // last version of the release - archiveCollectionUpdate.put(LAST_OF_RELEASE, false); - } - archiveCollectionUpdate.put(RELEASE_FROM_VERSION, releaseFromVersion); + + // And set the document in archive with the flag LAST_OF_VERSION to false archiveCollectionUpdate.put(LAST_OF_VERSION, false); // We update the information for the new version of the document - collectionUpdate.put(RELEASE_FROM_VERSION, Collections.singletonList(release)); collectionUpdate.put(VERSION, version + 1); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/AnnotationConverter.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/AnnotationConverter.java index abf9dc3be11..310c86df19c 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/AnnotationConverter.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/AnnotationConverter.java @@ -177,8 +177,7 @@ public List fromDBToAnnotation(List annotationList, Doc String variableSetId = variableSetUidIdMap.getString(String.valueOf(variableSetUid)); if (!annotationSetMap.containsKey(compoundKey)) { - annotationSetMap.put(compoundKey, new AnnotationSet(annSetName, variableSetId, new HashMap<>(), - Collections.emptyMap())); + annotationSetMap.put(compoundKey, new AnnotationSet(annSetName, variableSetId, new HashMap<>())); } annotationSetMap.get(compoundKey).getAnnotations().put(annotationDocument.getString(ID), annotationDocument.get(VALUE)); @@ -203,8 +202,7 @@ public List fromDBToAnnotation(List annotationList, Doc String variableSetId = variableSetUidIdMap.getString(String.valueOf(variableSetUid)); if (!annotationSetMap.containsKey(compoundKey)) { - annotationSetMap.put(compoundKey, new AnnotationSet(annSetName, variableSetId, new HashMap<>(), - Collections.emptyMap())); + annotationSetMap.put(compoundKey, new AnnotationSet(annSetName, variableSetId, new HashMap<>())); } // We provide the map from the annotation set in order to be automatically filled in diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/FamilyConverter.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/FamilyConverter.java index 16372fe2db9..7af75fa5e08 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/FamilyConverter.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/FamilyConverter.java @@ -16,17 +16,16 @@ package org.opencb.opencga.catalog.db.mongodb.converters; +import org.apache.commons.collections4.CollectionUtils; import org.bson.Document; import org.opencb.biodata.models.clinical.Phenotype; -import org.opencb.commons.utils.ListUtils; import org.opencb.opencga.catalog.db.api.FamilyDBAdaptor; import org.opencb.opencga.core.models.family.Family; -import org.opencb.opencga.core.models.individual.Individual; import org.opencb.opencga.core.models.study.VariableSet; -import java.util.HashMap; +import java.util.HashSet; import java.util.List; -import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; /** @@ -55,40 +54,41 @@ public Document convertToStorageType(Family object, List variableSe } public void validateDocumentToUpdate(Document document) { - List memberList = (List) document.get("members"); + List memberList = document.getList("members", Document.class); if (memberList != null) { // We make sure we don't store duplicates - Map individualMap = new HashMap<>(); + Set individualSet = new HashSet<>(); for (Document individual : memberList) { - long id = individual.getInteger("uid").longValue(); - int version = individual.getInteger("version"); - if (id > 0) { - Individual tmpIndividual = new Individual() - .setVersion(version); - tmpIndividual.setUid(id); - individualMap.put(id, tmpIndividual); + long uid = individual.get("uid", Number.class).longValue(); + if (uid <= 0) { + throw new IllegalArgumentException("Missing uid value for member '" + individual.getString("id") + "'."); + } + boolean unique = individualSet.add(uid); + if (!unique) { + throw new IllegalArgumentException("Duplicated member '" + individual.getString("id") + " (" + uid + ")' found."); } } document.put("members", - individualMap.entrySet().stream() + memberList.stream() .map(entry -> new Document() - .append("uid", entry.getValue().getUid()) - .append("version", entry.getValue().getVersion())) + .append("uid", entry.get("uid", Number.class).longValue()) + .append("version", entry.get("version", Number.class).intValue()) + ) .collect(Collectors.toList())); } - List disorderList = (List) document.get("disorders"); + List disorderList = document.getList("disorders", Document.class); if (disorderList != null) { for (Document disorder : disorderList) { - fixPhenotypeFields((List) disorder.get("evidences")); + fixPhenotypeFields(disorder.getList("evidences", Document.class)); } } - fixPhenotypeFields((List) document.get("phenotypes")); + fixPhenotypeFields(document.getList("phenotypes", Document.class)); } private void fixPhenotypeFields(List phenotypeList) { - if (ListUtils.isNotEmpty(phenotypeList)) { + if (CollectionUtils.isNotEmpty(phenotypeList)) { for (Document phenotype : phenotypeList) { phenotype.put("status", Phenotype.Status.UNKNOWN.name()); phenotype.put("ageOfOnset", "-1"); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/MigrationConverter.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/MigrationConverter.java index ea25a4dd106..cf9fbb78350 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/MigrationConverter.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/MigrationConverter.java @@ -1,7 +1,7 @@ package org.opencb.opencga.catalog.db.mongodb.converters; import org.bson.Document; -import org.opencb.opencga.catalog.migration.MigrationRun; +import org.opencb.opencga.core.models.migration.MigrationRun; public class MigrationConverter extends OpenCgaMongoConverter { diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/NoteConverter.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/NoteConverter.java new file mode 100644 index 00000000000..52c10756f2f --- /dev/null +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/NoteConverter.java @@ -0,0 +1,11 @@ +package org.opencb.opencga.catalog.db.mongodb.converters; + +import org.opencb.opencga.core.models.notes.Note; + +public class NoteConverter extends OpenCgaMongoConverter { + + public NoteConverter() { + super(Note.class); + } + +} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/OrganizationConverter.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/OrganizationConverter.java new file mode 100644 index 00000000000..6e8629be6d8 --- /dev/null +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/OrganizationConverter.java @@ -0,0 +1,34 @@ +package org.opencb.opencga.catalog.db.mongodb.converters; + +import org.apache.avro.generic.GenericRecord; +import org.bson.Document; +import org.opencb.opencga.catalog.db.mongodb.MongoDBUtils; +import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.core.config.AuthenticationOrigin; +import org.opencb.opencga.core.models.common.mixins.GenericRecordAvroJsonMixin; +import org.opencb.opencga.core.models.organizations.Organization; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class OrganizationConverter extends OpenCgaMongoConverter { + + public OrganizationConverter() { + super(Organization.class); + getObjectMapper().addMixIn(GenericRecord.class, GenericRecordAvroJsonMixin.class); + } + + public List convertAuthenticationOrigins(List authenticationOriginList) throws CatalogDBException { + if (authenticationOriginList == null || authenticationOriginList.isEmpty()) { + return Collections.emptyList(); + } + List authenticationOriginDocumentList = new ArrayList<>(authenticationOriginList.size()); + for (AuthenticationOrigin authenticationOrigin : authenticationOriginList) { + Document authOriginDocument = MongoDBUtils.getMongoDBDocument(authenticationOrigin, "AuthenticationOrigin"); + authenticationOriginDocumentList.add(authOriginDocument); + } + return authenticationOriginDocumentList; + + } +} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/UserConverter.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/UserConverter.java index 2aef9acf6fa..837349b6b8a 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/UserConverter.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/UserConverter.java @@ -18,36 +18,77 @@ import org.apache.avro.generic.GenericRecord; import org.bson.Document; +import org.opencb.opencga.catalog.db.api.UserDBAdaptor; import org.opencb.opencga.core.models.common.mixins.GenericRecordAvroJsonMixin; import org.opencb.opencga.core.models.user.User; +import org.opencb.opencga.core.models.user.UserInternal; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Created by pfurio on 19/01/16. */ public class UserConverter extends OpenCgaMongoConverter { + protected static Logger logger = LoggerFactory.getLogger(UserConverter.class); + public UserConverter() { super(User.class); getObjectMapper().addMixIn(GenericRecord.class, GenericRecordAvroJsonMixin.class); } + @Override + public Document convertToStorageType(User object) { + Document userDocument = super.convertToStorageType(object); + removeDeprecatedAccountObject(userDocument); + return userDocument; + } + @Override public User convertToDataModelType(Document document) { - // TODO: Remove this piece of code once we are sure User contains the migrated new account type from 1.4.2 - Document account = (Document) document.get("account"); - if (account != null && account.get("authentication") == null) { - // We make sure type is in upper case because we are now storing the enum names - String type = account.getString("type"); - account.put("type", type.toUpperCase()); - - String authOrigin = account.getString("authOrigin"); - Document authentication = new Document() - .append("id", authOrigin) - .append("application", false); - account.put("authentication", authentication); + User user = super.convertToDataModelType(document); + + restoreFromDeprecatedAccountObject(user); + addToDeprecatedAccountObject(user); + + return user; + } + + /** + * Remove 'account' object from the User document so it is no longer stored in the database. + * Remove after a few releases. + * + * @param userDocument User document. + */ + @Deprecated + private void removeDeprecatedAccountObject(Document userDocument) { + userDocument.remove(UserDBAdaptor.QueryParams.DEPRECATED_ACCOUNT.key()); + } + + /** + * Restores information from the account object to the corresponding internal.account object. + * Added to maintain backwards compatibility with the deprecated account object in TASK-6494 (v3.2.1) + * Remove after a few releases. + * + * @param user User object. + */ + @Deprecated + private void restoreFromDeprecatedAccountObject(User user) { + if (user.getAccount() != null) { + if (user.getInternal() == null) { + user.setInternal(new UserInternal()); + } + user.getInternal().setAccount(user.getAccount()); + logger.warn("Restoring user account information from deprecated account object to internal.account object. " + + "Please, run 'opencga-admin.sh migration run'."); } + } - return super.convertToDataModelType(document); + private void addToDeprecatedAccountObject(User user) { + // Add account to deprecated place + if (user.getInternal() != null && user.getInternal().getAccount() != null && user.getAccount() == null) { + user.setAccount(user.getInternal().getAccount()); + } } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/iterators/CatalogMongoDBIterator.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/iterators/CatalogMongoDBIterator.java index 50666864c1d..2389fb2db61 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/iterators/CatalogMongoDBIterator.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/iterators/CatalogMongoDBIterator.java @@ -25,10 +25,7 @@ import org.opencb.opencga.catalog.db.api.DBIterator; import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptor; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.function.UnaryOperator; import static org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptor.NATIVE_QUERY; @@ -185,4 +182,30 @@ protected QueryOptions createInnerQueryOptionsForVersionedEntity(QueryOptions op return queryOptions; } + protected boolean includeField(QueryOptions options, List fields) { + if (options.containsKey(QueryOptions.INCLUDE)) { + List currentIncludeList = options.getAsStringList(QueryOptions.INCLUDE); + for (String include : currentIncludeList) { + for (String field : fields) { + if (include.startsWith(field)) { + return true; + } + } + } + return false; + } else if (options.containsKey(QueryOptions.EXCLUDE)) { + List currentExcludeList = options.getAsStringList(QueryOptions.EXCLUDE); + for (String exclude : currentExcludeList) { + for (String field : fields) { + if (exclude.equals(field)) { + return false; + } + } + + } + return true; + } + return true; + } + } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/iterators/ClinicalAnalysisCatalogMongoDBIterator.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/iterators/ClinicalAnalysisCatalogMongoDBIterator.java index fa4ead98ef1..12565f22eba 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/iterators/ClinicalAnalysisCatalogMongoDBIterator.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/iterators/ClinicalAnalysisCatalogMongoDBIterator.java @@ -69,15 +69,15 @@ public class ClinicalAnalysisCatalogMongoDBIterator extends AnnotableCatalogM public ClinicalAnalysisCatalogMongoDBIterator(MongoDBIterator mongoCursor, ClientSession clientSession, AnnotableConverter converter, UnaryOperator filter, - MongoDBAdaptorFactory dbAdaptorFactory, QueryOptions options) { + OrganizationMongoDBAdaptorFactory dbAdaptorFactory, QueryOptions options) { this(mongoCursor, clientSession, converter, filter, dbAdaptorFactory, 0, null, options); } public ClinicalAnalysisCatalogMongoDBIterator(MongoDBIterator mongoCursor, ClientSession clientSession, AnnotableConverter converter, UnaryOperator filter, - MongoDBAdaptorFactory dbAdaptorFactory, long studyUid, String user, + OrganizationMongoDBAdaptorFactory dbAdaptorFactory, long studyUid, String user, QueryOptions options) { - super(mongoCursor, clientSession, converter, filter, options); + super(mongoCursor, clientSession, converter, filter, null); this.user = user; this.studyUid = studyUid; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/iterators/IndividualCatalogMongoDBIterator.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/iterators/IndividualCatalogMongoDBIterator.java index ef3cbd0d10e..cac67799a83 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/iterators/IndividualCatalogMongoDBIterator.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/iterators/IndividualCatalogMongoDBIterator.java @@ -26,7 +26,7 @@ import org.opencb.opencga.catalog.db.api.IndividualDBAdaptor; import org.opencb.opencga.catalog.db.api.SampleDBAdaptor; import org.opencb.opencga.catalog.db.mongodb.IndividualMongoDBAdaptor; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; import org.opencb.opencga.catalog.db.mongodb.SampleMongoDBAdaptor; import org.opencb.opencga.catalog.db.mongodb.converters.AnnotableConverter; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; @@ -51,7 +51,6 @@ public class IndividualCatalogMongoDBIterator extends AnnotableCatalogMongoDB private QueryOptions sampleQueryOptions; private IndividualMongoDBAdaptor individualDBAdaptor; - private Queue individualListBuffer; private Logger logger; @@ -60,13 +59,14 @@ public class IndividualCatalogMongoDBIterator extends AnnotableCatalogMongoDB public IndividualCatalogMongoDBIterator(MongoDBIterator mongoCursor, ClientSession clientSession, AnnotableConverter converter, UnaryOperator filter, - MongoDBAdaptorFactory dbAdaptorFactory, QueryOptions options) { + OrganizationMongoDBAdaptorFactory dbAdaptorFactory, QueryOptions options) { this(mongoCursor, clientSession, converter, filter, dbAdaptorFactory, 0, null, options); } public IndividualCatalogMongoDBIterator(MongoDBIterator mongoCursor, ClientSession clientSession, AnnotableConverter converter, UnaryOperator filter, - MongoDBAdaptorFactory dbAdaptorFactory, long studyUid, String user, QueryOptions options) { + OrganizationMongoDBAdaptorFactory dbAdaptorFactory, long studyUid, String user, + QueryOptions options) { super(mongoCursor, clientSession, converter, filter, options); this.user = user; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/iterators/InterpretationCatalogMongoDBIterator.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/iterators/InterpretationCatalogMongoDBIterator.java index e74020d6ba4..197a48ae652 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/iterators/InterpretationCatalogMongoDBIterator.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/iterators/InterpretationCatalogMongoDBIterator.java @@ -9,7 +9,7 @@ import org.opencb.commons.datastore.mongodb.MongoDBIterator; import org.opencb.opencga.catalog.db.api.ClinicalAnalysisDBAdaptor; import org.opencb.opencga.catalog.db.api.PanelDBAdaptor; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; import org.opencb.opencga.catalog.db.mongodb.PanelMongoDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; @@ -40,14 +40,15 @@ public class InterpretationCatalogMongoDBIterator extends CatalogMongoDBItera private static final String UID_VERSION_SEP = "___"; public InterpretationCatalogMongoDBIterator(MongoDBIterator mongoCursor, ClientSession clientSession, - GenericDocumentComplexConverter converter, MongoDBAdaptorFactory dbAdaptorFactory, - QueryOptions options) { + GenericDocumentComplexConverter converter, + OrganizationMongoDBAdaptorFactory dbAdaptorFactory, QueryOptions options) { this(mongoCursor, clientSession, converter, dbAdaptorFactory, 0, null, options); } public InterpretationCatalogMongoDBIterator(MongoDBIterator mongoCursor, ClientSession clientSession, - GenericDocumentComplexConverter converter, MongoDBAdaptorFactory dbAdaptorFactory, - long studyUid, String user, QueryOptions options) { + GenericDocumentComplexConverter converter, + OrganizationMongoDBAdaptorFactory dbAdaptorFactory, long studyUid, String user, + QueryOptions options) { super(mongoCursor, clientSession, converter, null); this.user = user; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/iterators/OrganizationCatalogMongoDBIterator.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/iterators/OrganizationCatalogMongoDBIterator.java new file mode 100644 index 00000000000..6f162997479 --- /dev/null +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/iterators/OrganizationCatalogMongoDBIterator.java @@ -0,0 +1,185 @@ +package org.opencb.opencga.catalog.db.mongodb.iterators; + +import com.mongodb.client.ClientSession; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.bson.Document; +import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.commons.datastore.mongodb.GenericDocumentComplexConverter; +import org.opencb.commons.datastore.mongodb.MongoDBIterator; +import org.opencb.opencga.catalog.db.api.NoteDBAdaptor; +import org.opencb.opencga.catalog.db.api.OrganizationDBAdaptor; +import org.opencb.opencga.catalog.db.mongodb.MigrationMongoDBAdaptor; +import org.opencb.opencga.catalog.db.mongodb.NoteMongoDBAdaptor; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.db.mongodb.ProjectMongoDBAdaptor; +import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; +import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogRuntimeException; +import org.opencb.opencga.core.models.notes.Note; +import org.opencb.opencga.core.response.OpenCGAResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; + +import static org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptor.NATIVE_QUERY; + +public class OrganizationCatalogMongoDBIterator extends CatalogMongoDBIterator { + + private final String user; + + private final QueryOptions options; + private final QueryOptions projectOptions; + + private final Queue organizationListBuffer; + private final MigrationMongoDBAdaptor migrationDBAdaptor; + private final ProjectMongoDBAdaptor projectMongoDBAdaptor; + private final NoteMongoDBAdaptor noteMongoDBAdaptor; + + private final Logger logger; + + + public OrganizationCatalogMongoDBIterator(MongoDBIterator mongoCursor, ClientSession clientSession, + GenericDocumentComplexConverter converter, OrganizationMongoDBAdaptorFactory dbAdaptorFactory, + QueryOptions options, String user) { + super(mongoCursor, clientSession, converter, null); + + this.options = options != null ? new QueryOptions(options) : new QueryOptions(); + this.projectOptions = createInnerQueryOptionsForVersionedEntity(this.options, OrganizationDBAdaptor.QueryParams.PROJECTS.key(), + true); + this.user = user; + + this.migrationDBAdaptor = dbAdaptorFactory.getMigrationDBAdaptor(); + this.noteMongoDBAdaptor = dbAdaptorFactory.getCatalogNotesDBAdaptor(); + this.projectMongoDBAdaptor = dbAdaptorFactory.getCatalogProjectDBAdaptor(); + + this.organizationListBuffer = new LinkedList<>(); + this.logger = LoggerFactory.getLogger(OrganizationCatalogMongoDBIterator.class); + } + + @Override + public boolean hasNext() { + if (organizationListBuffer.isEmpty()) { + fetchNextBatch(); + } + return !organizationListBuffer.isEmpty(); + } + + @Override + public E next() { + Document next = organizationListBuffer.remove(); + + if (filter != null) { + next = filter.apply(next); + } + + if (converter != null) { + return (E) converter.convertToDataModelType(next); + } else { + return (E) next; + } + } + + private void fetchNextBatch() { + if (mongoCursor.hasNext()) { + Document organizationDocument = mongoCursor.next(); + String orgId = organizationDocument.getString(OrganizationDBAdaptor.QueryParams.ID.key()); + + if (!options.getBoolean(NATIVE_QUERY)) { + List migrationFields = Arrays.asList(OrganizationDBAdaptor.QueryParams.INTERNAL.key(), + OrganizationDBAdaptor.QueryParams.INTERNAL_MIGRATION_EXECUTIONS.key()); + if (includeField(options, migrationFields)) { + List migrationRuns = migrationDBAdaptor.nativeGet().getResults(); + Document internal = organizationDocument.get(OrganizationDBAdaptor.QueryParams.INTERNAL.key(), Document.class); + if (internal == null) { + internal = new Document(); + organizationDocument.put(OrganizationDBAdaptor.QueryParams.INTERNAL.key(), internal); + } + internal.put("migrationExecutions", migrationRuns); + } + + List noteField = Collections.singletonList(OrganizationDBAdaptor.QueryParams.NOTES.key()); + if (includeField(options, noteField)) { + Query query = new Query(NoteDBAdaptor.QueryParams.SCOPE.key(), Note.Scope.ORGANIZATION.name()); + if (!options.getBoolean(OrganizationDBAdaptor.IS_ORGANIZATION_ADMIN_OPTION)) { + query.append(NoteDBAdaptor.QueryParams.VISIBILITY.key(), Note.Visibility.PUBLIC.name()); + } + + QueryOptions noteOptions = createInnerQueryOptionsForVersionedEntity(options, + OrganizationDBAdaptor.QueryParams.NOTES.key(), true); + noteOptions.put(QueryOptions.LIMIT, 1000); + try { + OpenCGAResult result = noteMongoDBAdaptor.nativeGet(clientSession, query, noteOptions); + organizationDocument.put(OrganizationDBAdaptor.QueryParams.NOTES.key(), result.getResults()); + } catch (CatalogDBException e) { + throw CatalogRuntimeException.internalException(e, "Could not fetch notes for organization '" + orgId + "'."); + } + } + + List projectField = Collections.singletonList(OrganizationDBAdaptor.QueryParams.PROJECTS.key()); + if (includeField(options, projectField)) { + OpenCGAResult openCGAResult = null; + try { + if (StringUtils.isNotEmpty(user)) { + openCGAResult = projectMongoDBAdaptor.nativeGet(clientSession, new Query(), projectOptions, user); + } else { + openCGAResult = projectMongoDBAdaptor.nativeGet(clientSession, new Query(), projectOptions); + } + organizationDocument.put(OrganizationDBAdaptor.QueryParams.PROJECTS.key(), openCGAResult.getResults()); + } catch (CatalogAuthorizationException e) { + logger.warn("Could not fetch projects for organization '{}'.", orgId, e); + } catch (CatalogDBException | RuntimeException e) { + throw CatalogRuntimeException.internalException(e, "Could not fetch projects for organization '" + orgId + "'."); + } + } + } + + // TODO: Remove this code after several releases (TASK-5923) + recoverSecretKeyTask5923(organizationDocument); + + organizationListBuffer.add(organizationDocument); + } + } + + // TODO: Remove this method after several releases (TASK-5923) + /** + * This method should only be necessary in order to be able to run the migration for TASK-5923 because OpenCGA looks for the config. + * This configuration was changed in TASK-5923, therefore, OpenCGA would be unable to initialise without it. + * Once the migration is run, this code will do nothing, so it should be removed in the future. + * + * @param organizationDocument Organization document. + */ + private void recoverSecretKeyTask5923(Document organizationDocument) { + Document configuration = organizationDocument.get(OrganizationDBAdaptor.QueryParams.CONFIGURATION.key(), Document.class); + if (configuration != null) { + String secretKey = null; + String algorithm = null; + long expiration = 3600L; + List authenticationOrigins = configuration.getList("authenticationOrigins", Document.class); + if (CollectionUtils.isNotEmpty(authenticationOrigins)) { + for (Document authenticationOrigin : authenticationOrigins) { + if (authenticationOrigin.getString("type").equals("OPENCGA")) { + secretKey = authenticationOrigin.getString("secretKey"); + algorithm = authenticationOrigin.getString("algorithm"); + Number expiration1 = authenticationOrigin.get("expiration", Number.class); + expiration = expiration1 != null ? expiration1.longValue() : expiration; + +// authenticationOrigin.put("id", "OPENCGA"); + } + } + if (StringUtils.isNotEmpty(secretKey)) { + Document token = configuration.get("token", Document.class); + if (token == null || StringUtils.isEmpty(token.getString("secretKey"))) { + token = new Document("secretKey", secretKey) + .append("algorithm", algorithm) + .append("expiration", expiration); + configuration.put("token", token); + } + } + } + } + } + +} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/iterators/ProjectCatalogMongoDBIterator.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/iterators/ProjectCatalogMongoDBIterator.java index 0cc8fc56180..8cfde189a02 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/iterators/ProjectCatalogMongoDBIterator.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/iterators/ProjectCatalogMongoDBIterator.java @@ -10,7 +10,7 @@ import org.opencb.opencga.catalog.db.api.ProjectDBAdaptor; import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptor; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; import org.opencb.opencga.catalog.db.mongodb.StudyMongoDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; @@ -38,7 +38,7 @@ public class ProjectCatalogMongoDBIterator extends CatalogMongoDBIterator private static final String UID = ProjectDBAdaptor.QueryParams.UID.key(); public ProjectCatalogMongoDBIterator(MongoDBIterator mongoCursor, ClientSession clientSession, - GenericDocumentComplexConverter converter, MongoDBAdaptorFactory dbAdaptorFactory, + GenericDocumentComplexConverter converter, OrganizationMongoDBAdaptorFactory dbAdaptorFactory, QueryOptions options, String user) { super(mongoCursor, clientSession, converter, null); @@ -47,7 +47,7 @@ public ProjectCatalogMongoDBIterator(MongoDBIterator mongoCursor, Clie this.options = options != null ? new QueryOptions(options) : new QueryOptions(); this.includeStudyInfo = includeStudyInfo(); this.studyQueryOptions = createInnerQueryOptionsForVersionedEntity(this.options, ProjectDBAdaptor.QueryParams.STUDIES.key(), false); - this.studyQueryOptions = MongoDBAdaptor.filterQueryOptions(this.studyQueryOptions, + this.studyQueryOptions = MongoDBAdaptor.filterQueryOptionsToIncludeKeys(this.studyQueryOptions, Collections.singletonList(MongoDBAdaptor.PRIVATE_PROJECT)); this.user = user; @@ -87,7 +87,7 @@ private void fetchNextBatch() { // Get next BUFFER_SIZE documents int counter = 0; while (mongoCursor.hasNext() && counter < BUFFER_SIZE) { - Document projectDocument = mongoCursor.next().get("projects", Document.class); + Document projectDocument = mongoCursor.next(); projectListBuffer.add(projectDocument); counter++; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/iterators/StudyCatalogMongoDBIterator.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/iterators/StudyCatalogMongoDBIterator.java index f5fe9dea875..957fb94a258 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/iterators/StudyCatalogMongoDBIterator.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/iterators/StudyCatalogMongoDBIterator.java @@ -16,12 +16,28 @@ package org.opencb.opencga.catalog.db.mongodb.iterators; +import com.mongodb.client.ClientSession; +import org.apache.commons.lang3.StringUtils; import org.bson.Document; +import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.mongodb.GenericDocumentComplexConverter; import org.opencb.commons.datastore.mongodb.MongoDBIterator; +import org.opencb.opencga.catalog.db.api.NoteDBAdaptor; +import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; +import org.opencb.opencga.catalog.db.mongodb.AuthorizationMongoDBUtils; +import org.opencb.opencga.catalog.db.mongodb.NoteMongoDBAdaptor; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.utils.ParamUtils; - +import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.models.notes.Note; +import org.opencb.opencga.core.response.OpenCGAResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.List; import java.util.function.Function; /** @@ -29,31 +45,26 @@ */ public class StudyCatalogMongoDBIterator extends CatalogMongoDBIterator { - private Function studyFilter; + private final Function studyFilter; + private final String user; + private final QueryOptions options; + private final NoteMongoDBAdaptor noteMongoDBAdaptor; private Document previousDocument; - private QueryOptions options; - - public StudyCatalogMongoDBIterator(MongoDBIterator mongoCursor, QueryOptions options) { - this(mongoCursor, options, null, null); - } - public StudyCatalogMongoDBIterator(MongoDBIterator mongoCursor, QueryOptions options, - GenericDocumentComplexConverter converter) { - this(mongoCursor, options, converter, null); - } + private final Logger logger; - public StudyCatalogMongoDBIterator(MongoDBIterator mongoCursor, QueryOptions options, - Function studyFilter) { - this(mongoCursor, options, null, studyFilter); - } - - public StudyCatalogMongoDBIterator(MongoDBIterator mongoCursor, QueryOptions options, - GenericDocumentComplexConverter converter, Function studyFilter) { - super(mongoCursor, null, converter, null); + public StudyCatalogMongoDBIterator(MongoDBIterator mongoCursor, ClientSession clientSession, + OrganizationMongoDBAdaptorFactory dbAdaptorFactory, QueryOptions options, + GenericDocumentComplexConverter converter, Function studyFilter, String user) { + super(mongoCursor, clientSession, converter, null); this.mongoCursor = mongoCursor; this.converter = converter; this.studyFilter = studyFilter; this.options = ParamUtils.defaultObject(options, QueryOptions::new); + this.user = user; + this.noteMongoDBAdaptor = dbAdaptorFactory.getCatalogNotesDBAdaptor(); + + this.logger = LoggerFactory.getLogger(StudyCatalogMongoDBIterator.class); getNextStudy(); } @@ -72,7 +83,29 @@ private void getNextStudy() { } } - addAclInformation(previousDocument, options); + if (previousDocument != null) { + List noteField = Collections.singletonList(StudyDBAdaptor.QueryParams.NOTES.key()); + if (includeField(options, noteField)) { + Query query = new Query() + .append(NoteDBAdaptor.QueryParams.STUDY_UID.key(), previousDocument.get(StudyDBAdaptor.QueryParams.UID.key())) + .append(NoteDBAdaptor.QueryParams.SCOPE.key(), Note.Scope.STUDY.name()); + if (!isAtLeastStudyAdmin(previousDocument)) { + query.append(NoteDBAdaptor.QueryParams.VISIBILITY.key(), Note.Visibility.PUBLIC.name()); + } + + QueryOptions noteOptions = createInnerQueryOptionsForVersionedEntity(options, + StudyDBAdaptor.QueryParams.NOTES.key(), true); + noteOptions.put(QueryOptions.LIMIT, 1000); + try { + OpenCGAResult result = noteMongoDBAdaptor.nativeGet(clientSession, query, noteOptions); + previousDocument.put(StudyDBAdaptor.QueryParams.NOTES.key(), result.getResults()); + } catch (CatalogDBException e) { + logger.warn("Could not obtain the organization notes", e); + } + } + + addAclInformation(previousDocument, options); + } } else { this.previousDocument = null; } @@ -95,6 +128,17 @@ public E next() { } } + private boolean isAtLeastStudyAdmin(Document studyDoc) { + if (StringUtils.isEmpty(user)) { + return true; + } + if (user.startsWith(ParamConstants.ADMIN_ORGANIZATION + ":")) { + return true; + } + List users = AuthorizationMongoDBUtils.getAdminUsers(studyDoc); + return users.contains(user); + } + @Override public void close() { mongoCursor.close(); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/CatalogAuthenticationException.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/CatalogAuthenticationException.java index 7d20f0b3a98..8ac1b5c385f 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/CatalogAuthenticationException.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/CatalogAuthenticationException.java @@ -61,8 +61,31 @@ public static CatalogAuthenticationException incorrectUserOrPassword(String doma return new CatalogAuthenticationException(domain + ": Incorrect user or password.", e); } + public static CatalogAuthenticationException userIsBanned(String userId) { + return new CatalogAuthenticationException("Too many login attempts. The account for user '" + userId + "' is banned." + + " Please, talk to your organization owner/administrator."); + } + + public static CatalogAuthenticationException userIsSuspended(String userId) { + return new CatalogAuthenticationException("The account for user '" + userId + "' is suspended. Please, talk to your organization" + + " owner/administrator."); + } + + public static CatalogAuthenticationException accountIsExpired(String userId, String expirationDate) { + return new CatalogAuthenticationException("The account for user '" + userId + "' expired on " + expirationDate + ". Please," + + " talk to your organization owner/administrator."); + } + + public static CatalogAuthenticationException passwordExpired(String userId, String expirationDate) { + return new CatalogAuthenticationException("The password for the user account '" + userId + "' expired on " + expirationDate + + ". Please, reset your password or talk to your organization owner/administrator."); + } + public static CatalogAuthenticationException userNotAllowed(String domain) { return new CatalogAuthenticationException(domain + ": User not allowed to access the system."); } + public static CatalogAuthenticationException userNotFound(String organizationId, String userId) { + return new CatalogAuthenticationException("User '" + userId + "' not found in organization '" + organizationId + "'."); + } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/CatalogAuthorizationException.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/CatalogAuthorizationException.java index 2cd44d4a8ac..546d3117203 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/CatalogAuthorizationException.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/CatalogAuthorizationException.java @@ -20,6 +20,28 @@ * @author Jacobo Coll <jacobo167@gmail.com> */ public class CatalogAuthorizationException extends CatalogException { + + public enum ErrorCode { + KILL_JOB("kill", "Only the job owner or the study administrator can kill a job."); + + private final String permission; + private final String message; + + ErrorCode(String permission, String message) { + this.permission = permission; + this.message = message; + } + + public String getPermission() { + return permission; + } + + public String getMessage() { + return message; + } + } + + public CatalogAuthorizationException(String message) { super(message); } @@ -48,10 +70,6 @@ public static CatalogAuthorizationException cantExecute(String userId, String re return deny(userId, "execute", resource, id, name); } - public static CatalogAuthorizationException adminOnlySupportedOperation() { - return new CatalogAuthorizationException("Operation only supported for 'admin' user"); - } - @Deprecated public static CatalogAuthorizationException deny(String userId, String permission, String resource, long id, String name) { return new CatalogAuthorizationException("Permission denied. " @@ -67,15 +85,64 @@ public static CatalogAuthorizationException deny(String userId, String permissio + resource + " { id: " + id + (name == null || name.isEmpty() ? "" : ", name: \"" + name + "\"") + " }"); } + public static CatalogAuthorizationException deny(String userId, String resource, String id, ErrorCode errorCode) { + return deny(userId, resource, id, errorCode, null); + } + + public static CatalogAuthorizationException deny(String userId, String resource, String id, ErrorCode errorCode, Throwable cause) { + return new CatalogAuthorizationException("Permission denied. " + + (userId == null || userId.isEmpty() ? "" : "User '" + userId + "'") + + " cannot " + errorCode.getPermission() + " " + resource + " { id: " + id + " }. " + errorCode.getMessage(), cause); + } + public static CatalogAuthorizationException deny(String userId, String description) { return new CatalogAuthorizationException("Permission denied. " + (userId == null || userId.isEmpty() ? "" : "User '" + userId + "'") + " cannot " + description); } + public static CatalogAuthorizationException deny(Throwable cause) { + return new CatalogAuthorizationException("Permission denied.", cause); + } + public static CatalogAuthorizationException denyAny(String userId, String permission, String resource) { return new CatalogAuthorizationException("Permission denied. " + (userId == null || userId.isEmpty() ? "" : "User '" + userId + "'") + " cannot " + permission + " any " + resource + "."); } + public static CatalogAuthorizationException notOrganizationOwner() { + return notOrganizationOwner("perform this action"); + } + + public static CatalogAuthorizationException notOrganizationOwner(String action) { + return new CatalogAuthorizationException("Permission denied: Only the organization owner can " + action); + } + + public static CatalogAuthorizationException notOrganizationOwnerOrAdmin() { + return notOrganizationOwnerOrAdmin("perform this action"); + } + + public static CatalogAuthorizationException notOrganizationOwnerOrAdmin(String action) { + return new CatalogAuthorizationException("Permission denied: Only the owner or administrators of the organization can " + action); + } + + public static CatalogAuthorizationException notStudyAdmin(String action) { + return new CatalogAuthorizationException("Permission denied: Only the study administrators can " + action); + } + + public static CatalogAuthorizationException notStudyMember(String action) { + return new CatalogAuthorizationException("Permission denied: Only the members of the study can " + action); + } + + public static CatalogAuthorizationException opencgaAdminOnlySupportedOperation() { + return opencgaAdminOnlySupportedOperation("perform this action"); + } + + public static CatalogAuthorizationException opencgaAdminOnlySupportedOperation(String action) { + if (action == null || action.isEmpty()) { + action = "perform this action"; + } + return new CatalogAuthorizationException("Permission denied: Only the OPENCGA ADMINISTRATOR users can " + action); + } + } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/CatalogException.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/CatalogException.java index dc22f437349..4bc90c42f09 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/CatalogException.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/CatalogException.java @@ -39,7 +39,7 @@ public CatalogException(Throwable cause) { } public static CatalogException notFound(String entity, List entries) { - return new CatalogException("Missing " + entity + ": " + StringUtils.join(entries, ", ") + " not found."); + return new CatalogException("Missing " + entity + ": '" + StringUtils.join(entries, "', '") + "' not found."); } public static CatalogException appendMessage(CatalogException e, String message) { diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/CatalogRuntimeException.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/CatalogRuntimeException.java new file mode 100644 index 00000000000..81dc17b16af --- /dev/null +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/CatalogRuntimeException.java @@ -0,0 +1,34 @@ +package org.opencb.opencga.catalog.exceptions; + +import org.apache.commons.lang3.StringUtils; + +public class CatalogRuntimeException extends IllegalArgumentException { + + public CatalogRuntimeException(String message) { + super(message); + } + + public CatalogRuntimeException(String message, Throwable cause) { + super(message, cause); + } + + public static CatalogRuntimeException internalException(Exception e, String message) { + if (e instanceof CatalogRuntimeException) { + return ((CatalogRuntimeException) e); + } else { + if (StringUtils.isEmpty(message)) { + message = e.getMessage(); + if (StringUtils.isEmpty(message)) { + message = e.toString(); + } + } + return new CatalogRuntimeException("Internal exception: " + message, e); + } + } + + public static CatalogRuntimeException internalException(Exception e) { + return internalException(e, ""); + } + + +} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/io/CatalogIOManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/io/CatalogIOManager.java index 551476b874b..699bc9c99da 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/io/CatalogIOManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/io/CatalogIOManager.java @@ -24,27 +24,27 @@ import java.net.URI; import java.net.URISyntaxException; +import java.nio.file.Path; import java.nio.file.Paths; public class CatalogIOManager { - /** + /* * OpenCGA folders are created in the ROOTDIR. - * OPENCGA_USERS_FOLDER contains users workspaces organized by 'userId' + * OPENCGA_ORGANIZATIONS_FOLDER contains organization workspaces organized by 'organizationId' * OPENCGA_ANONYMOUS_USERS_FOLDER contains anonymous users workspaces organized by 'randomStringId' * OPENCGA_BIN_FOLDER contains all packaged binaries delivered within OpenCGA */ - private static final String OPENCGA_USERS_FOLDER = "users/"; + private static final String OPENCGA_ORGANIZATIONS_FOLDER = "orgs/"; private static final String OPENCGA_ANONYMOUS_USERS_FOLDER = "anonymous/"; private static final String OPENCGA_BIN_FOLDER = "bin/"; - /** - * Users folders are created inside user workspace. - * USER_PROJECTS_FOLDER this folder stores all the projects with the studies and files - * USER_BIN_FOLDER contains user specific binaries + /* + * ORGANIZATION_PROJECTS_FOLDER this folder stores all the projects with the studies and files. + * ORGANIZATION_BIN_FOLDER contains user specific binaries */ - protected static final String USER_PROJECTS_FOLDER = "projects/"; - protected static final String USER_BIN_FOLDER = "bin/"; + protected static final String ORGANIZATION_PROJECTS_FOLDER = "projects/"; + protected static final String ORGANIZATION_BIN_FOLDER = "bin/"; protected static Logger logger = LoggerFactory.getLogger(CatalogIOManager.class); protected URI rootDir; private URI jobDir; @@ -67,8 +67,8 @@ public void createDefaultOpenCGAFolders() throws CatalogIOException { } ioManager.checkDirectoryUri(rootDir, true); - if (!ioManager.exists(getUsersUri())) { - ioManager.createDirectory(getUsersUri()); + if (!ioManager.exists(getOrganizationsUri())) { + ioManager.createDirectory(getOrganizationsUri()); } if (!ioManager.exists(rootDir.resolve(OPENCGA_ANONYMOUS_USERS_FOLDER))) { @@ -90,42 +90,43 @@ protected void checkParam(String param) throws CatalogIOException { } } - public URI getUsersUri() { - return Paths.get(rootDir).resolve(OPENCGA_USERS_FOLDER).toUri(); + public URI getOrganizationsUri() { + return Paths.get(rootDir).resolve(OPENCGA_ORGANIZATIONS_FOLDER).toUri(); } - public URI getUserUri(String userId) throws CatalogIOException { - checkParam(userId); - return Paths.get(getUsersUri()).resolve(userId.endsWith("/") ? userId : (userId + "/")).toUri(); + public URI getOrganizationsUri(String organizationId) throws CatalogIOException { + checkParam(organizationId); + return Paths.get(getOrganizationsUri()).resolve(organizationId.endsWith("/") ? organizationId : (organizationId + "/")).toUri(); } - public URI getProjectsUri(String userId) throws CatalogIOException { - return Paths.get(getUserUri(userId)).resolve(USER_PROJECTS_FOLDER).toUri(); + public URI getProjectsUri(String organizationId) throws CatalogIOException { + return Paths.get(getOrganizationsUri(organizationId)).resolve(ORGANIZATION_PROJECTS_FOLDER).toUri(); } - public URI getProjectUri(String userId, String projectId) throws CatalogIOException { - return Paths.get(getProjectsUri(userId)).resolve(projectId.endsWith("/") ? projectId : (projectId + "/")).toUri(); + public URI getProjectUri(String organizationId, String projectId) throws CatalogIOException { + return Paths.get(getProjectsUri(organizationId)).resolve(projectId.endsWith("/") ? projectId : (projectId + "/")).toUri(); } - private URI getStudyUri(String userId, String projectId, String studyId) throws CatalogIOException { + private URI getStudyUri(String organizationId, String projectId, String studyId) throws CatalogIOException { checkParam(studyId); - return Paths.get(getProjectUri(userId, projectId)).resolve(studyId.endsWith("/") ? studyId : (studyId + "/")).toUri(); + return Paths.get(getProjectUri(organizationId, projectId)).resolve(studyId.endsWith("/") ? studyId : (studyId + "/")).toUri(); } - public URI createUser(String userId) throws CatalogIOException { - checkParam(userId); + public URI createOrganization(String organizationId) throws CatalogIOException { + checkParam(organizationId); - URI usersUri = getUsersUri(); - ioManager.checkDirectoryUri(usersUri, true); + URI organizationsUri = getOrganizationsUri(); + ioManager.checkDirectoryUri(organizationsUri, true); - URI userPath = getUserUri(userId); + URI organizationUri = getOrganizationsUri(organizationId); + Path organizationPath = Paths.get(organizationUri); try { - if (!ioManager.exists(userPath)) { - ioManager.createDirectory(userPath); - ioManager.createDirectory(Paths.get(userPath).resolve(CatalogIOManager.USER_PROJECTS_FOLDER).toUri()); - ioManager.createDirectory(Paths.get(userPath).resolve(CatalogIOManager.USER_BIN_FOLDER).toUri()); + if (!ioManager.exists(organizationUri)) { + ioManager.createDirectory(organizationUri); + ioManager.createDirectory(organizationPath.resolve(CatalogIOManager.ORGANIZATION_PROJECTS_FOLDER).toUri()); + ioManager.createDirectory(organizationPath.resolve(CatalogIOManager.ORGANIZATION_BIN_FOLDER).toUri()); - return userPath; + return organizationUri; } } catch (CatalogIOException e) { throw e; @@ -133,16 +134,16 @@ public URI createUser(String userId) throws CatalogIOException { return null; } - public void deleteUser(String userId) throws CatalogIOException { - URI userUri = getUserUri(userId); - ioManager.checkUriExists(userUri); - ioManager.deleteDirectory(userUri); + public void deleteOrganization(String organization) throws CatalogIOException { + URI organizationsUri = getOrganizationsUri(organization); + ioManager.checkUriExists(organizationsUri); + ioManager.deleteDirectory(organizationsUri); } - public URI createProject(String userId, String projectId) throws CatalogIOException { + public URI createProject(String organizationId, String projectId) throws CatalogIOException { checkParam(projectId); - URI projectUri = getProjectUri(userId, projectId); + URI projectUri = getProjectUri(organizationId, projectId); try { if (!ioManager.exists(projectUri)) { projectUri = ioManager.createDirectory(projectUri, true); @@ -154,8 +155,8 @@ public URI createProject(String userId, String projectId) throws CatalogIOExcept return projectUri; } - public URI createStudy(String userId, String projectId, String studyId) throws CatalogIOException { - URI studyUri = getStudyUri(userId, projectId, studyId); + public URI createStudy(String organizationId, String projectId, String studyId) throws CatalogIOException { + URI studyUri = getStudyUri(organizationId, projectId, studyId); ioManager.checkUriScheme(studyUri); try { if (!ioManager.exists(studyUri)) { diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AbstractManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AbstractManager.java index 5f12e37db83..da7566340ac 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AbstractManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AbstractManager.java @@ -17,7 +17,6 @@ package org.opencb.opencga.catalog.managers; import org.apache.commons.collections4.CollectionUtils; -import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.QueryParam; @@ -30,7 +29,6 @@ import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.models.InternalGetDataResult; import org.opencb.opencga.core.api.ParamConstants; -import org.opencb.opencga.core.config.AuthenticationOrigin; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.models.IPrivateStudyUid; import org.opencb.opencga.core.models.study.Group; @@ -54,18 +52,6 @@ public abstract class AbstractManager { protected Configuration configuration; protected final DBAdaptorFactory catalogDBAdaptorFactory; - protected final UserDBAdaptor userDBAdaptor; - protected final ProjectDBAdaptor projectDBAdaptor; - protected final StudyDBAdaptor studyDBAdaptor; - protected final FileDBAdaptor fileDBAdaptor; - protected final IndividualDBAdaptor individualDBAdaptor; - protected final SampleDBAdaptor sampleDBAdaptor; - protected final CohortDBAdaptor cohortDBAdaptor; - protected final FamilyDBAdaptor familyDBAdaptor; - protected final JobDBAdaptor jobDBAdaptor; - protected final PanelDBAdaptor panelDBAdaptor; - protected final ClinicalAnalysisDBAdaptor clinicalDBAdaptor; - protected final InterpretationDBAdaptor interpretationDBAdaptor; public static final String OPENCGA = ParamConstants.OPENCGA_USER_ID; public static final String ANONYMOUS = ParamConstants.ANONYMOUS_USER_ID; @@ -81,25 +67,80 @@ public abstract class AbstractManager { this.authorizationManager = authorizationManager; this.auditManager = auditManager; this.configuration = configuration; - this.userDBAdaptor = catalogDBAdaptorFactory.getCatalogUserDBAdaptor(); - this.studyDBAdaptor = catalogDBAdaptorFactory.getCatalogStudyDBAdaptor(); - this.fileDBAdaptor = catalogDBAdaptorFactory.getCatalogFileDBAdaptor(); - this.individualDBAdaptor = catalogDBAdaptorFactory.getCatalogIndividualDBAdaptor(); - this.sampleDBAdaptor = catalogDBAdaptorFactory.getCatalogSampleDBAdaptor(); - this.jobDBAdaptor = catalogDBAdaptorFactory.getCatalogJobDBAdaptor(); - this.cohortDBAdaptor = catalogDBAdaptorFactory.getCatalogCohortDBAdaptor(); - this.familyDBAdaptor = catalogDBAdaptorFactory.getCatalogFamilyDBAdaptor(); - this.panelDBAdaptor = catalogDBAdaptorFactory.getCatalogPanelDBAdaptor(); - this.clinicalDBAdaptor = catalogDBAdaptorFactory.getClinicalAnalysisDBAdaptor(); - this.interpretationDBAdaptor = catalogDBAdaptorFactory.getInterpretationDBAdaptor(); this.catalogDBAdaptorFactory = catalogDBAdaptorFactory; this.catalogManager = catalogManager; - projectDBAdaptor = catalogDBAdaptorFactory.getCatalogProjectDbAdaptor(); - logger = LoggerFactory.getLogger(this.getClass()); } + protected DBAdaptorFactory getCatalogDBAdaptorFactory() { + return catalogDBAdaptorFactory; + } + + protected MigrationDBAdaptor getMigrationDBAdaptor(String organization) throws CatalogDBException { + return getCatalogDBAdaptorFactory().getMigrationDBAdaptor(organization); + } + + protected MetaDBAdaptor getCatalogMetaDBAdaptor(String organization) throws CatalogDBException { + return getCatalogDBAdaptorFactory().getCatalogMetaDBAdaptor(organization); + } + + protected OrganizationDBAdaptor getOrganizationDBAdaptor(String organization) throws CatalogDBException { + return getCatalogDBAdaptorFactory().getCatalogOrganizationDBAdaptor(organization); + } + + protected UserDBAdaptor getUserDBAdaptor(String organization) throws CatalogDBException { + return getCatalogDBAdaptorFactory().getCatalogUserDBAdaptor(organization); + } + + protected ProjectDBAdaptor getProjectDBAdaptor(String organization) throws CatalogDBException { + return getCatalogDBAdaptorFactory().getCatalogProjectDbAdaptor(organization); + } + + protected StudyDBAdaptor getStudyDBAdaptor(String organization) throws CatalogDBException { + return getCatalogDBAdaptorFactory().getCatalogStudyDBAdaptor(organization); + } + + protected FileDBAdaptor getFileDBAdaptor(String organization) throws CatalogDBException { + return getCatalogDBAdaptorFactory().getCatalogFileDBAdaptor(organization); + } + + protected SampleDBAdaptor getSampleDBAdaptor(String organization) throws CatalogDBException { + return getCatalogDBAdaptorFactory().getCatalogSampleDBAdaptor(organization); + } + + protected IndividualDBAdaptor getIndividualDBAdaptor(String organization) throws CatalogDBException { + return getCatalogDBAdaptorFactory().getCatalogIndividualDBAdaptor(organization); + } + + protected JobDBAdaptor getJobDBAdaptor(String organization) throws CatalogDBException { + return getCatalogDBAdaptorFactory().getCatalogJobDBAdaptor(organization); + } + + protected AuditDBAdaptor getAuditDbAdaptor(String organization) throws CatalogDBException { + return getCatalogDBAdaptorFactory().getCatalogAuditDbAdaptor(organization); + } + + protected CohortDBAdaptor getCohortDBAdaptor(String organization) throws CatalogDBException { + return getCatalogDBAdaptorFactory().getCatalogCohortDBAdaptor(organization); + } + + protected PanelDBAdaptor getPanelDBAdaptor(String organization) throws CatalogDBException { + return getCatalogDBAdaptorFactory().getCatalogPanelDBAdaptor(organization); + } + + protected FamilyDBAdaptor getFamilyDBAdaptor(String organization) throws CatalogDBException { + return getCatalogDBAdaptorFactory().getCatalogFamilyDBAdaptor(organization); + } + + protected ClinicalAnalysisDBAdaptor getClinicalAnalysisDBAdaptor(String organization) throws CatalogDBException { + return getCatalogDBAdaptorFactory().getClinicalAnalysisDBAdaptor(organization); + } + + protected InterpretationDBAdaptor getInterpretationDBAdaptor(String organization) throws CatalogDBException { + return getCatalogDBAdaptorFactory().getInterpretationDBAdaptor(organization); + } + protected void fixQueryObject(Query query) { changeQueryId(query, ParamConstants.INTERNAL_STATUS_PARAM, "internal.status"); } @@ -121,50 +162,6 @@ protected void changeQueryId(Query query, String currentKey, String newKey) { } } - /** - * Prior to the conversion to a numerical featureId, there is a need to know in which user/project/study look for the string. - * This method calculates those parameters to know how to obtain the numerical id. - * - * @param userId User id of the user asking for the id. If no user is found in featureStr, we will assume that it is asking for its - * projects/studies... - * @param featureStr Feature id in string format. Could be one of [user@aliasProject:aliasStudy:XXXXX - * | user@aliasStudy:XXXXX | aliasStudy:XXXXX | XXXXX]. - * @return an objectMap with the following possible keys: "user", "project", "study", "featureName" - */ - protected ObjectMap parseFeatureId(String userId, String featureStr) { - ObjectMap result = new ObjectMap("user", userId); - - String[] split = featureStr.split("@"); - if (split.length == 2) { // user@project:study - result.put("user", split[0]); - featureStr = split[1]; - } - - split = featureStr.split(":", 3); - if (split.length == 2) { - result.put("study", split[0]); - result.put("featureName", split[1]); - } else if (split.length == 3) { - result.put("project", split[0]); - result.put("study", split[1]); - result.put("featureName", split[2]); - } else { - result.put("featureName", featureStr); - } - return result; - } - - AuthenticationOrigin getAuthenticationOrigin(String authOrigin) { - if (configuration.getAuthentication().getAuthenticationOrigins() != null) { - for (AuthenticationOrigin authenticationOrigin : configuration.getAuthentication().getAuthenticationOrigins()) { - if (authOrigin.equals(authenticationOrigin.getId())) { - return authenticationOrigin; - } - } - } - return null; - } - /** * Return the results in the OpenCGAResult object in the same order they were queried by the list of entries. @@ -293,16 +290,17 @@ List getMissingFields(List original * - '@{groupId}' referring to a {@link Group}. * - '{userId}' referring to a specific user. * + * @param organization organization * @param studyId studyId * @param members List of members * @throws CatalogDBException CatalogDBException * @throws CatalogParameterException if there is any formatting error. * @throws CatalogAuthorizationException if the user is not authorised to perform the query. */ - protected void checkMembers(long studyId, List members) + protected void checkMembers(String organization, long studyId, List members) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { for (String member : members) { - checkMember(studyId, member); + checkMember(organization, studyId, member); } } @@ -314,23 +312,24 @@ protected void checkMembers(long studyId, List members) * - '@{groupId}' referring to a {@link Group}. * - '{userId}' referring to a specific user. * + * @param organization organization * @param studyId studyId * @param member member * @throws CatalogDBException CatalogDBException * @throws CatalogParameterException if there is any formatting error. * @throws CatalogAuthorizationException if the user is not authorised to perform the query. */ - protected void checkMember(long studyId, String member) + protected void checkMember(String organization, long studyId, String member) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { if (member.equals("*")) { return; } else if (member.startsWith("@")) { - OpenCGAResult queryResult = studyDBAdaptor.getGroup(studyId, member, Collections.emptyList()); + OpenCGAResult queryResult = getStudyDBAdaptor(organization).getGroup(studyId, member, Collections.emptyList()); if (queryResult.getNumResults() == 0) { throw CatalogDBException.idNotFound("Group", member); } } else { - userDBAdaptor.checkId(member); + getUserDBAdaptor(organization).checkId(member); } } @@ -365,89 +364,4 @@ protected static Class getTypeClass(QueryParam.Type type) { } } - @Deprecated - public static class MyResourceId { - private String user; - private long studyId; - private long resourceId; - - public MyResourceId() { - } - - public MyResourceId(String user, long studyId, long resourceId) { - this.user = user; - this.studyId = studyId; - this.resourceId = resourceId; - } - - public String getUser() { - return user; - } - - public MyResourceId setUser(String user) { - this.user = user; - return this; - } - - public long getStudyId() { - return studyId; - } - - public MyResourceId setStudyId(long studyId) { - this.studyId = studyId; - return this; - } - - public long getResourceId() { - return resourceId; - } - - public MyResourceId setResourceId(long resourceId) { - this.resourceId = resourceId; - return this; - } - } - - @Deprecated - public static class MyResourceIds { - private String user; - private long studyId; - private List resourceIds; - - public MyResourceIds() { - } - - public MyResourceIds(String user, long studyId, List resourceIds) { - this.user = user; - this.studyId = studyId; - this.resourceIds = resourceIds; - } - - public String getUser() { - return user; - } - - public MyResourceIds setUser(String user) { - this.user = user; - return this; - } - - public long getStudyId() { - return studyId; - } - - public MyResourceIds setStudyId(long studyId) { - this.studyId = studyId; - return this; - } - - public List getResourceIds() { - return resourceIds; - } - - public MyResourceIds setResourceIds(List resourceIds) { - this.resourceIds = resourceIds; - return this; - } - } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AdminManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AdminManager.java index b57426ebf04..7edf1030496 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AdminManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AdminManager.java @@ -1,6 +1,7 @@ package org.opencb.opencga.catalog.managers; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.time.StopWatch; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; @@ -8,10 +9,14 @@ import org.opencb.opencga.catalog.db.DBAdaptorFactory; import org.opencb.opencga.catalog.db.api.UserDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.io.CatalogIOManager; +import org.opencb.opencga.catalog.utils.CatalogFqn; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.config.Configuration; +import org.opencb.opencga.core.models.Acl; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.audit.AuditRecord; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.study.Group; @@ -22,7 +27,11 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; public class AdminManager extends AbstractManager { @@ -36,63 +45,51 @@ public class AdminManager extends AbstractManager { this.catalogIOManager = catalogIOManager; } - public OpenCGAResult userSearch(Query query, QueryOptions options, String token) + public OpenCGAResult userSearch(String organizationId, Query query, QueryOptions options, String token) throws CatalogException { - query = ParamUtils.defaultObject(query, Query::new); - options = ParamUtils.defaultObject(options, QueryOptions::new); - ObjectMap auditParams = new ObjectMap() + .append("organizationId", organizationId) .append("query", query) .append("options", options) .append("token", token); - String userId = catalogManager.getUserManager().getUserId(token); + JwtPayload jwtPayload = catalogManager.getUserManager().validateToken(token); try { - authorizationManager.checkIsInstallationAdministrator(userId); - - // Fix query object - if (query.containsKey(ParamConstants.USER)) { - query.put(UserDBAdaptor.QueryParams.ID.key(), query.get(ParamConstants.USER)); - query.remove(ParamConstants.USER); - } - if (query.containsKey(ParamConstants.USER_ACCOUNT_TYPE)) { - query.put(UserDBAdaptor.QueryParams.ACCOUNT_TYPE.key(), query.get(ParamConstants.USER_ACCOUNT_TYPE)); - query.remove(ParamConstants.USER_ACCOUNT_TYPE); - } - if (query.containsKey(ParamConstants.USER_AUTHENTICATION_ORIGIN)) { - query.put(UserDBAdaptor.QueryParams.ACCOUNT_AUTHENTICATION_ID.key(), query.get(ParamConstants.USER_AUTHENTICATION_ORIGIN)); - query.remove(ParamConstants.USER_AUTHENTICATION_ORIGIN); - } - if (query.containsKey(ParamConstants.USER_CREATION_DATE)) { - query.put(UserDBAdaptor.QueryParams.ACCOUNT_CREATION_DATE.key(), query.get(ParamConstants.USER_CREATION_DATE)); - query.remove(ParamConstants.USER_CREATION_DATE); - } - - OpenCGAResult userDataResult = userDBAdaptor.get(query, options); - auditManager.auditSearch(userId, Enums.Resource.USER, "", "", auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); - return userDataResult; + authorizationManager.checkIsOpencgaAdministrator(jwtPayload); } catch (CatalogException e) { - auditManager.auditSearch(userId, Enums.Resource.USER, "", "", auditParams, + auditManager.auditSearch(organizationId, jwtPayload.getUserId(organizationId), Enums.Resource.USER, "", "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } + + // Fix query object + if (query.containsKey(ParamConstants.USER)) { + query.put(UserDBAdaptor.QueryParams.ID.key(), query.get(ParamConstants.USER)); + query.remove(ParamConstants.USER); + } + if (query.containsKey(ParamConstants.USER_CREATION_DATE)) { + query.put(UserDBAdaptor.QueryParams.INTERNAL_ACCOUNT_CREATION_DATE.key(), query.get(ParamConstants.USER_CREATION_DATE)); + query.remove(ParamConstants.USER_CREATION_DATE); + } + + return catalogManager.getUserManager().search(organizationId, query, options, token); } - public OpenCGAResult updateGroups(String userId, List studyIds, List groupIds, + public OpenCGAResult updateGroups(String organizationId, String userId, List studyIds, List groupIds, ParamUtils.AddRemoveAction action, String token) throws CatalogException { ObjectMap auditParams = new ObjectMap() + .append("organizationId", organizationId) .append("userId", userId) .append("studyIds", studyIds) .append("groupIds", groupIds) .append("action", action) .append("token", token); - String authenticatedUser = catalogManager.getUserManager().getUserId(token); + JwtPayload jwtPayload = catalogManager.getUserManager().validateToken(token); try { - authorizationManager.checkIsInstallationAdministrator(authenticatedUser); + authorizationManager.checkIsOpencgaAdministrator(jwtPayload); // Check userId exists Query query = new Query(UserDBAdaptor.QueryParams.ID.key(), userId); - OpenCGAResult count = userDBAdaptor.count(query); + OpenCGAResult count = getUserDBAdaptor(organizationId).count(query); if (count.getNumMatches() == 0) { throw new CatalogException("User '" + userId + "' not found."); } @@ -106,7 +103,8 @@ public OpenCGAResult updateGroups(String userId, List studyIds, L } // Check studyIds exist - List studies = catalogManager.getStudyManager().resolveIds(studyIds, authenticatedUser); + List studies = catalogManager.getStudyManager().resolveIds(studyIds, jwtPayload.getUserId(organizationId), + organizationId); List studyUids = new ArrayList<>(studies.size()); for (Study study : studies) { if (ParamConstants.ADMIN_STUDY_FQN.equals(study.getFqn())) { @@ -119,56 +117,123 @@ public OpenCGAResult updateGroups(String userId, List studyIds, L } } - OpenCGAResult result = studyDBAdaptor.updateUserFromGroups(userId, studyUids, groupIds, action); + OpenCGAResult result = getStudyDBAdaptor(organizationId).updateUserFromGroups(userId, studyUids, groupIds, action); - auditManager.audit(userId, Enums.Action.UPDATE_USERS_FROM_STUDY_GROUP, Enums.Resource.STUDY, "", "", "", "", auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.audit(organizationId, jwtPayload.getUserId(organizationId), Enums.Action.UPDATE_USERS_FROM_STUDY_GROUP, + Enums.Resource.STUDY, "", "", "", "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return result; } catch (CatalogException e) { - auditManager.audit(userId, Enums.Action.UPDATE_USERS_FROM_STUDY_GROUP, Enums.Resource.STUDY, "", "", "", "", auditParams, + auditManager.audit(organizationId, jwtPayload.getUserId(organizationId), Enums.Action.UPDATE_USERS_FROM_STUDY_GROUP, + Enums.Resource.STUDY, "", "", "", "", auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + throw e; + } + } + + public List getOrganizationIds(String token) throws CatalogException { + JwtPayload payload = catalogManager.getUserManager().validateToken(token); + try { + authorizationManager.checkIsOpencgaAdministrator(payload); + List organizationIds = getCatalogDBAdaptorFactory().getOrganizationIds(); + + auditManager.audit(ParamConstants.ADMIN_ORGANIZATION, payload.getUserId(), Enums.Action.FETCH_ORGANIZATION_IDS, + Enums.Resource.STUDY, "", "", "", "", new ObjectMap(), new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + return organizationIds; + } catch (CatalogException e) { + auditManager.audit(ParamConstants.ADMIN_ORGANIZATION, payload.getUserId(), Enums.Action.FETCH_ORGANIZATION_IDS, + Enums.Resource.STUDY, "", "", "", "", new ObjectMap(), new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } + } /** * Add a user to all the remote groups he/she may belong for a particular authentication origin. * Also remove the user from other groups he/she may have been associated in the past for the same authentication origin. * - * @param userId User id. - * @param remoteGroupIds List of group ids the user must be associated with. + * @param organizationId Organization id. + * @param userId User id. + * @param remoteGroupIds List of group ids the user must be associated with. * @param authenticationOriginId The authentication origin the groups must be associated to. - * @param token Administrator token. + * @param token Administrator token. * @return An empty OpenCGAResult. * @throws CatalogException If the token is invalid or belongs to a user other thant the Administrator and if userId does not exist. */ - public OpenCGAResult syncRemoteGroups(String userId, List remoteGroupIds, - String authenticationOriginId, String token) throws CatalogException { + public OpenCGAResult syncRemoteGroups(String organizationId, String userId, List remoteGroupIds, + String authenticationOriginId, String token) throws CatalogException { ObjectMap auditParams = new ObjectMap() + .append("organizationId", organizationId) .append("userId", userId) .append("remoteGroupIds", remoteGroupIds) .append("authenticationOriginId", authenticationOriginId) .append("token", token); - String authenticatedUser = catalogManager.getUserManager().getUserId(token); + JwtPayload jwtPayload = catalogManager.getUserManager().validateToken(token); try { - authorizationManager.checkIsInstallationAdministrator(authenticatedUser); + authorizationManager.checkIsOpencgaAdministrator(jwtPayload); // Check userId exists Query query = new Query(UserDBAdaptor.QueryParams.ID.key(), userId); - OpenCGAResult count = userDBAdaptor.count(query); + OpenCGAResult count = getUserDBAdaptor(organizationId).count(query); if (count.getNumMatches() == 0) { throw new CatalogException("User '" + userId + "' not found."); } - OpenCGAResult result = studyDBAdaptor.resyncUserWithSyncedGroups(userId, remoteGroupIds, authenticationOriginId); + OpenCGAResult result = getStudyDBAdaptor(organizationId).resyncUserWithSyncedGroups(userId, remoteGroupIds, + authenticationOriginId); - auditManager.audit(userId, Enums.Action.UPDATE_USERS_FROM_STUDY_GROUP, Enums.Resource.STUDY, "", "", "", "", auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.audit(organizationId, jwtPayload.getUserId(organizationId), Enums.Action.UPDATE_USERS_FROM_STUDY_GROUP, + Enums.Resource.STUDY, "", "", "", "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return result; } catch (CatalogException e) { - auditManager.audit(userId, Enums.Action.UPDATE_USERS_FROM_STUDY_GROUP, Enums.Resource.STUDY, "", "", "", "", auditParams, + auditManager.audit(organizationId, jwtPayload.getUserId(organizationId), Enums.Action.UPDATE_USERS_FROM_STUDY_GROUP, + Enums.Resource.STUDY, "", "", "", "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } + public OpenCGAResult getEffectivePermissions(String studyStr, List entryIdList, String category, String token) + throws CatalogException { + return getEffectivePermissions(studyStr, entryIdList, Collections.emptyList(), category, token); + } + + public OpenCGAResult getEffectivePermissions(String studyStr, List entryIdList, List permissionList, + String category, String token) throws CatalogException { + StopWatch stopWatch = StopWatch.createStarted(); + ObjectMap auditParams = new ObjectMap() + .append("studyStr", studyStr) + .append("entryIdList", entryIdList) + .append("permissionList", permissionList) + .append("category", category) + .append("token", token); + JwtPayload jwtPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, jwtPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = jwtPayload.getUserId(organizationId); + try { + Study study = catalogManager.getStudyManager().resolveId(studyFqn, null, jwtPayload); + authorizationManager.checkIsAtLeastStudyAdministrator(organizationId, study.getUid(), userId); + + ParamUtils.checkParameter(category, "category"); + ParamUtils.checkNotEmptyArray(entryIdList, "entry id list"); + + Enums.Resource resource; + try { + resource = Enums.Resource.valueOf(category.toUpperCase()); + } catch (IllegalArgumentException e) { + String allowedResources = Arrays.stream(Enums.Resource.values()).map(Enum::name).collect(Collectors.joining(", ")); + throw new CatalogParameterException("Unexpected category '" + category + "' passed. Allowed categories are: " + + allowedResources); + } + List effectivePermissions = authorizationManager.getEffectivePermissions(organizationId, study.getUid(), entryIdList, + permissionList, resource); + auditManager.audit(organizationId, userId, Enums.Action.FETCH_ACLS, Enums.Resource.STUDY, "", "", "", "", auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + return new OpenCGAResult<>((int) stopWatch.getTime(TimeUnit.MILLISECONDS), effectivePermissions); + } catch (CatalogException e) { + auditManager.audit(organizationId, userId, Enums.Action.FETCH_ACLS, Enums.Resource.STUDY, "", "", "", "", auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + throw e; + } + } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AnnotationSetManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AnnotationSetManager.java index cf3e601e498..22984987b8c 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AnnotationSetManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AnnotationSetManager.java @@ -17,11 +17,11 @@ package org.opencb.opencga.catalog.managers; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.commons.utils.ListUtils; import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager; import org.opencb.opencga.catalog.db.DBAdaptorFactory; import org.opencb.opencga.catalog.db.api.AnnotationSetDBAdaptor; @@ -30,9 +30,11 @@ import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.utils.AnnotationUtils; +import org.opencb.opencga.catalog.utils.CatalogFqn; import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.config.Configuration; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.PrivateStudyUid; import org.opencb.opencga.core.models.common.Annotable; import org.opencb.opencga.core.models.common.AnnotationSet; @@ -41,7 +43,10 @@ import org.opencb.opencga.core.models.file.File; import org.opencb.opencga.core.models.file.FileCreateParams; import org.opencb.opencga.core.models.job.Job; -import org.opencb.opencga.core.models.study.*; +import org.opencb.opencga.core.models.study.Study; +import org.opencb.opencga.core.models.study.StudyPermissions; +import org.opencb.opencga.core.models.study.Variable; +import org.opencb.opencga.core.models.study.VariableSet; import org.opencb.opencga.core.response.OpenCGAResult; import java.io.IOException; @@ -95,8 +100,6 @@ protected void validateNewAnnotationSets(List variableSetList, List Iterator iterator = annotationSetList.iterator(); while (iterator.hasNext()) { AnnotationSet annotationSet = iterator.next(); - String annotationSetName = annotationSet.getId(); - ParamUtils.checkAlias(annotationSetName, "annotationSetName"); // Get the variable set if (!variableSetMap.containsKey(annotationSet.getVariableSetId())) { @@ -104,6 +107,14 @@ protected void validateNewAnnotationSets(List variableSetList, List } VariableSet variableSet = variableSetMap.get(annotationSet.getVariableSetId()); + if (variableSet.isUnique() && StringUtils.isEmpty(annotationSet.getId())) { + // If no annotation set id is provided, replicate the variable set id + annotationSet.setId(variableSet.getId()); + } + + String annotationSetId = annotationSet.getId(); + ParamUtils.checkAlias(annotationSetId, "annotationSet.id"); + // Check validity of annotations and duplicities AnnotationUtils.checkAnnotationSet(variableSet, annotationSet, consideredAnnotationSetsList, true); @@ -114,8 +125,9 @@ protected void validateNewAnnotationSets(List variableSetList, List public OpenCGAResult loadTsvAnnotations(String studyStr, String variableSetId, String path, TsvAnnotationParams tsvParams, ObjectMap params, String toolId, String token) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(token); - Study study = catalogManager.getStudyManager().resolveId(studyStr, userId, StudyManager.INCLUDE_VARIABLE_SET); + JwtPayload payload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, payload); + Study study = catalogManager.getStudyManager().resolveId(studyFqn, StudyManager.INCLUDE_VARIABLE_SET, payload); ParamUtils.checkObj(variableSetId, "VariableSetId"); ParamUtils.checkObj(tsvParams, "AnnotationTsvParams"); @@ -144,8 +156,8 @@ public OpenCGAResult loadTsvAnnotations(String studyStr, String variableSet } Query query = new Query(FileDBAdaptor.QueryParams.PATH.key(), path); - OpenCGAResult search = catalogManager.getFileManager().search(study.getFqn(), query, FileManager.INCLUDE_FILE_URI_PATH, - token); + OpenCGAResult search = catalogManager.getFileManager().search(study.getFqn(), query, + FileManager.INCLUDE_FILE_URI_PATH, token); if (search.getNumResults() == 0) { // File not found under the path. User must have provided a content so we can create the file. if (StringUtils.isEmpty(tsvParams.getContent())) { @@ -172,8 +184,8 @@ public OpenCGAResult loadTsvAnnotations(String studyStr, String variableSet return catalogManager.getJobManager().submit(study.getFqn(), toolId, Enums.Priority.MEDIUM, jobParams, token); } - protected void checkUpdateAnnotations(Study study, T entry, ObjectMap parameters, QueryOptions options, - VariableSet.AnnotableDataModels annotableEntity, + protected void checkUpdateAnnotations(String organizationId, Study study, T entry, ObjectMap parameters, + QueryOptions options, VariableSet.AnnotableDataModels annotableEntity, AnnotationSetDBAdaptor dbAdaptor, String user) throws CatalogException { List variableSetList = study.getVariableSets(); @@ -241,7 +253,7 @@ protected void checkUpdateAnnotations(Study study, T entry // Validate annotable data model VariableSet variableSet = variableSetMap.get(annotationSet.getVariableSetId()); - if (ListUtils.isNotEmpty(variableSet.getEntities()) + if (CollectionUtils.isNotEmpty(variableSet.getEntities()) && !variableSet.getEntities().contains(annotableEntity)) { throw new CatalogException("Cannot annotate " + annotableEntity + " using VariableSet '" + variableSet.getId() + "'. VariableSet is intended only for '" @@ -249,11 +261,10 @@ protected void checkUpdateAnnotations(Study study, T entry } // Create new annotationSet - if (variableSet.isConfidential()) { if (!confidentialPermissionsChecked) { - authorizationManager.checkStudyPermission(study.getUid(), user, - StudyPermissions.Permissions.CONFIDENTIAL_VARIABLE_SET_ACCESS, ""); + authorizationManager.checkStudyPermission(organizationId, study.getUid(), user, + StudyPermissions.Permissions.CONFIDENTIAL_VARIABLE_SET_ACCESS); confidentialPermissionsChecked = true; } } @@ -267,7 +278,7 @@ protected void checkUpdateAnnotations(Study study, T entry // Add the new annotationSet to the list of annotations to be updated finalAnnotationList.add(annotationSet); } else { - throw new CatalogException("Neither the annotationSetName nor the variableSetId matches an existing " + throw new CatalogException("Neither the annotationSetId nor the variableSetId matches an existing " + "AnnotationSet to perform an update or a VariableSet to create a new annotation."); } } @@ -350,7 +361,7 @@ protected void checkUpdateAnnotations(Study study, T entry } // Obtain all the variable sets from the study - OpenCGAResult studyDataResult = studyDBAdaptor.get(study.getUid(), + OpenCGAResult studyDataResult = getStudyDBAdaptor(organizationId).get(study.getUid(), new QueryOptions(QueryOptions.INCLUDE, StudyDBAdaptor.QueryParams.VARIABLE_SET.key())); if (studyDataResult.getNumResults() == 0) { diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AuditManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AuditManager.java index b95d3b0ff0e..6c25baa36ff 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AuditManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AuditManager.java @@ -20,18 +20,20 @@ import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.core.models.audit.AuditRecord; import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager; import org.opencb.opencga.catalog.db.DBAdaptorFactory; import org.opencb.opencga.catalog.db.api.AuditDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.utils.CatalogFqn; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.catalog.utils.UuidUtils; import org.opencb.opencga.core.common.GitRepositoryState; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.Configuration; +import org.opencb.opencga.core.models.JwtPayload; +import org.opencb.opencga.core.models.audit.AuditRecord; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.study.Study; import org.opencb.opencga.core.response.OpenCGAResult; @@ -51,26 +53,26 @@ public class AuditManager { private final CatalogManager catalogManager; private final AuthorizationManager authorizationManager; - private final AuditDBAdaptor auditDBAdaptor; + private final DBAdaptorFactory dbAdaptorFactory; private final Map> auditRecordMap; private static final int MAX_BATCH_SIZE = 100; - public AuditManager(AuthorizationManager authorizationManager, CatalogManager catalogManager, DBAdaptorFactory catalogDBAdaptorFactory, + public AuditManager(AuthorizationManager authorizationManager, CatalogManager catalogManager, DBAdaptorFactory dbAdaptorFactory, Configuration configuration) { this.catalogManager = catalogManager; this.authorizationManager = authorizationManager; - this.auditDBAdaptor = catalogDBAdaptorFactory.getCatalogAuditDbAdaptor(); + this.dbAdaptorFactory = dbAdaptorFactory; this.auditRecordMap = new HashMap<>(); } - public void audit(AuditRecord auditRecord) throws CatalogException { - auditDBAdaptor.insertAuditRecord(auditRecord); + public void audit(String organizationId, AuditRecord auditRecord) throws CatalogException { + dbAdaptorFactory.getCatalogAuditDbAdaptor(organizationId).insertAuditRecord(auditRecord); } - public void audit(List auditRecordList) throws CatalogException { + public void audit(String organizationId, List auditRecordList) throws CatalogException { for (AuditRecord auditRecord : auditRecordList) { - auditDBAdaptor.insertAuditRecord(auditRecord); + dbAdaptorFactory.getCatalogAuditDbAdaptor(organizationId).insertAuditRecord(auditRecord); } } @@ -78,13 +80,13 @@ public void initAuditBatch(String operationId) { this.auditRecordMap.put(operationId, new LinkedList<>()); } - public void finishAuditBatch(String operationId) throws CatalogException { + public void finishAuditBatch(String organizationId, String operationId) throws CatalogException { if (!this.auditRecordMap.containsKey(operationId)) { throw new CatalogException("Cannot audit. Operation id '" + operationId + "' not found."); } try { if (!this.auditRecordMap.get(operationId).isEmpty()) { - auditDBAdaptor.insertAuditRecords(this.auditRecordMap.get(operationId)); + dbAdaptorFactory.getCatalogAuditDbAdaptor(organizationId).insertAuditRecords(this.auditRecordMap.get(operationId)); } } catch (CatalogDBException e) { logger.error("Could not audit operation '{}' -> Error: {}", operationId, e.getMessage(), e); @@ -93,97 +95,111 @@ public void finishAuditBatch(String operationId) throws CatalogException { } } - public void auditCreate(String userId, Enums.Resource resource, String resourceId, String resourceUuid, String studyId, - String studyUuid, ObjectMap params, AuditRecord.Status status) { - audit(userId, Enums.Action.CREATE, resource, resourceId, resourceUuid, studyId, studyUuid, params, status, new ObjectMap()); - } - - public void auditCreate(String userId, Enums.Action action, Enums.Resource resource, String resourceId, String resourceUuid, + public void auditCreate(String organizationId, String userId, Enums.Resource resource, String resourceId, String resourceUuid, String studyId, String studyUuid, ObjectMap params, AuditRecord.Status status) { - String operationUuid = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); - audit(operationUuid, userId, action, resource, resourceId, resourceUuid, studyId, studyUuid, params, status, new ObjectMap()); + audit(organizationId, userId, Enums.Action.CREATE, resource, resourceId, resourceUuid, studyId, studyUuid, params, status, + new ObjectMap()); } - public void auditUpdate(String operationId, String userId, Enums.Resource resource, String resourceId, String resourceUuid, - String studyId, String studyUuid, ObjectMap params, AuditRecord.Status status) { - audit(operationId, userId, Enums.Action.UPDATE, resource, resourceId, resourceUuid, studyId, studyUuid, params, status, + public void auditCreate(String organizationId, String userId, Enums.Action action, Enums.Resource resource, String resourceId, + String resourceUuid, String studyId, String studyUuid, ObjectMap params, AuditRecord.Status status) { + String operationUuid = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); + audit(organizationId, operationUuid, userId, action, resource, resourceId, resourceUuid, studyId, studyUuid, params, status, new ObjectMap()); } - public void auditUpdate(String userId, Enums.Resource resource, String resourceId, String resourceUuid, String studyId, - String studyUuid, ObjectMap params, AuditRecord.Status status) { - audit(userId, Enums.Action.UPDATE, resource, resourceId, resourceUuid, studyId, studyUuid, params, status, new ObjectMap()); + public void auditUpdate(String organizationId, String operationId, String userId, Enums.Resource resource, String resourceId, + String resourceUuid, String studyId, String studyUuid, ObjectMap params, AuditRecord.Status status) { + audit(organizationId, operationId, userId, Enums.Action.UPDATE, resource, resourceId, resourceUuid, studyId, studyUuid, params, + status, new ObjectMap()); } - public void auditDelete(String userId, Enums.Resource resource, String resourceId, String resourceUuid, String studyId, - String studyUuid, ObjectMap params, AuditRecord.Status status) { - audit(userId, Enums.Action.UPDATE, resource, resourceId, resourceUuid, studyId, studyUuid, params, status, new ObjectMap()); + public void auditUpdate(String organizationId, String userId, Enums.Resource resource, String resourceId, String resourceUuid, + String studyId, String studyUuid, ObjectMap params, AuditRecord.Status status) { + audit(organizationId, userId, Enums.Action.UPDATE, resource, resourceId, resourceUuid, studyId, studyUuid, params, status, + new ObjectMap()); } - public void auditDelete(String operationId, String userId, Enums.Resource resource, String resourceId, String resourceUuid, + public void auditDelete(String organizationId, String userId, Enums.Resource resource, String resourceId, String resourceUuid, String studyId, String studyUuid, ObjectMap params, AuditRecord.Status status) { - audit(operationId, userId, Enums.Action.UPDATE, resource, resourceId, resourceUuid, studyId, studyUuid, params, status, + audit(organizationId, userId, Enums.Action.UPDATE, resource, resourceId, resourceUuid, studyId, studyUuid, params, status, new ObjectMap()); } - public void auditUser(String userId, Enums.Action action, String resourceId, AuditRecord.Status status) { - audit(userId, action, Enums.Resource.USER, resourceId, "", "", "", new ObjectMap(), status, new ObjectMap()); + public void auditDelete(String organizationId, String operationId, String userId, Enums.Resource resource, String resourceId, + String resourceUuid, String studyId, String studyUuid, ObjectMap params, AuditRecord.Status status) { + audit(organizationId, operationId, userId, Enums.Action.UPDATE, resource, resourceId, resourceUuid, studyId, studyUuid, params, + status, new ObjectMap()); } - public void auditUser(String userId, Enums.Action action, String resourceId, ObjectMap params, AuditRecord.Status status) { - audit(userId, action, Enums.Resource.USER, resourceId, "", "", "", params, status, new ObjectMap()); + public void auditUser(String organizationId, String userId, Enums.Action action, String resourceId, AuditRecord.Status status) { + audit(organizationId, userId, action, Enums.Resource.USER, resourceId, "", "", "", new ObjectMap(), status, new ObjectMap()); } - public void auditInfo(String userId, Enums.Resource resource, String resourceId, String resourceUuid, String studyId, - String studyUuid, ObjectMap params, AuditRecord.Status status) { - audit(userId, Enums.Action.INFO, resource, resourceId, resourceUuid, studyId, studyUuid, params, status, new ObjectMap()); + public void auditUser(String organizationId, String userId, Enums.Action action, String resourceId, ObjectMap params, + AuditRecord.Status status) { + audit(organizationId, userId, action, Enums.Resource.USER, resourceId, "", "", "", params, status, new ObjectMap()); } - public void auditInfo(String operationId, String userId, Enums.Resource resource, String resourceId, String resourceUuid, + public void auditInfo(String organizationId, String userId, Enums.Resource resource, String resourceId, String resourceUuid, String studyId, String studyUuid, ObjectMap params, AuditRecord.Status status) { - audit(operationId, userId, Enums.Action.INFO, resource, resourceId, resourceUuid, studyId, studyUuid, params, status, + audit(organizationId, userId, Enums.Action.INFO, resource, resourceId, resourceUuid, studyId, studyUuid, params, status, new ObjectMap()); } - public void auditSearch(String userId, Enums.Resource resource, String studyId, String studyUuid, ObjectMap params, - AuditRecord.Status status) { - audit(userId, Enums.Action.SEARCH, resource, "", "", studyId, studyUuid, params, status, new ObjectMap()); + public void auditInfo(String organizationId, String operationId, String userId, Enums.Resource resource, String resourceId, + String resourceUuid, String studyId, String studyUuid, ObjectMap params, AuditRecord.Status status) { + audit(organizationId, operationId, userId, Enums.Action.INFO, resource, resourceId, resourceUuid, studyId, studyUuid, params, + status, new ObjectMap()); } - public void auditCount(String userId, Enums.Resource resource, String studyId, String studyUuid, ObjectMap params, - AuditRecord.Status status) { - audit(userId, Enums.Action.COUNT, resource, "", "", studyId, studyUuid, params, status, new ObjectMap()); + public void auditSearch(String organizationId, String userId, Enums.Resource resource, String studyId, String studyUuid, + ObjectMap params, AuditRecord.Status status) { + auditSearch(organizationId, userId, resource, studyId, studyUuid, params, status, new ObjectMap()); } - public void auditDistinct(String userId, Enums.Resource resource, String studyId, String studyUuid, ObjectMap params, - AuditRecord.Status status) { - audit(userId, Enums.Action.DISTINCT, resource, "", "", studyId, studyUuid, params, status, new ObjectMap()); + public void auditSearch(String organizationId, String userId, Enums.Resource resource, String studyId, String studyUuid, + ObjectMap params, AuditRecord.Status status, ObjectMap attributes) { + audit(organizationId, userId, Enums.Action.SEARCH, resource, "", "", studyId, studyUuid, params, status, attributes); } - public void auditFacet(String userId, Enums.Resource resource, String studyId, String studyUuid, ObjectMap params, - AuditRecord.Status status) { - audit(userId, Enums.Action.FACET, resource, "", "", studyUuid, studyId, params, status, new ObjectMap()); + public void auditCount(String organizationId, String userId, Enums.Resource resource, String studyId, String studyUuid, + ObjectMap params, AuditRecord.Status status) { + audit(organizationId, userId, Enums.Action.COUNT, resource, "", "", studyId, studyUuid, params, status, new ObjectMap()); } - public void audit(String userId, Enums.Action action, Enums.Resource resource, String resourceId, String resourceUuid, - String studyId, String studyUuid, ObjectMap params, AuditRecord.Status status) { - audit(userId, action, resource, resourceId, resourceUuid, studyId, studyUuid, params, status, new ObjectMap()); + public void auditDistinct(String organizationId, String userId, Enums.Resource resource, String studyId, String studyUuid, + ObjectMap params, AuditRecord.Status status) { + audit(organizationId, userId, Enums.Action.DISTINCT, resource, "", "", studyId, studyUuid, params, status, new ObjectMap()); } - public void audit(String userId, Enums.Action action, Enums.Resource resource, String resourceId, String resourceUuid, - String studyId, String studyUuid, ObjectMap params, AuditRecord.Status status, ObjectMap attributes) { - audit(UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT), userId, action, resource, resourceId, resourceUuid, studyId, studyUuid, - params, status, attributes); + public void auditFacet(String organizationId, String userId, Enums.Resource resource, String studyId, String studyUuid, + ObjectMap params, AuditRecord.Status status) { + audit(organizationId, userId, Enums.Action.FACET, resource, "", "", studyUuid, studyId, params, status, new ObjectMap()); } - public void audit(String operationId, String userId, Enums.Action action, Enums.Resource resource, String resourceId, + public void audit(String organizationId, String userId, Enums.Action action, Enums.Resource resource, String resourceId, String resourceUuid, String studyId, String studyUuid, ObjectMap params, AuditRecord.Status status) { - audit(operationId, userId, action, resource, resourceId, resourceUuid, studyId, studyUuid, params, status, new ObjectMap()); + audit(organizationId, userId, action, resource, resourceId, resourceUuid, studyId, studyUuid, params, status, new ObjectMap()); } - public void audit(String operationId, String userId, Enums.Action action, Enums.Resource resource, String resourceId, + public void audit(String organizationId, String userId, Enums.Action action, Enums.Resource resource, String resourceId, String resourceUuid, String studyId, String studyUuid, ObjectMap params, AuditRecord.Status status, ObjectMap attributes) { + audit(organizationId, UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT), userId, action, resource, resourceId, resourceUuid, + studyId, studyUuid, params, status, attributes); + } + + public void audit(String organizationId, String operationId, String userId, Enums.Action action, Enums.Resource resource, + String resourceId, String resourceUuid, String studyId, String studyUuid, ObjectMap params, + AuditRecord.Status status) { + audit(organizationId, operationId, userId, action, resource, resourceId, resourceUuid, studyId, studyUuid, params, status, + new ObjectMap()); + } + + public void audit(String organizationId, String operationId, String userId, Enums.Action action, Enums.Resource resource, + String resourceId, String resourceUuid, String studyId, String studyUuid, ObjectMap params, AuditRecord.Status status, + ObjectMap attributes) { String apiVersion = GitRepositoryState.getInstance().getBuildVersion(); Date date = TimeUtils.getDate(); @@ -197,8 +213,10 @@ public void audit(String operationId, String userId, Enums.Action action, Enums. if (this.auditRecordMap.size() == MAX_BATCH_SIZE) { try { - auditDBAdaptor.insertAuditRecords(this.auditRecordMap.get(operationId)); + dbAdaptorFactory.getCatalogAuditDbAdaptor(organizationId).insertAuditRecords(this.auditRecordMap.get(operationId)); } catch (CatalogDBException e) { + // FIXME : How to raise attention on this silent error? + // This is a critical error that should not happen. logger.error("Could not audit operation '{}' -> Error: {}", operationId, e.getMessage(), e); } finally { this.auditRecordMap.get(operationId).clear(); @@ -206,8 +224,10 @@ public void audit(String operationId, String userId, Enums.Action action, Enums. } } else { try { - auditDBAdaptor.insertAuditRecord(auditRecord); - } catch (CatalogDBException e) { + dbAdaptorFactory.getCatalogAuditDbAdaptor(organizationId).insertAuditRecord(auditRecord); + } catch (CatalogDBException | RuntimeException e) { + // FIXME : How to raise attention on this silent error? + // This is a critical error that should not happen. logger.error("Could not audit '{}' -> Error: {}", auditRecord, e.getMessage(), e); } } @@ -217,8 +237,11 @@ public OpenCGAResult search(String studyStr, Query query, QueryOpti query = ParamUtils.defaultObject(query, Query::new); options = ParamUtils.defaultObject(options, QueryOptions::new); - String userId = catalogManager.getUserManager().getUserId(token); - Study study = catalogManager.getStudyManager().resolveId(studyStr, userId); + JwtPayload payload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, payload); + String organizationId = studyFqn.getOrganizationId(); + String userId = payload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyFqn, null, payload); ObjectMap auditParams = new ObjectMap() .append("studyId", studyStr) @@ -226,18 +249,18 @@ public OpenCGAResult search(String studyStr, Query query, QueryOpti .append("options", options) .append("token", token); try { - authorizationManager.checkIsOwnerOrAdmin(study.getUid(), userId); + authorizationManager.checkIsAtLeastStudyAdministrator(organizationId, study.getUid(), userId); query.remove(AuditDBAdaptor.QueryParams.STUDY_ID.key()); query.put(AuditDBAdaptor.QueryParams.STUDY_UUID.key(), study.getUuid()); - OpenCGAResult result = auditDBAdaptor.get(query, options); + OpenCGAResult result = dbAdaptorFactory.getCatalogAuditDbAdaptor(organizationId).get(query, options); - auditSearch(userId, Enums.Resource.AUDIT, study.getId(), study.getUuid(), auditParams, + auditSearch(organizationId, userId, Enums.Resource.AUDIT, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return result; } catch (CatalogException e) { - auditSearch(userId, Enums.Resource.AUDIT, study.getId(), study.getUuid(), auditParams, + auditSearch(organizationId, userId, Enums.Resource.AUDIT, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -261,10 +284,12 @@ public OpenCGAResult groupBy(Query query, String fields, QueryOptions options, S } public OpenCGAResult groupBy(Query query, List fields, QueryOptions options, String token) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(token); - if (authorizationManager.isInstallationAdministrator(userId)) { - return auditDBAdaptor.groupBy(query, fields, options); + JwtPayload payload = catalogManager.getUserManager().validateToken(token); + String organizationId = payload.getOrganization(); + String userId = payload.getUserId(); + if (authorizationManager.isAtLeastOrganizationOwnerOrAdmin(organizationId, userId)) { + return dbAdaptorFactory.getCatalogAuditDbAdaptor(organizationId).groupBy(query, fields, options); } - throw new CatalogAuthorizationException("Only root of OpenCGA can query the audit database"); + throw CatalogAuthorizationException.notOrganizationOwnerOrAdmin("query the audit database"); } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java index 855c396b74f..030505395d0 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java @@ -19,10 +19,15 @@ import org.apache.commons.lang3.StringUtils; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.catalog.auth.authentication.CatalogAuthenticationManager; import org.opencb.opencga.catalog.auth.authentication.JwtManager; +import org.opencb.opencga.catalog.auth.authentication.azure.AuthenticationFactory; +import org.opencb.opencga.catalog.auth.authorization.AuthorizationDBAdaptorFactory; import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager; +import org.opencb.opencga.catalog.auth.authorization.AuthorizationMongoDBAdaptorFactory; import org.opencb.opencga.catalog.auth.authorization.CatalogAuthorizationManager; import org.opencb.opencga.catalog.db.DBAdaptorFactory; +import org.opencb.opencga.catalog.db.api.OrganizationDBAdaptor; import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; @@ -31,14 +36,18 @@ import org.opencb.opencga.catalog.io.CatalogIOManager; import org.opencb.opencga.catalog.io.IOManagerFactory; import org.opencb.opencga.catalog.migration.MigrationManager; +import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.JwtUtils; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.common.PasswordUtils; import org.opencb.opencga.core.common.UriUtils; -import org.opencb.opencga.core.config.Admin; import org.opencb.opencga.core.config.Configuration; +import org.opencb.opencga.core.config.Optimizations; +import org.opencb.opencga.core.models.JwtPayload; +import org.opencb.opencga.core.models.organizations.*; +import org.opencb.opencga.core.models.project.ProjectCreateParams; +import org.opencb.opencga.core.models.project.ProjectOrganism; import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.models.user.Account; import org.opencb.opencga.core.models.user.User; import org.opencb.opencga.core.response.OpenCGAResult; import org.slf4j.Logger; @@ -47,20 +56,26 @@ import java.net.URISyntaxException; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; import static org.opencb.opencga.catalog.managers.AbstractManager.OPENCGA; -import static org.opencb.opencga.core.api.ParamConstants.ADMIN_PROJECT; -import static org.opencb.opencga.core.api.ParamConstants.ADMIN_STUDY; +import static org.opencb.opencga.core.api.ParamConstants.*; public class CatalogManager implements AutoCloseable { protected static Logger logger = LoggerFactory.getLogger(CatalogManager.class); private DBAdaptorFactory catalogDBAdaptorFactory; + private AuthorizationDBAdaptorFactory authorizationDBAdaptorFactory; private IOManagerFactory ioManagerFactory; private CatalogIOManager catalogIOManager; + private AuthenticationFactory authenticationFactory; private AdminManager adminManager; + private NoteManager noteManager; + private OrganizationManager organizationManager; private UserManager userManager; private ProjectManager projectManager; private StudyManager studyManager; @@ -83,29 +98,57 @@ public class CatalogManager implements AutoCloseable { public CatalogManager(Configuration configuration) throws CatalogException { this.configuration = configuration; + init(); + } + + private void init() throws CatalogException { logger.debug("CatalogManager configureIOManager"); configureIOManager(configuration); logger.debug("CatalogManager configureDBAdaptorFactory"); catalogDBAdaptorFactory = new MongoDBAdaptorFactory(configuration, ioManagerFactory); + authorizationDBAdaptorFactory = new AuthorizationMongoDBAdaptorFactory((MongoDBAdaptorFactory) catalogDBAdaptorFactory, + configuration); + authenticationFactory = new AuthenticationFactory(catalogDBAdaptorFactory, configuration); logger.debug("CatalogManager configureManager"); configureManagers(configuration); } - public String getCatalogDatabase() { - return catalogDBAdaptorFactory.getCatalogDatabase(configuration.getDatabasePrefix()); + public String getCatalogDatabase(String organizationId) { + return catalogDBAdaptorFactory.getCatalogDatabase(configuration.getDatabasePrefix(), organizationId); + } + + public String getCatalogAdminDatabase() { + return getCatalogDatabase(ADMIN_ORGANIZATION); + } + + public List getCatalogDatabaseNames() throws CatalogDBException { + List databaseNames = new LinkedList<>(); + for (String organizationId : catalogDBAdaptorFactory.getOrganizationIds()) { + databaseNames.add(getCatalogDatabase(organizationId)); + } + return databaseNames; } private void configureManagers(Configuration configuration) throws CatalogException { -// catalogClient = new CatalogDBClient(this); - //TODO: Check if catalog is empty - //TODO: Setup catalog if it's empty. - this.initializeAdmin(configuration); - authorizationManager = new CatalogAuthorizationManager(this.catalogDBAdaptorFactory, configuration); + initializeAdmin(configuration); + for (String organizationId : catalogDBAdaptorFactory.getOrganizationIds()) { + QueryOptions options = new QueryOptions(OrganizationManager.INCLUDE_ORGANIZATION_CONFIGURATION); + options.put(OrganizationDBAdaptor.IS_ORGANIZATION_ADMIN_OPTION, true); + Organization organization = catalogDBAdaptorFactory.getCatalogOrganizationDBAdaptor(organizationId).get(options).first(); + if (organization != null) { + authenticationFactory.configureOrganizationAuthenticationManager(organization); + } + } + authorizationManager = new CatalogAuthorizationManager(catalogDBAdaptorFactory, authorizationDBAdaptorFactory); auditManager = new AuditManager(authorizationManager, this, this.catalogDBAdaptorFactory, configuration); migrationManager = new MigrationManager(this, catalogDBAdaptorFactory, configuration); + noteManager = new NoteManager(authorizationManager, auditManager, this, catalogDBAdaptorFactory, configuration); adminManager = new AdminManager(authorizationManager, auditManager, this, catalogDBAdaptorFactory, catalogIOManager, configuration); - userManager = new UserManager(authorizationManager, auditManager, this, catalogDBAdaptorFactory, catalogIOManager, configuration); + organizationManager = new OrganizationManager(authorizationManager, auditManager, this, catalogDBAdaptorFactory, catalogIOManager, + authenticationFactory, configuration); + userManager = new UserManager(authorizationManager, auditManager, this, catalogDBAdaptorFactory, catalogIOManager, + authenticationFactory, configuration); projectManager = new ProjectManager(authorizationManager, auditManager, this, catalogDBAdaptorFactory, catalogIOManager, configuration); studyManager = new StudyManager(authorizationManager, auditManager, this, catalogDBAdaptorFactory, ioManagerFactory, @@ -123,38 +166,41 @@ private void configureManagers(Configuration configuration) throws CatalogExcept } private void initializeAdmin(Configuration configuration) throws CatalogDBException { - if (configuration.getAdmin() == null) { - configuration.setAdmin(new Admin()); - } - - String secretKey = ParamUtils.defaultString(configuration.getAdmin().getSecretKey(), - PasswordUtils.getStrongRandomPassword(JwtManager.SECRET_KEY_MIN_LENGTH)); - String algorithm = ParamUtils.defaultString(configuration.getAdmin().getAlgorithm(), "HS256"); - if (existsCatalogDB()) { - secretKey = catalogDBAdaptorFactory.getCatalogMetaDBAdaptor().readSecretKey(); - algorithm = catalogDBAdaptorFactory.getCatalogMetaDBAdaptor().readAlgorithm(); - } - configuration.getAdmin().setAlgorithm(algorithm); - configuration.getAdmin().setSecretKey(secretKey); - } - - public void updateJWTParameters(ObjectMap params, String token) throws CatalogException { - if (!OPENCGA.equals(userManager.getUserId(token))) { - throw new CatalogException("Operation only allowed for the OpenCGA admin"); + // TODO: Each organization will have different configurations +// if (configuration.getAdmin() == null) { +// configuration.setAdmin(new Admin()); +// } +// +// String secretKey = ParamUtils.defaultString(configuration.getAdmin().getSecretKey(), +// PasswordUtils.getStrongRandomPassword(JwtManager.SECRET_KEY_MIN_LENGTH)); +// String algorithm = ParamUtils.defaultString(configuration.getAdmin().getAlgorithm(), "HS256"); +// if (existsCatalogDB()) { +// secretKey = catalogDBAdaptorFactory.getCatalogMetaDBAdaptor().readSecretKey(); +// algorithm = catalogDBAdaptorFactory.getCatalogMetaDBAdaptor().readAlgorithm(); +// } +// configuration.getAdmin().setAlgorithm(algorithm); +// configuration.getAdmin().setSecretKey(secretKey); + } + + public void updateJWTParameters(String organizationId, ObjectMap params, String token) throws CatalogException { + JwtPayload payload = userManager.validateToken(token); + String userId = payload.getUserId(); + if (!authorizationManager.isAtLeastOrganizationOwnerOrAdmin(organizationId, userId)) { + throw CatalogAuthorizationException.notOrganizationOwnerOrAdmin(); } - if (params == null || params.size() == 0) { + if (params == null || params.isEmpty()) { return; } - catalogDBAdaptorFactory.getCatalogMetaDBAdaptor().updateJWTParameters(params); + catalogDBAdaptorFactory.getCatalogMetaDBAdaptor(organizationId).updateJWTParameters(params); } - public boolean getDatabaseStatus() { + public boolean getDatabaseStatus() throws CatalogDBException { return catalogDBAdaptorFactory.getDatabaseStatus(); } - public boolean getCatalogDatabaseStatus() { + public boolean getCatalogDatabaseStatus() throws CatalogDBException { if (existsCatalogDB()) { return catalogDBAdaptorFactory.getDatabaseStatus(); } else { @@ -166,41 +212,37 @@ public boolean getCatalogDatabaseStatus() { * Checks if the database exists. * * @return true if the database exists. + * @throws CatalogDBException CatalogDBException */ - public boolean existsCatalogDB() { + public boolean existsCatalogDB() throws CatalogDBException { return catalogDBAdaptorFactory.isCatalogDBReady(); } - public void installCatalogDB(String secretKey, String password, String email, String organization, boolean force) - throws CatalogException { - installCatalogDB(secretKey, password, email, organization, force, false); - } - - public void installCatalogDB(String secretKey, String password, String email, String organization, boolean force, boolean test) - throws CatalogException { + public void installCatalogDB(String algorithm, String secretKey, String password, String email, boolean force) throws CatalogException { if (existsCatalogDB()) { if (force) { // The password of the old db should match the one to be used in the new installation. Otherwise, they can obtain the same // results calling first to "catalog delete" and then "catalog install" deleteCatalogDB(password); + init(); } else { // Check admin password ... try { userManager.loginAsAdmin(password); - logger.warn("A database called " + getCatalogDatabase() + " already exists"); + logger.warn("A database called {} already exists", getCatalogAdminDatabase()); return; } catch (CatalogException e) { - throw new CatalogException("A database called " + getCatalogDatabase() + " with a different admin" + throw new CatalogException("A database called " + getCatalogAdminDatabase() + " with a different admin" + " password already exists. If you are aware of that installation, please delete it first."); } } } try { - logger.info("Installing database {} in {}", getCatalogDatabase(), configuration.getCatalog().getDatabase().getHosts()); - privateInstall(secretKey, password, email, organization, test); + logger.info("Installing database {} in {}", getCatalogAdminDatabase(), configuration.getCatalog().getDatabase().getHosts()); + privateInstall(algorithm, secretKey, password, email); String token = userManager.loginAsAdmin(password).getToken(); - installIndexes(token); + installIndexes(ADMIN_ORGANIZATION, token); } catch (Exception e) { try { clearCatalog(); @@ -211,43 +253,65 @@ public void installCatalogDB(String secretKey, String password, String email, St } } - private void privateInstall(String secretKey, String password, String email, String organization, boolean test) - throws CatalogException { + private void privateInstall(String algorithm, String secretKey, String password, String email) throws CatalogException { if (existsCatalogDB()) { throw new CatalogException("Nothing to install. There already exists a catalog database"); } if (!PasswordUtils.isStrongPassword(password)) { throw new CatalogException("Invalid password. Check password strength for user "); } + if (StringUtils.isEmpty(secretKey)) { + logger.info("Generating secret key"); + secretKey = PasswordUtils.getStrongRandomPassword(JwtManager.SECRET_KEY_MIN_LENGTH); + } ParamUtils.checkParameter(secretKey, "secretKey"); ParamUtils.checkParameter(password, "password"); - JwtUtils.validateJWTKey(configuration.getAdmin().getAlgorithm(), secretKey); - configuration.getAdmin().setSecretKey(secretKey); + JwtUtils.validateJWTKey(algorithm, secretKey); - if (!test) { - catalogDBAdaptorFactory.createAllCollections(configuration); - } - catalogDBAdaptorFactory.initialiseMetaCollection(configuration.getAdmin()); catalogIOManager.createDefaultOpenCGAFolders(); - User user = new User(OPENCGA, new Account().setType(Account.AccountType.ADMINISTRATOR).setExpirationDate("")) + OrganizationConfiguration organizationConfiguration = new OrganizationConfiguration( + Collections.singletonList(CatalogAuthenticationManager.createOpencgaAuthenticationOrigin()), + Constants.DEFAULT_USER_EXPIRATION_DATE, new Optimizations(), new TokenConfiguration(algorithm, secretKey, 3600L)); + organizationManager.create(new OrganizationCreateParams(ADMIN_ORGANIZATION, ADMIN_ORGANIZATION, null, null, + organizationConfiguration, null), + QueryOptions.empty(), null); + + User user = new User(OPENCGA) .setEmail(StringUtils.isEmpty(email) ? "opencga@admin.com" : email) - .setOrganization(organization); + .setOrganization(ADMIN_ORGANIZATION); userManager.create(user, password, null); + String token = userManager.login(ADMIN_ORGANIZATION, OPENCGA, password).getToken(); - String token = userManager.login(OPENCGA, password).getToken(); - projectManager.create(ADMIN_PROJECT, ADMIN_PROJECT, "Default project", "", "", "", null, token); - studyManager.create(ADMIN_PROJECT, new Study().setId(ADMIN_STUDY).setDescription("Default study"), QueryOptions.empty(), token); + // Add OPENCGA as owner of ADMIN_ORGANIZATION + organizationManager.update(ADMIN_ORGANIZATION, new OrganizationUpdateParams().setOwner(OPENCGA), QueryOptions.empty(), token); + projectManager.create(new ProjectCreateParams().setId(ADMIN_PROJECT).setDescription("Default project") + .setOrganism(new ProjectOrganism("Homo sapiens", "grch38")), null, token); + studyManager.create(ADMIN_PROJECT, new Study().setId(ADMIN_STUDY).setDescription("Default study"), + QueryOptions.empty(), token); // Skip old available migrations - migrationManager.skipPendingMigrations(token); + migrationManager.skipPendingMigrations(ADMIN_ORGANIZATION, token); } public void installIndexes(String token) throws CatalogException { - if (!OPENCGA.equals(userManager.getUserId(token))) { - throw new CatalogAuthorizationException("Only the admin can install new indexes"); + JwtPayload payload = userManager.validateToken(token); + if (!authorizationManager.isOpencgaAdministrator(payload)) { + throw new CatalogException("Operation only allowed for the opencga administrator"); + } + for (String organizationId : organizationManager.getOrganizationIds(token)) { + catalogDBAdaptorFactory.createIndexes(organizationId); } - catalogDBAdaptorFactory.createIndexes(); + } + + public void installIndexes(String organizationId, String token) throws CatalogException { + JwtPayload payload = userManager.validateToken(token); + String userId = payload.getUserId(organizationId); + if (!authorizationManager.isAtLeastOrganizationOwnerOrAdmin(organizationId, userId)) { + throw CatalogAuthorizationException.notOrganizationOwnerOrAdmin(); + } + + catalogDBAdaptorFactory.createIndexes(organizationId); } public void deleteCatalogDB(String password) throws CatalogException { @@ -255,7 +319,8 @@ public void deleteCatalogDB(String password) throws CatalogException { userManager.loginAsAdmin(password); } catch (CatalogException e) { // Validate that the admin user exists. - OpenCGAResult result = catalogDBAdaptorFactory.getCatalogUserDBAdaptor().get(OPENCGA, QueryOptions.empty()); + OpenCGAResult result = catalogDBAdaptorFactory.getCatalogUserDBAdaptor(ADMIN_ORGANIZATION) + .get(OPENCGA, QueryOptions.empty()); if (result.getNumResults() == 1) { // Admin user exists so we have to fail. Password must be incorrect. throw e; @@ -314,6 +379,14 @@ public AdminManager getAdminManager() { return adminManager; } + public NoteManager getNotesManager() { + return noteManager; + } + + public OrganizationManager getOrganizationManager() { + return organizationManager; + } + public UserManager getUserManager() { return userManager; } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java index 7d22001ea66..20b228aeed8 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManager.java @@ -17,18 +17,20 @@ package org.opencb.opencga.catalog.managers; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectReader; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.StopWatch; import org.opencb.biodata.models.clinical.ClinicalAnalyst; import org.opencb.biodata.models.clinical.ClinicalAudit; import org.opencb.biodata.models.clinical.ClinicalComment; import org.opencb.biodata.models.clinical.Disorder; -import org.opencb.biodata.models.common.Status; import org.opencb.commons.datastore.core.Event; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.result.Error; +import org.opencb.commons.utils.FileUtils; import org.opencb.commons.utils.ListUtils; import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager; import org.opencb.opencga.catalog.db.DBAdaptorFactory; @@ -36,30 +38,31 @@ import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; +import org.opencb.opencga.catalog.models.ClinicalAnalysisLoadResult; import org.opencb.opencga.catalog.models.InternalGetDataResult; -import org.opencb.opencga.catalog.utils.AnnotationUtils; -import org.opencb.opencga.catalog.utils.Constants; -import org.opencb.opencga.catalog.utils.ParamUtils; -import org.opencb.opencga.catalog.utils.UuidUtils; +import org.opencb.opencga.catalog.utils.*; import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.common.GitRepositoryState; import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.models.AclEntryList; import org.opencb.opencga.core.models.AclParams; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.audit.AuditRecord; import org.opencb.opencga.core.models.clinical.*; -import org.opencb.opencga.core.models.common.AnnotationSet; -import org.opencb.opencga.core.models.common.Enums; -import org.opencb.opencga.core.models.common.FlagAnnotation; -import org.opencb.opencga.core.models.common.FlagValue; +import org.opencb.opencga.core.models.common.*; import org.opencb.opencga.core.models.family.Family; +import org.opencb.opencga.core.models.family.FamilyCreateParams; import org.opencb.opencga.core.models.file.File; import org.opencb.opencga.core.models.file.FileReferenceParam; import org.opencb.opencga.core.models.individual.Individual; +import org.opencb.opencga.core.models.individual.IndividualUpdateParams; import org.opencb.opencga.core.models.panel.Panel; import org.opencb.opencga.core.models.panel.PanelReferenceParam; import org.opencb.opencga.core.models.sample.Sample; +import org.opencb.opencga.core.models.sample.SampleCreateParams; +import org.opencb.opencga.core.models.sample.SampleReferenceParam; import org.opencb.opencga.core.models.study.Study; import org.opencb.opencga.core.models.study.StudyPermissions; import org.opencb.opencga.core.models.study.VariableSet; @@ -71,13 +74,18 @@ import org.slf4j.LoggerFactory; import javax.annotation.Nullable; +import java.io.BufferedReader; import java.io.IOException; +import java.nio.file.Path; import java.util.*; +import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; import static org.opencb.opencga.catalog.auth.authorization.CatalogAuthorizationManager.checkPermissions; +import static org.opencb.opencga.catalog.utils.ParamUtils.SaveInterpretationAs.PRIMARY; +import static org.opencb.opencga.catalog.utils.ParamUtils.SaveInterpretationAs.SECONDARY; import static org.opencb.opencga.core.common.JacksonUtils.getUpdateObjectMapper; /** @@ -138,9 +146,10 @@ Enums.Resource getEntity() { // } // // QueryOptions queryOptions = options != null ? new QueryOptions(options) : new QueryOptions(); -// OpenCGAResult analysisDataResult = clinicalDBAdaptor.get(studyUid, queryCopy, queryOptions, user); +// OpenCGAResult analysisDataResult = getClinicalAnalysisDBAdaptor(organizationId).get(studyUid, queryCopy, +// queryOptions, user); // if (analysisDataResult.getNumResults() == 0) { -// analysisDataResult = clinicalDBAdaptor.get(queryCopy, queryOptions); +// analysisDataResult = getClinicalAnalysisDBAdaptor(organizationId).get(queryCopy, queryOptions); // if (analysisDataResult.getNumResults() == 0) { // throw new CatalogException("Clinical Analysis '" + entry + "' not found"); // } else { @@ -155,7 +164,7 @@ Enums.Resource getEntity() { // } @Override - InternalGetDataResult internalGet(long studyUid, List entryList, @Nullable Query query, + InternalGetDataResult internalGet(String organizationId, long studyUid, List entryList, @Nullable Query query, QueryOptions options, String user, boolean ignoreException) throws CatalogException { if (ListUtils.isEmpty(entryList)) { @@ -188,13 +197,14 @@ InternalGetDataResult internalGet(long studyUid, List // Ensure the field by which we are querying for will be kept in the results queryOptions = keepFieldInQueryOptions(queryOptions, idQueryParam.key()); - OpenCGAResult analysisDataResult = clinicalDBAdaptor.get(studyUid, queryCopy, queryOptions, user); + OpenCGAResult analysisDataResult = getClinicalAnalysisDBAdaptor(organizationId).get(studyUid, queryCopy, + queryOptions, user); if (ignoreException || analysisDataResult.getNumResults() == uniqueList.size()) { return keepOriginalOrder(uniqueList, clinicalStringFunction, analysisDataResult, ignoreException, false); } // Query without adding the user check - OpenCGAResult resultsNoCheck = clinicalDBAdaptor.get(queryCopy, queryOptions); + OpenCGAResult resultsNoCheck = getClinicalAnalysisDBAdaptor(organizationId).get(queryCopy, queryOptions); if (resultsNoCheck.getNumResults() == analysisDataResult.getNumResults()) { throw CatalogException.notFound("clinical analyses", @@ -211,27 +221,34 @@ public DBIterator iterator(String studyStr, Query query, Query query = ParamUtils.defaultObject(query, Query::new); options = ParamUtils.defaultObject(options, QueryOptions::new); - String userId = catalogManager.getUserManager().getUserId(sessionId); - Study study = catalogManager.getStudyManager().resolveId(studyStr, userId, StudyManager.INCLUDE_VARIABLE_SET); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(sessionId); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyFqn, StudyManager.INCLUDE_VARIABLE_SET, tokenPayload); - fixQueryObject(study, query, userId, sessionId); + fixQueryObject(organizationId, study, query, userId, sessionId); AnnotationUtils.fixQueryOptionAnnotation(options); query.append(ClinicalAnalysisDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - return clinicalDBAdaptor.iterator(study.getUid(), query, options, userId); + return getClinicalAnalysisDBAdaptor(organizationId).iterator(study.getUid(), query, options, userId); } @Override - public OpenCGAResult create(String studyStr, ClinicalAnalysis clinicalAnalysis, QueryOptions options, String token) - throws CatalogException { + public OpenCGAResult create(String studyStr, ClinicalAnalysis clinicalAnalysis, + QueryOptions options, String token) throws CatalogException { return create(studyStr, clinicalAnalysis, null, options, token); } public OpenCGAResult create(String studyStr, ClinicalAnalysis clinicalAnalysis, Boolean skipCreateDefaultInterpretation, QueryOptions options, String token) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(token); - Study study = catalogManager.getStudyManager().resolveId(studyStr, userId, StudyManager.INCLUDE_VARIABLE_SET_AND_CONFIGURATION); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyFqn, StudyManager.INCLUDE_VARIABLE_SET_AND_CONFIGURATION, + tokenPayload); ObjectMap auditParams = new ObjectMap() .append("study", studyStr) @@ -246,7 +263,8 @@ public OpenCGAResult create(String studyStr, ClinicalAnalysis } ClinicalAnalysisStudyConfiguration clinicalConfiguration = study.getInternal().getConfiguration().getClinical(); - authorizationManager.checkStudyPermission(study.getUid(), userId, StudyPermissions.Permissions.WRITE_CLINICAL_ANALYSIS); + authorizationManager.checkStudyPermission(organizationId, study.getUid(), userId, + StudyPermissions.Permissions.WRITE_CLINICAL_ANALYSIS); options = ParamUtils.defaultObject(options, QueryOptions::new); ParamUtils.checkObj(clinicalAnalysis, "clinicalAnalysis"); @@ -256,7 +274,7 @@ public OpenCGAResult create(String studyStr, ClinicalAnalysis List events = new LinkedList<>(); - clinicalAnalysis.setStatus(ParamUtils.defaultObject(clinicalAnalysis.getStatus(), Status::new)); + clinicalAnalysis.setStatus(ParamUtils.defaultObject(clinicalAnalysis.getStatus(), ClinicalStatus::new)); clinicalAnalysis.setInternal(ClinicalAnalysisInternal.init()); clinicalAnalysis.setDisorder(ParamUtils.defaultObject(clinicalAnalysis.getDisorder(), new Disorder("", "", "", Collections.emptyMap(), "", Collections.emptyList()))); @@ -272,20 +290,20 @@ public OpenCGAResult create(String studyStr, ClinicalAnalysis clinicalAnalysis.setRequest(ParamUtils.defaultObject(clinicalAnalysis.getRequest(), ClinicalRequest::new)); // ---------- Check and init report fields - validateAndInitReport(study, clinicalAnalysis.getReport(), userId); + validateAndInitReport(organizationId, study, clinicalAnalysis.getReport(), userId); // ---------- Check and init responsible fields ClinicalResponsible responsible = clinicalAnalysis.getResponsible(); if (StringUtils.isEmpty(responsible.getId())) { responsible.setId(userId); } - fillResponsible(responsible); + fillResponsible(organizationId, responsible); // ---------- Check and init request fields ClinicalRequest request = clinicalAnalysis.getRequest(); if (StringUtils.isNotEmpty(request.getId())) { request.setDate(ParamUtils.checkDateOrGetCurrentDate(request.getDate(), "request.date")); - fillResponsible(request.getResponsible()); + fillResponsible(organizationId, request.getResponsible()); } if (clinicalAnalysis.getQualityControl().getComments() != null) { @@ -311,7 +329,7 @@ public OpenCGAResult create(String studyStr, ClinicalAnalysis Set panelIds = clinicalAnalysis.getPanels().stream().map(Panel::getId).collect(Collectors.toSet()); Query query = new Query(PanelDBAdaptor.QueryParams.ID.key(), panelIds); OpenCGAResult panelResult = - panelDBAdaptor.get(study.getUid(), query, PanelManager.INCLUDE_PANEL_IDS, userId); + getPanelDBAdaptor(organizationId).get(study.getUid(), query, PanelManager.INCLUDE_PANEL_IDS, userId); if (panelResult.getNumResults() < panelIds.size()) { throw new CatalogException("Some panels were not found or user doesn't have permissions to see them"); } @@ -324,19 +342,25 @@ public OpenCGAResult create(String studyStr, ClinicalAnalysis QueryOptions userInclude = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(UserDBAdaptor.QueryParams.ID.key(), UserDBAdaptor.QueryParams.NAME.key(), UserDBAdaptor.QueryParams.EMAIL.key())); if (clinicalAnalysis.getAnalysts() == null) { - userList = userDBAdaptor.get(userId, userInclude).getResults(); + userList = getUserDBAdaptor(organizationId).get(userId, userInclude).getResults(); } else { // Validate users Set userIds = new HashSet<>(); for (ClinicalAnalyst analyst : clinicalAnalysis.getAnalysts()) { - userIds.add(analyst.getId()); + if (StringUtils.isNotEmpty(analyst.getId())) { + userIds.add(analyst.getId()); + } } - Query query = new Query(UserDBAdaptor.QueryParams.ID.key(), userIds); - OpenCGAResult result = userDBAdaptor.get(query, userInclude); - if (result.getNumResults() < userIds.size()) { - throw new CatalogException("Some clinical analysts could not be found."); + if (CollectionUtils.isNotEmpty(userIds)) { + Query query = new Query(UserDBAdaptor.QueryParams.ID.key(), userIds); + OpenCGAResult result = getUserDBAdaptor(organizationId).get(query, userInclude); + if (result.getNumResults() < userIds.size()) { + throw new CatalogException("Some clinical analysts could not be found."); + } + userList = result.getResults(); + } else { + userList = getUserDBAdaptor(organizationId).get(userId, userInclude).getResults(); } - userList = result.getResults(); } List clinicalAnalystList = new ArrayList<>(userList.size()); for (User user : userList) { @@ -370,7 +394,8 @@ public OpenCGAResult create(String studyStr, ClinicalAnalysis if (member.getSamples() != null && !member.getSamples().isEmpty()) { Query query = new Query(SampleDBAdaptor.QueryParams.UID.key(), member.getSamples().stream().map(Sample::getUid).collect(Collectors.toList())); - OpenCGAResult sampleResult = sampleDBAdaptor.get(study.getUid(), query, new QueryOptions(), userId); + OpenCGAResult sampleResult = getSampleDBAdaptor(organizationId).get(study.getUid(), query, + new QueryOptions(), userId); member.setSamples(sampleResult.getResults()); } } @@ -492,8 +517,9 @@ public OpenCGAResult create(String studyStr, ClinicalAnalysis // Validate the proband has a disorder validateDisorder(clinicalAnalysis); } else { - OpenCGAResult individualOpenCGAResult = catalogManager.getIndividualManager().internalGet(study.getUid(), - clinicalAnalysis.getProband().getId(), new Query(), new QueryOptions(), userId); + OpenCGAResult individualOpenCGAResult = catalogManager.getIndividualManager() + .internalGet(organizationId, study.getUid(), clinicalAnalysis.getProband().getId(), new Query(), new QueryOptions(), + userId); if (individualOpenCGAResult.getNumResults() == 0) { throw new CatalogException("Proband '" + clinicalAnalysis.getProband().getId() + "' not found."); } @@ -539,17 +565,10 @@ public OpenCGAResult create(String studyStr, ClinicalAnalysis } } } - List files = obtainFiles(study, clinicalAnalysis, userId); if (clinicalAnalysis.getFiles() != null && !clinicalAnalysis.getFiles().isEmpty()) { - Set fileIds = clinicalAnalysis.getFiles().stream().map(File::getId).collect(Collectors.toSet()); - String notFoundFiles = files.stream().map(File::getId).filter(f -> !fileIds.contains(f)).collect(Collectors.joining(", ")); - if (StringUtils.isNotEmpty(notFoundFiles)) { - throw new CatalogException("Files '" + notFoundFiles + "' not found or do not belong to any participant."); - } - List filteredFiles = files.stream().filter(f -> fileIds.contains(f.getId())).collect(Collectors.toList()); - clinicalAnalysis.setFiles(filteredFiles); + validateFiles(organizationId, study, clinicalAnalysis, userId); } else { - clinicalAnalysis.setFiles(files); + obtainFiles(organizationId, study, clinicalAnalysis, userId); } clinicalAnalysis.setCreationDate(ParamUtils.checkDateOrGetCurrentDate(clinicalAnalysis.getCreationDate(), @@ -569,10 +588,10 @@ public OpenCGAResult create(String studyStr, ClinicalAnalysis validateCustomPriorityParameters(clinicalAnalysis, clinicalConfiguration); validateCustomFlagParameters(clinicalAnalysis, clinicalConfiguration); validateCustomConsentParameters(clinicalAnalysis, clinicalConfiguration); - validateStatusParameter(clinicalAnalysis, clinicalConfiguration); + validateStatusParameter(clinicalAnalysis, clinicalConfiguration, userId, true); if (StringUtils.isNotEmpty(clinicalAnalysis.getStatus().getId())) { - List clinicalStatusValues = clinicalConfiguration.getStatus().get(clinicalAnalysis.getType()); + List clinicalStatusValues = clinicalConfiguration.getStatus(); for (ClinicalStatusValue clinicalStatusValue : clinicalStatusValues) { if (clinicalAnalysis.getStatus().getId().equals(clinicalStatusValue.getId())) { if (clinicalStatusValue.getType() == ClinicalStatusValue.ClinicalStatusType.CLOSED) { @@ -598,37 +617,152 @@ public OpenCGAResult create(String studyStr, ClinicalAnalysis } if (clinicalAnalysis.getInterpretation() != null) { - catalogManager.getInterpretationManager().validateNewInterpretation(study, clinicalAnalysis.getInterpretation(), - clinicalAnalysis, userId); + catalogManager.getInterpretationManager().validateNewInterpretation(organizationId, study, + clinicalAnalysis.getInterpretation(), clinicalAnalysis, userId); clinicalAuditList.add(new ClinicalAudit(userId, ClinicalAudit.Action.CREATE_INTERPRETATION, "Create interpretation '" + clinicalAnalysis.getInterpretation().getId() + "'", TimeUtils.getTime())); } clinicalAuditList.add(new ClinicalAudit(userId, ClinicalAudit.Action.CREATE_CLINICAL_ANALYSIS, "Create ClinicalAnalysis '" + clinicalAnalysis.getId() + "'", TimeUtils.getTime())); - OpenCGAResult insert = clinicalDBAdaptor.insert(study.getUid(), clinicalAnalysis, study.getVariableSets(), - clinicalAuditList, options); + OpenCGAResult insert = getClinicalAnalysisDBAdaptor(organizationId).insert(study.getUid(), clinicalAnalysis, + study.getVariableSets(), clinicalAuditList, options); insert.addEvents(events); - auditManager.auditCreate(userId, Enums.Resource.CLINICAL_ANALYSIS, clinicalAnalysis.getId(), clinicalAnalysis.getUuid(), - study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.auditCreate(organizationId, userId, Enums.Resource.CLINICAL_ANALYSIS, clinicalAnalysis.getId(), + clinicalAnalysis.getUuid(), study.getId(), study.getUuid(), auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { // Fetch updated clinical analysis - OpenCGAResult queryResult = clinicalDBAdaptor.get(study.getUid(), clinicalAnalysis.getId(), - QueryOptions.empty()); + OpenCGAResult queryResult = getClinicalAnalysisDBAdaptor(organizationId).get(study.getUid(), + clinicalAnalysis.getId(), QueryOptions.empty()); insert.setResults(queryResult.getResults()); } return insert; } catch (CatalogException e) { - auditManager.auditCreate(userId, Enums.Resource.CLINICAL_ANALYSIS, clinicalAnalysis.getId(), "", study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditCreate(organizationId, userId, Enums.Resource.CLINICAL_ANALYSIS, clinicalAnalysis.getId(), "", study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } - private void validateAndInitReport(Study study, ClinicalReport report, String userId) throws CatalogException { + public ClinicalAnalysisLoadResult load(String studyStr, Path filePath, String token) throws CatalogException, IOException { + ClinicalAnalysisLoadResult result = new ClinicalAnalysisLoadResult(); + + int counter = 0; + ObjectReader objectReader = JacksonUtils.getDefaultObjectMapper().readerFor(ClinicalAnalysis.class); + + StopWatch stopWatch = StopWatch.createStarted(); + try (BufferedReader br = FileUtils.newBufferedReader(filePath)) { + while (true) { + String line = br.readLine(); + if (line == null) { + break; + } + ClinicalAnalysis clinicalAnalysis = null; + try { + clinicalAnalysis = objectReader.readValue(line); + logger.info("Loading clinical analysis {}...", clinicalAnalysis.getId()); + load(clinicalAnalysis, studyStr, token); + logger.info("... clinical analysis {} loaded !", clinicalAnalysis.getId()); + counter++; + } catch (Exception e) { + logger.error("Error loading clinical analysis" + (clinicalAnalysis != null ? (": " + clinicalAnalysis.getId()) : "") + + ": " + e.getMessage()); + result.getFailures().put(clinicalAnalysis.getId(), e.getMessage()); + e.printStackTrace(); + } + } + } + stopWatch.stop(); + + result.setNumLoaded(counter) + .setFilename(filePath.getFileName().toString()) + .setTime((int) stopWatch.getTime(TimeUnit.SECONDS)); + + return result; + } + + private void load(ClinicalAnalysis clinicalAnalysis, String study, String token) throws CatalogException { + Map> individualSamples = new HashMap<>(); + + // Create samples + for (Individual member : clinicalAnalysis.getFamily().getMembers()) { + if (CollectionUtils.isNotEmpty(member.getSamples())) { + individualSamples.put(member.getId(), member.getSamples().stream().map(Sample::getId).collect(Collectors.toList())); + for (Sample sample : member.getSamples()) { + try { + Sample sampleForCreation = SampleCreateParams.of(sample).toSample(); + sampleForCreation.setIndividualId(null); + catalogManager.getSampleManager().create(study, sampleForCreation, QueryOptions.empty(), token); + } catch (CatalogException e) { + if (!e.getMessage().contains("already exists")) { + throw e; + } + } + } + } + } + + // Create family with individuals + try { + Family family = FamilyCreateParams.of(clinicalAnalysis.getFamily()).toFamily(); + catalogManager.getFamilyManager().create(study, family, QueryOptions.empty(), token); + } catch (CatalogException e) { + if (!e.getMessage().contains("already exists")) { + throw e; + } + } + + // Associate individuals and samples + for (Map.Entry> entry : individualSamples.entrySet()) { + catalogManager.getIndividualManager().update(study, entry.getKey(), new IndividualUpdateParams().setSamples( + entry.getValue().stream().map(s -> new SampleReferenceParam().setId(s)).collect(Collectors.toList())), + QueryOptions.empty(), token); + } + + for (Panel panel : clinicalAnalysis.getPanels()) { + try { + catalogManager.getPanelManager().create(study, panel, QueryOptions.empty(), token); + } catch (CatalogException e) { + if (!e.getMessage().contains("already exists")) { + throw e; + } + } + } + + // Save primary and secondary interpretations for creating later + Interpretation primaryInterpretation = clinicalAnalysis.getInterpretation(); + clinicalAnalysis.setInterpretation(null); + + List secondaryInterpretations = clinicalAnalysis.getSecondaryInterpretations(); + clinicalAnalysis.setSecondaryInterpretations(null); + + + // Create clinical analysis + ClinicalAnalysis caToCreate = ClinicalAnalysisCreateParams.of(clinicalAnalysis).toClinicalAnalysis(); + caToCreate.setAnalyst(new ClinicalAnalyst()); + catalogManager.getClinicalAnalysisManager().create(study, caToCreate, QueryOptions.empty(), token); + + // Create primary interpretation + if (primaryInterpretation != null) { + primaryInterpretation.setId(null); + catalogManager.getInterpretationManager().create(study, clinicalAnalysis.getId(), primaryInterpretation, PRIMARY, + QueryOptions.empty(), token); + } + + // Create secondary interpretations + for (Interpretation secondaryInterpretation : secondaryInterpretations) { + secondaryInterpretation.setId(null); + catalogManager.getInterpretationManager().create(study, clinicalAnalysis.getId(), secondaryInterpretation, SECONDARY, + QueryOptions.empty(), token); + } + } + + private void validateAndInitReport(String organizationId, Study study, ClinicalReport report, String userId) + throws CatalogException { if (report == null) { return; } @@ -642,22 +776,22 @@ private void validateAndInitReport(Study study, ClinicalReport report, String us } } if (CollectionUtils.isNotEmpty(report.getFiles())) { - List files = obtainFiles(study, userId, report.getFiles()); + List files = obtainFiles(organizationId, study, userId, report.getFiles()); report.setFiles(files); } if (CollectionUtils.isNotEmpty(report.getSupportingEvidences())) { - List files = obtainFiles(study, userId, report.getSupportingEvidences()); + List files = obtainFiles(organizationId, study, userId, report.getSupportingEvidences()); report.setSupportingEvidences(files); } } - private void fillResponsible(ClinicalResponsible responsible) throws CatalogException { + private void fillResponsible(String organizationId, ClinicalResponsible responsible) throws CatalogException { if (responsible == null) { return; } QueryOptions userInclude = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(UserDBAdaptor.QueryParams.ID.key(), UserDBAdaptor.QueryParams.NAME.key(), UserDBAdaptor.QueryParams.EMAIL.key())); - OpenCGAResult result = userDBAdaptor.get(responsible.getId(), userInclude); + OpenCGAResult result = getUserDBAdaptor(organizationId).get(responsible.getId(), userInclude); if (result.getNumResults() == 0) { throw new CatalogException("Responsible user '" + responsible.getId() + "' not found."); } @@ -665,17 +799,15 @@ private void fillResponsible(ClinicalResponsible responsible) throws CatalogExce responsible.setEmail(ParamUtils.defaultString(responsible.getEmail(), result.first().getEmail())); } - private void validateStatusParameter(ClinicalAnalysis clinicalAnalysis, ClinicalAnalysisStudyConfiguration clinicalConfiguration) - throws CatalogException { + private void validateStatusParameter(ClinicalAnalysis clinicalAnalysis, ClinicalAnalysisStudyConfiguration clinicalConfiguration, + String userId, boolean initIfUndefined) throws CatalogException { // Status - if (clinicalConfiguration.getStatus() == null - || CollectionUtils.isEmpty(clinicalConfiguration.getStatus().get(clinicalAnalysis.getType()))) { - throw new CatalogException("Missing status configuration in study for type '" + clinicalAnalysis.getType() - + "'. Please add a proper set of valid statuses."); + if (CollectionUtils.isEmpty(clinicalConfiguration.getStatus())) { + throw new CatalogException("Missing status configuration in study. Please add a proper set of valid statuses."); } if (StringUtils.isNotEmpty(clinicalAnalysis.getStatus().getId())) { Map statusMap = new HashMap<>(); - for (ClinicalStatusValue status : clinicalConfiguration.getStatus().get(clinicalAnalysis.getType())) { + for (ClinicalStatusValue status : clinicalConfiguration.getStatus()) { statusMap.put(status.getId(), status); } if (!statusMap.containsKey(clinicalAnalysis.getStatus().getId())) { @@ -684,10 +816,26 @@ private void validateStatusParameter(ClinicalAnalysis clinicalAnalysis, Clinical } ClinicalStatusValue clinicalStatusValue = statusMap.get(clinicalAnalysis.getStatus().getId()); clinicalAnalysis.getStatus().setDescription(clinicalStatusValue.getDescription()); - clinicalAnalysis.getStatus().setDate(TimeUtils.getTime()); + clinicalAnalysis.getStatus().setType(clinicalStatusValue.getType()); } else if (clinicalAnalysis.getStatus().getId() == null) { - clinicalAnalysis.getStatus().setId(""); + if (initIfUndefined) { + // Look for first status of type NOT_STARTED + for (ClinicalStatusValue status : clinicalConfiguration.getStatus()) { + if (status.getType() == ClinicalStatusValue.ClinicalStatusType.NOT_STARTED) { + clinicalAnalysis.getStatus().setId(status.getId()); + clinicalAnalysis.getStatus().setDescription(status.getDescription()); + clinicalAnalysis.getStatus().setType(status.getType()); + break; + } + } + } else { + throw new CatalogException("Missing status id in clinical analysis"); + } } + clinicalAnalysis.getStatus().setDate(TimeUtils.getTime()); + clinicalAnalysis.getStatus().setVersion(GitRepositoryState.getInstance().getBuildVersion()); + clinicalAnalysis.getStatus().setCommit(GitRepositoryState.getInstance().getCommitId()); + clinicalAnalysis.getStatus().setAuthor(userId); } private void validateCustomConsentParameters(ClinicalAnalysis clinicalAnalysis, @@ -742,12 +890,11 @@ private void validateCustomFlagParameters(ClinicalAnalysis clinicalAnalysis, Cli throws CatalogException { // Flag definition if (CollectionUtils.isNotEmpty(clinicalAnalysis.getFlags())) { - if (CollectionUtils.isEmpty(clinicalConfiguration.getFlags().get(clinicalAnalysis.getType()))) { - throw new CatalogException("Missing flags configuration in study for type '" + clinicalAnalysis.getType() - + "'. Please add a proper set of valid priorities."); + if (CollectionUtils.isEmpty(clinicalConfiguration.getFlags())) { + throw new CatalogException("Missing flags configuration. Please add a proper set of valid flags."); } Map supportedFlags = new HashMap<>(); - for (FlagValue flagValue : clinicalConfiguration.getFlags().get(clinicalAnalysis.getType())) { + for (FlagValue flagValue : clinicalConfiguration.getFlags()) { supportedFlags.put(flagValue.getId(), flagValue); } @@ -761,8 +908,8 @@ private void validateCustomFlagParameters(ClinicalAnalysis clinicalAnalysis, Cli flag.setDescription(supportedFlags.get(flag.getId()).getDescription()); flag.setDate(TimeUtils.getTime()); } else { - throw new CatalogException("Flag '" + flag.getId() + "' not supported. Supported flags for Clinical Analyses of " - + "type '" + clinicalAnalysis.getType() + "' are: '" + String.join(", ", supportedFlags.keySet()) + "'."); + throw new CatalogException("Flag '" + flag.getId() + "' not supported. Supported flags for Clinical Analyses are: '" + + String.join(", ", supportedFlags.keySet()) + "'."); } } clinicalAnalysis.setFlags(new ArrayList<>(flagMap.values())); @@ -830,7 +977,7 @@ private void validateDisorder(ClinicalAnalysis clinicalAnalysis) throws CatalogE } } - private List obtainFiles(Study study, ClinicalAnalysis clinicalAnalysis, String userId) throws CatalogException { + private void obtainFiles(String organizationId, Study study, ClinicalAnalysis clinicalAnalysis, String userId) throws CatalogException { Set sampleSet = new HashSet<>(); if (clinicalAnalysis.getFamily() != null && clinicalAnalysis.getFamily().getMembers() != null) { for (Individual member : clinicalAnalysis.getFamily().getMembers()) { @@ -854,22 +1001,26 @@ private List obtainFiles(Study study, ClinicalAnalysis clinicalAnalysis, S Query query = new Query() .append(FileDBAdaptor.QueryParams.SAMPLE_IDS.key(), new ArrayList<>(sampleSet)) .append(FileDBAdaptor.QueryParams.BIOFORMAT.key(), Arrays.asList(File.Bioformat.ALIGNMENT, File.Bioformat.VARIANT)); - OpenCGAResult fileResults = fileDBAdaptor.get(study.getUid(), query, FileManager.INCLUDE_FILE_URI_PATH, userId); - return fileResults.getResults(); + OpenCGAResult fileResults = getFileDBAdaptor(organizationId).get(study.getUid(), query, FileManager.INCLUDE_FILE_URI_PATH, + userId); + clinicalAnalysis.setFiles(fileResults.getResults()); +// return fileResults.getResults(); } - return Collections.emptyList(); +// return Collections.emptyList(); } - private List obtainFiles(Study study, String userId, List files) throws CatalogException { + private List obtainFiles(String organizationId, Study study, String userId, List files) throws CatalogException { Query query = new Query(FileDBAdaptor.QueryParams.ID.key(), files.stream().map(File::getId).collect(Collectors.toSet())); - List results = fileDBAdaptor.get(study.getUid(), query, FileManager.INCLUDE_FILE_URI_PATH, userId).getResults(); + List results = getFileDBAdaptor(organizationId).get(study.getUid(), query, FileManager.INCLUDE_FILE_URI_PATH, userId) + .getResults(); if (results.size() < files.size()) { throw new CatalogException("Some of the files were not found"); } return results; } - private void validateFiles(Study study, ClinicalAnalysis clinicalAnalysis, String userId) throws CatalogException { + private void validateFiles(String organizationId, Study study, ClinicalAnalysis clinicalAnalysis, String userId) + throws CatalogException { Map sampleMap = new HashMap<>(); if (clinicalAnalysis.getFamily() != null && clinicalAnalysis.getFamily().getMembers() != null) { for (Individual member : clinicalAnalysis.getFamily().getMembers()) { @@ -893,7 +1044,7 @@ private void validateFiles(Study study, ClinicalAnalysis clinicalAnalysis, Strin Query query = new Query(FileDBAdaptor.QueryParams.ID.key(), clinicalAnalysis.getFiles().stream().map(File::getId).collect(Collectors.toList())); QueryOptions fileOptions = keepFieldInQueryOptions(FileManager.INCLUDE_FILE_URI_PATH, FileDBAdaptor.QueryParams.SAMPLE_IDS.key()); - OpenCGAResult fileResults = fileDBAdaptor.get(study.getUid(), query, fileOptions, userId); + OpenCGAResult fileResults = getFileDBAdaptor(organizationId).get(study.getUid(), query, fileOptions, userId); if (fileResults.getNumResults() != clinicalAnalysis.getFiles().size()) { Set fileIds = clinicalAnalysis.getFiles().stream().map(File::getId).collect(Collectors.toSet()); @@ -921,6 +1072,7 @@ private void validateFiles(Study study, ClinicalAnalysis clinicalAnalysis, Strin } } } + } // for (File caFile : clinicalAnalysis.getFiles()) { // List fileIds = caFile.getFiles().stream().map(File::getId).collect(Collectors.toList()); @@ -935,219 +1087,67 @@ private void validateFiles(Study study, ClinicalAnalysis clinicalAnalysis, Strin // } // } // } - } -// private void obtainFiles(Study study, ClinicalAnalysis clinicalAnalysis, String userId) throws CatalogException { -// Set sampleSet = new HashSet<>(); -// if (clinicalAnalysis.getFamily() != null && clinicalAnalysis.getFamily().getMembers() != null) { -// for (Individual member : clinicalAnalysis.getFamily().getMembers()) { -// if (member.getSamples() != null) { -// for (Sample sample : member.getSamples()) { -// sampleSet.add(sample.getId()); -// } -// } -// } -// } else if (clinicalAnalysis.getProband() != null && clinicalAnalysis.getProband().getSamples() != null) { -// for (Sample sample : clinicalAnalysis.getProband().getSamples()) { -// sampleSet.add(sample.getId()); -// } +// private Individual getFullValidatedMember(String organizationId, Individual member, Study study, String sessionId) +// throws CatalogException { +// if (member == null) { +// return null; // } // -// if (clinicalAnalysis.getFiles() != null && !clinicalAnalysis.getFiles().isEmpty()) { -// throw new CatalogException("Cannot obtain map of files if this is already provided"); +// if (StringUtils.isEmpty(member.getId())) { +// throw new CatalogException("Missing member id"); // } // -// if (!sampleSet.isEmpty()) { -// Query query = new Query() -// .append(FileDBAdaptor.QueryParams.SAMPLE_IDS.key(), new ArrayList<>(sampleSet)) -// .append(FileDBAdaptor.QueryParams.BIOFORMAT.key(), Arrays.asList(File.Bioformat.ALIGNMENT, File.Bioformat.VARIANT)); -// OpenCGAResult fileResults = fileDBAdaptor.get(study.getUid(), query, FileManager.INCLUDE_FILE_URI_PATH, userId); -// clinicalAnalysis.setFiles(fileResults.getResults()); -// } -// } - -// private void validateFiles(Study study, ClinicalAnalysis clinicalAnalysis, String userId) throws CatalogException { -// Map sampleMap = new HashMap<>(); -// if (clinicalAnalysis.getFamily() != null && clinicalAnalysis.getFamily().getMembers() != null) { -// for (Individual member : clinicalAnalysis.getFamily().getMembers()) { -// if (member.getSamples() != null) { -// for (Sample sample : member.getSamples()) { -// sampleMap.put(sample.getId(), sample.getUid()); -// } -// } -// } -// } else if (clinicalAnalysis.getProband() != null && clinicalAnalysis.getProband().getSamples() != null) { -// for (Sample sample : clinicalAnalysis.getProband().getSamples()) { -// sampleMap.put(sample.getId(), sample.getUid()); -// } -// } +// Individual finalMember; // -// if (clinicalAnalysis.getFiles() == null || clinicalAnalysis.getFiles().isEmpty()) { -// throw new CatalogException("Found empty map of files"); -// } +// // List of samples relevant for the clinical analysis +// List samples = member.getSamples(); +// +// if (member.getUid() <= 0) { +// OpenCGAResult individualDataResult = catalogManager.getIndividualManager().get(study.getFqn(), +// member.getId(), new QueryOptions(), sessionId); +// if (individualDataResult.getNumResults() == 0) { +// throw new CatalogException("Member " + member.getId() + " not found"); +// } // -// // Look for all the samples associated to the files -// Query query = new Query(FileDBAdaptor.QueryParams.ID.key(), -// clinicalAnalysis.getFiles().stream().map(File::getId).collect(Collectors.toList())); -// QueryOptions fileOptions = keepFieldInQueryOptions(FileManager.INCLUDE_FILE_URI_PATH, FileDBAdaptor.QueryParams.SAMPLE_IDS.key()); -// OpenCGAResult fileResults = fileDBAdaptor.get(study.getUid(), query, fileOptions, userId); +// finalMember = individualDataResult.first(); +// } else { +// finalMember = member; +// if (ListUtils.isNotEmpty(samples) && StringUtils.isEmpty(samples.get(0).getUuid())) { +// // We don't have the full sample information... +// OpenCGAResult individualDataResult = catalogManager.getIndividualManager().get(study.getFqn(), +// finalMember.getId(), new QueryOptions(QueryOptions.INCLUDE, IndividualDBAdaptor.QueryParams.SAMPLES.key()), +// sessionId); +// if (individualDataResult.getNumResults() == 0) { +// throw new CatalogException("Member " + finalMember.getId() + " not found"); +// } // -// if (fileResults.getNumResults() != clinicalAnalysis.getFiles().size()) { -// Set fileIds = clinicalAnalysis.getFiles().stream().map(File::getId).collect(Collectors.toSet()); -// String notFoundFiles = fileResults.getResults().stream().map(File::getId).filter(f -> !fileIds.contains(f)) -// .collect(Collectors.joining(", ")); -// throw new CatalogException("Files '" + notFoundFiles + "' not found"); +// finalMember.setSamples(individualDataResult.first().getSamples()); +// } // } // -// // Complete file information -// clinicalAnalysis.setFiles(fileResults.getResults()); +// if (ListUtils.isNotEmpty(finalMember.getSamples())) { +// List finalSampleList = null; +// if (ListUtils.isNotEmpty(samples)) { // -// // Validate the file ids passed are related to the samples -// for (File file : clinicalAnalysis.getFiles()) { -// if (CollectionUtils.isNotEmpty(file.getSampleIds())) { -// boolean found = false; -// for (String sampleId : file.getSampleIds()) { -// if (sampleMap.containsKey(sampleId)) { -// found = true; -// break; -// } +// Map sampleMap = new HashMap<>(); +// for (Sample sample : finalMember.getSamples()) { +// sampleMap.put(sample.getId(), sample); // } -// if (!found) { -// throw new CatalogException("Clinical analysis file (" + file.getId() + ") contains sample ids not related to any " -// + "member/proband"); +// +// finalSampleList = new ArrayList<>(samples.size()); +// +// // We keep only the original list of samples passed +// for (Sample sample : samples) { +// finalSampleList.add(sampleMap.get(sample.getId())); // } // } +// finalMember.setSamples(finalSampleList); // } // -//// for (File caFile : clinicalAnalysis.getFiles()) { -//// List fileIds = caFile.getFiles().stream().map(File::getId).collect(Collectors.toList()); -//// InternalGetDataResult fileResult = catalogManager.getFileManager().internalGet(study.getUid(), fileIds, new Query(), -//// new QueryOptions(), userId, false); -//// // Validate sample id belongs to files -//// for (File file : fileResult.getResults()) { -//// if (!file.getSamples().stream().map(Sample::getUid).collect(Collectors.toSet()) -//// .contains(sampleMap.get(caFile.getSampleId()))) { -//// throw new CatalogException("Associated file '" + file.getPath() + "' seems not to be related to sample '" -//// + caFile.getSampleId() + "'."); -//// } -//// } -//// } +// return finalMember; // } - private Family getFullValidatedFamily(Family family, Study study, String sessionId) throws CatalogException { - if (family == null) { - return null; - } - - if (StringUtils.isEmpty(family.getId())) { - throw new CatalogException("Missing family id"); - } - - // List of members relevant for the clinical analysis - List selectedMembers = family.getMembers(); - - OpenCGAResult familyDataResult = catalogManager.getFamilyManager().get(study.getFqn(), family.getId(), new QueryOptions(), - sessionId); - if (familyDataResult.getNumResults() == 0) { - throw new CatalogException("Family " + family.getId() + " not found"); - } - Family finalFamily = familyDataResult.first(); - - if (ListUtils.isNotEmpty(selectedMembers)) { - if (ListUtils.isEmpty(finalFamily.getMembers())) { - throw new CatalogException("Family " + family.getId() + " does not have any members associated"); - } - - Map memberMap = new HashMap<>(); - for (Individual member : finalFamily.getMembers()) { - memberMap.put(member.getId(), member); - } - - List finalMembers = new ArrayList<>(selectedMembers.size()); - for (Individual selectedMember : selectedMembers) { - Individual fullMember = memberMap.get(selectedMember.getId()); - if (fullMember == null) { - throw new CatalogException("Member " + selectedMember.getId() + " does not belong to family " + family.getId()); - } - fullMember.setSamples(selectedMember.getSamples()); - finalMembers.add(getFullValidatedMember(fullMember, study, sessionId)); - } - - finalFamily.setMembers(finalMembers); - } else { - if (ListUtils.isNotEmpty(finalFamily.getMembers())) { - Query query = new Query() - .append(IndividualDBAdaptor.QueryParams.UID.key(), finalFamily.getMembers().stream() - .map(Individual::getUid).collect(Collectors.toList())) - .append(IndividualDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - OpenCGAResult individuals = individualDBAdaptor.get(study.getUid(), query, QueryOptions.empty(), - catalogManager.getUserManager().getUserId(sessionId)); - finalFamily.setMembers(individuals.getResults()); - } - } - - return finalFamily; - } - - private Individual getFullValidatedMember(Individual member, Study study, String sessionId) throws CatalogException { - if (member == null) { - return null; - } - - if (StringUtils.isEmpty(member.getId())) { - throw new CatalogException("Missing member id"); - } - - Individual finalMember; - - // List of samples relevant for the clinical analysis - List samples = member.getSamples(); - - if (member.getUid() <= 0) { - OpenCGAResult individualDataResult = catalogManager.getIndividualManager().get(study.getFqn(), member.getId(), - new QueryOptions(), sessionId); - if (individualDataResult.getNumResults() == 0) { - throw new CatalogException("Member " + member.getId() + " not found"); - } - - finalMember = individualDataResult.first(); - } else { - finalMember = member; - if (ListUtils.isNotEmpty(samples) && StringUtils.isEmpty(samples.get(0).getUuid())) { - // We don't have the full sample information... - OpenCGAResult individualDataResult = catalogManager.getIndividualManager().get(study.getFqn(), - finalMember.getId(), new QueryOptions(QueryOptions.INCLUDE, IndividualDBAdaptor.QueryParams.SAMPLES.key()), - sessionId); - if (individualDataResult.getNumResults() == 0) { - throw new CatalogException("Member " + finalMember.getId() + " not found"); - } - - finalMember.setSamples(individualDataResult.first().getSamples()); - } - } - - if (ListUtils.isNotEmpty(finalMember.getSamples())) { - List finalSampleList = null; - if (ListUtils.isNotEmpty(samples)) { - - Map sampleMap = new HashMap<>(); - for (Sample sample : finalMember.getSamples()) { - sampleMap.put(sample.getId(), sample); - } - - finalSampleList = new ArrayList<>(samples.size()); - - // We keep only the original list of samples passed - for (Sample sample : samples) { - finalSampleList.add(sampleMap.get(sample.getId())); - } - } - finalMember.setSamples(finalSampleList); - } - - return finalMember; - } - public OpenCGAResult update(String studyStr, Query query, ClinicalAnalysisUpdateParams updateParams, QueryOptions options, String token) throws CatalogException { return update(studyStr, query, updateParams, false, options, token); @@ -1155,8 +1155,11 @@ public OpenCGAResult update(String studyStr, Query query, Clin public OpenCGAResult update(String studyStr, Query query, ClinicalAnalysisUpdateParams updateParams, boolean ignoreException, QueryOptions options, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId, StudyManager.INCLUDE_VARIABLE_SET_AND_CONFIGURATION); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyFqn, StudyManager.INCLUDE_VARIABLE_SET_AND_CONFIGURATION, tokenPayload); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -1177,13 +1180,13 @@ public OpenCGAResult update(String studyStr, Query query, Clin DBIterator iterator; try { - fixQueryObject(study, query, userId, token); + fixQueryObject(organizationId, study, query, userId, token); AnnotationUtils.fixQueryOptionAnnotation(options); query.append(ClinicalAnalysisDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - iterator = clinicalDBAdaptor.iterator(study.getUid(), query, new QueryOptions(), userId); + iterator = getClinicalAnalysisDBAdaptor(organizationId).iterator(study.getUid(), query, new QueryOptions(), userId); } catch (CatalogException e) { - auditManager.auditUpdate(operationId, userId, Enums.Resource.CLINICAL_ANALYSIS, "", "", study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.CLINICAL_ANALYSIS, "", "", study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -1192,10 +1195,11 @@ public OpenCGAResult update(String studyStr, Query query, Clin while (iterator.hasNext()) { ClinicalAnalysis clinicalAnalysis = iterator.next(); try { - OpenCGAResult queryResult = update(study, clinicalAnalysis, updateParams, userId, options); + OpenCGAResult queryResult = update(organizationId, study, clinicalAnalysis, updateParams, userId, + options); result.append(queryResult); - auditManager.auditUpdate(operationId, userId, Enums.Resource.CLINICAL_ANALYSIS, clinicalAnalysis.getId(), + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.CLINICAL_ANALYSIS, clinicalAnalysis.getId(), clinicalAnalysis.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { @@ -1204,20 +1208,23 @@ public OpenCGAResult update(String studyStr, Query query, Clin result.setNumErrors(result.getNumErrors() + 1); logger.error("Could not update clinical analysis {}: {}", clinicalAnalysis.getId(), e.getMessage()); - auditManager.auditUpdate(operationId, userId, Enums.Resource.CLINICAL_ANALYSIS, clinicalAnalysis.getId(), + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.CLINICAL_ANALYSIS, clinicalAnalysis.getId(), clinicalAnalysis.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); } } - auditManager.finishAuditBatch(operationId); + auditManager.finishAuditBatch(organizationId, operationId); return endResult(result, ignoreException); } public OpenCGAResult update(String studyStr, String clinicalId, ClinicalAnalysisUpdateParams updateParams, QueryOptions options, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId, StudyManager.INCLUDE_VARIABLE_SET_AND_CONFIGURATION); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyFqn, StudyManager.INCLUDE_VARIABLE_SET_AND_CONFIGURATION, tokenPayload); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -1238,7 +1245,8 @@ public OpenCGAResult update(String studyStr, String clinicalId OpenCGAResult result = OpenCGAResult.empty(); String clinicalUuid = ""; try { - OpenCGAResult internalResult = internalGet(study.getUid(), clinicalId, QueryOptions.empty(), userId); + OpenCGAResult internalResult = internalGet(organizationId, study.getUid(), clinicalId, QueryOptions.empty(), + userId); if (internalResult.getNumResults() == 0) { throw new CatalogException("Clinical analysis '" + clinicalId + "' not found"); } @@ -1248,10 +1256,10 @@ public OpenCGAResult update(String studyStr, String clinicalId clinicalId = clinicalAnalysis.getId(); clinicalUuid = clinicalAnalysis.getUuid(); - OpenCGAResult updateResult = update(study, clinicalAnalysis, updateParams, userId, options); + OpenCGAResult updateResult = update(organizationId, study, clinicalAnalysis, updateParams, userId, options); result.append(updateResult); - auditManager.auditUpdate(operationId, userId, Enums.Resource.CLINICAL_ANALYSIS, clinicalAnalysis.getId(), + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.CLINICAL_ANALYSIS, clinicalAnalysis.getId(), clinicalAnalysis.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { @@ -1260,7 +1268,7 @@ public OpenCGAResult update(String studyStr, String clinicalId result.setNumErrors(result.getNumErrors() + 1); logger.error("Could not update clinical analysis {}: {}", clinicalId, e.getMessage()); - auditManager.auditUpdate(operationId, userId, Enums.Resource.CLINICAL_ANALYSIS, clinicalId, clinicalUuid, + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.CLINICAL_ANALYSIS, clinicalId, clinicalUuid, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -1271,7 +1279,8 @@ public OpenCGAResult update(String studyStr, String clinicalId /** * Update a Clinical Analysis from catalog. * - * @param studyStr Study id in string format. Could be one of [id|user@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy]. + * @param studyStr Study id in string format. Could be one of + * [id|organization@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy] * @param clinicalIds List of clinical analysis ids. Could be either the id or uuid. * @param updateParams Data model filled only with the parameters to be updated. * @param options QueryOptions object. @@ -1287,8 +1296,11 @@ public OpenCGAResult update(String studyStr, List clin public OpenCGAResult update(String studyStr, List clinicalIds, ClinicalAnalysisUpdateParams updateParams, boolean ignoreException, QueryOptions options, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId, StudyManager.INCLUDE_VARIABLE_SET_AND_CONFIGURATION); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyFqn, StudyManager.INCLUDE_VARIABLE_SET_AND_CONFIGURATION, tokenPayload); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -1314,7 +1326,8 @@ public OpenCGAResult update(String studyStr, List clin String clinicalAnalysisId = id; String clinicalAnalysisUuid = ""; try { - OpenCGAResult internalResult = internalGet(study.getUid(), id, QueryOptions.empty(), userId); + OpenCGAResult internalResult = internalGet(organizationId, study.getUid(), id, QueryOptions.empty(), + userId); if (internalResult.getNumResults() == 0) { throw new CatalogException("Clinical analysis '" + id + "' not found"); } @@ -1324,10 +1337,11 @@ public OpenCGAResult update(String studyStr, List clin clinicalAnalysisId = clinicalAnalysis.getId(); clinicalAnalysisUuid = clinicalAnalysis.getUuid(); - OpenCGAResult updateResult = update(study, clinicalAnalysis, updateParams, userId, options); + OpenCGAResult updateResult = update(organizationId, study, clinicalAnalysis, updateParams, userId, + options); result.append(updateResult); - auditManager.auditUpdate(operationId, userId, Enums.Resource.CLINICAL_ANALYSIS, clinicalAnalysis.getId(), + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.CLINICAL_ANALYSIS, clinicalAnalysis.getId(), clinicalAnalysis.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { @@ -1336,16 +1350,17 @@ public OpenCGAResult update(String studyStr, List clin result.setNumErrors(result.getNumErrors() + 1); logger.error("Could not update clinical analysis {}: {}", clinicalAnalysisId, e.getMessage()); - auditManager.auditUpdate(operationId, userId, Enums.Resource.CLINICAL_ANALYSIS, clinicalAnalysisId, clinicalAnalysisUuid, - study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.CLINICAL_ANALYSIS, clinicalAnalysisId, + clinicalAnalysisUuid, study.getId(), study.getUuid(), auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); } } - auditManager.finishAuditBatch(operationId); + auditManager.finishAuditBatch(organizationId, operationId); return endResult(result, ignoreException); } - private OpenCGAResult update(Study study, ClinicalAnalysis clinicalAnalysis, + private OpenCGAResult update(String organizationId, Study study, ClinicalAnalysis clinicalAnalysis, ClinicalAnalysisUpdateParams updateParams, String userId, QueryOptions options) throws CatalogException { ClinicalAnalysisUpdateParams updateParamsClone; @@ -1358,19 +1373,19 @@ private OpenCGAResult update(Study study, ClinicalAnalysis cli throw new CatalogException("Empty update parameters. Nothing to update."); } - validateAndInitReport(study, updateParamsClone.getReport(), userId); + validateAndInitReport(organizationId, study, updateParamsClone.getReport(), userId); // ---------- Check and init responsible fields ClinicalResponsible responsible = updateParamsClone.getResponsible(); if (responsible != null && StringUtils.isNotEmpty(responsible.getId())) { - fillResponsible(responsible); + fillResponsible(organizationId, responsible); } // ---------- Check and init request fields ClinicalRequest request = updateParamsClone.getRequest(); if (request != null && StringUtils.isNotEmpty(request.getId())) { request.setDate(ParamUtils.checkDateOrGetCurrentDate(request.getDate(), "request.date")); - fillResponsible(request.getResponsible()); + fillResponsible(organizationId, request.getResponsible()); } ObjectMap parameters; @@ -1388,16 +1403,66 @@ private OpenCGAResult update(Study study, ClinicalAnalysis cli } ClinicalAnalysisStudyConfiguration clinicalConfiguration = study.getInternal().getConfiguration().getClinical(); + // Get the clinical status that are CLOSED and DONE + Set closedStatus = new HashSet<>(); + Set doneStatus = new HashSet<>(); + for (ClinicalStatusValue clinicalStatusValue : clinicalConfiguration.getStatus()) { + if (clinicalStatusValue.getType().equals(ClinicalStatusValue.ClinicalStatusType.CLOSED)) { + closedStatus.add(clinicalStatusValue.getId()); + } else if (clinicalStatusValue.getType().equals(ClinicalStatusValue.ClinicalStatusType.DONE)) { + doneStatus.add(clinicalStatusValue.getId()); + } + } + + // If the current clinical analysis: + // - is locked or panelLocked + // - the user wants to update the locked or panelLocked status + // - the user wants to update the status to/from a done|closed status + boolean adminPermissionsChecked = false; + if (clinicalAnalysis.isLocked() || clinicalAnalysis.isPanelLocked() + || clinicalAnalysis.getStatus().getType() == ClinicalStatusValue.ClinicalStatusType.CLOSED + || clinicalAnalysis.getStatus().getType() == ClinicalStatusValue.ClinicalStatusType.DONE + || updateParamsClone.getLocked() != null + || updateParams.getPanelLocked() != null + || (updateParams.getStatus() != null && (closedStatus.contains(updateParams.getStatus().getId()) + || doneStatus.contains(updateParams.getStatus().getId())))) { + authorizationManager.checkClinicalAnalysisPermission(organizationId, study.getUid(), clinicalAnalysis.getUid(), userId, + ClinicalAnalysisPermissions.ADMIN); + + // Current status is of type CLOSED + if (clinicalAnalysis.getStatus().getType() == ClinicalStatusValue.ClinicalStatusType.CLOSED) { + // The only allowed action is to remove the CLOSED status + if (updateParams.getStatus() == null || StringUtils.isEmpty(updateParams.getStatus().getId())) { + throw new CatalogException("Cannot update a ClinicalAnalysis with a " + ClinicalStatusValue.ClinicalStatusType.CLOSED + + " status. You need to remove the " + ClinicalStatusValue.ClinicalStatusType.CLOSED + " status to be able " + + "to perform further updates on the ClinicalAnalysis."); + } else if (closedStatus.contains(updateParams.getStatus().getId())) { + // Users should be able to change from one CLOSED status to a different one but we should still control that no further + // modifications are made + if (parameters.size() > 1) { + throw new CatalogException("Cannot update a ClinicalAnalysis with a " + + ClinicalStatusValue.ClinicalStatusType.CLOSED + " status. You need to remove the " + + ClinicalStatusValue.ClinicalStatusType.CLOSED + " status to be able to perform further updates on " + + "the ClinicalAnalysis."); + } else if (clinicalAnalysis.getStatus().getId().equals(updateParams.getStatus().getId())) { + throw new CatalogException("ClinicalAnalysis already have the status '" + clinicalAnalysis.getStatus().getId() + + "' of type " + ClinicalStatusValue.ClinicalStatusType.CLOSED); + } + } + } + + adminPermissionsChecked = true; + } // Check permissions... // Only check write annotation permissions if the user wants to update the annotation sets - if (updateParamsClone.getAnnotationSets() != null) { - authorizationManager.checkClinicalAnalysisPermission(study.getUid(), clinicalAnalysis.getUid(), userId, + if (!adminPermissionsChecked && updateParamsClone.getAnnotationSets() != null) { + authorizationManager.checkClinicalAnalysisPermission(organizationId, study.getUid(), clinicalAnalysis.getUid(), userId, ClinicalAnalysisPermissions.WRITE_ANNOTATIONS); } // Only check update permissions if the user wants to update anything apart from the annotation sets - if ((parameters.size() == 1 && !parameters.containsKey(SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key())) - || parameters.size() > 1) { - authorizationManager.checkClinicalAnalysisPermission(study.getUid(), clinicalAnalysis.getUid(), userId, + if (!adminPermissionsChecked && ((parameters.size() == 1 + && !parameters.containsKey(SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key())) || parameters.size() > 1)) { + authorizationManager.checkClinicalAnalysisPermission(organizationId, study.getUid(), clinicalAnalysis.getUid(), userId, ClinicalAnalysisPermissions.WRITE); } @@ -1481,7 +1546,7 @@ private OpenCGAResult update(Study study, ClinicalAnalysis cli // Check analysts exist if (!analystIdList.isEmpty()) { Query query = new Query(UserDBAdaptor.QueryParams.ID.key(), analystIdList); - OpenCGAResult userResult = userDBAdaptor.get(query, userOptions); + OpenCGAResult userResult = getUserDBAdaptor(organizationId).get(query, userOptions); if (userResult.getNumResults() < analystIdList.size()) { throw new CatalogException("Some analysts were not found."); } @@ -1524,17 +1589,18 @@ private OpenCGAResult update(Study study, ClinicalAnalysis cli clinicalAnalysis.setFiles(updateParamsClone.getFiles().stream().map(FileReferenceParam::toFile).collect(Collectors.toList())); // Validate files - validateFiles(study, clinicalAnalysis, userId); + validateFiles(organizationId, study, clinicalAnalysis, userId); parameters.put(ClinicalAnalysisDBAdaptor.QueryParams.FILES.key(), clinicalAnalysis.getFiles()); } - if (CollectionUtils.isNotEmpty(updateParamsClone.getPanels()) && updateParamsClone.getPanelLock() != null - && updateParamsClone.getPanelLock()) { - throw new CatalogException("Updating the list of panels and setting 'panelLock' to true at the same time is not allowed."); + if (CollectionUtils.isNotEmpty(updateParamsClone.getPanels()) && updateParamsClone.getPanelLocked() != null + && updateParamsClone.getPanelLocked()) { + throw new CatalogException("Updating the list of panels and setting '" + + ClinicalAnalysisDBAdaptor.QueryParams.PANEL_LOCKED.key() + "' to true at the same time is not allowed."); } if (CollectionUtils.isNotEmpty(updateParamsClone.getPanels())) { - if (clinicalAnalysis.isPanelLock() && (updateParamsClone.getPanelLock() == null || updateParamsClone.getPanelLock())) { + if (clinicalAnalysis.isPanelLocked() && (updateParamsClone.getPanelLocked() == null || updateParamsClone.getPanelLocked())) { throw new CatalogException("Cannot update panels from ClinicalAnalysis '" + clinicalAnalysis.getId() + "'. " + "'panelLocked' field from ClinicalAnalysis is set to true."); } @@ -1543,7 +1609,7 @@ private OpenCGAResult update(Study study, ClinicalAnalysis cli List panelIds = updateParamsClone.getPanels().stream().map(PanelReferenceParam::getId).collect(Collectors.toList()); Query query = new Query(PanelDBAdaptor.QueryParams.ID.key(), panelIds); OpenCGAResult panelResult = - panelDBAdaptor.get(study.getUid(), query, PanelManager.INCLUDE_PANEL_IDS, userId); + getPanelDBAdaptor(organizationId).get(study.getUid(), query, PanelManager.INCLUDE_PANEL_IDS, userId); if (panelResult.getNumResults() < panelIds.size()) { throw new CatalogException("Some panels were not found or user doesn't have permissions to see them."); } @@ -1551,15 +1617,17 @@ private OpenCGAResult update(Study study, ClinicalAnalysis cli parameters.put(ClinicalAnalysisDBAdaptor.QueryParams.PANELS.key(), panelResult.getResults()); } - if (updateParamsClone.getPanelLock() != null && updateParamsClone.getPanelLock() && !clinicalAnalysis.isPanelLock()) { + if (updateParamsClone.getPanelLocked() != null && updateParamsClone.getPanelLocked() && !clinicalAnalysis.isPanelLocked()) { // if user wants to set panelLock to true // We need to check if the CA has interpretations. If so, the interpretations should contain at least one of the case panels // in order to set panelLock to true. Otherwise, that action is not allowed. Set panelIds = clinicalAnalysis.getPanels().stream().map(Panel::getId).collect(Collectors.toSet()); String exceptionMsgPrefix = "The interpretation '"; - String exceptionMsgSuffix = "' does not contain any of the case panels. 'panelLock' can only be set to true if all" + String exceptionMsgSuffix = "' does not contain any of the case panels. '" + + ClinicalAnalysisDBAdaptor.QueryParams.PANEL_LOCKED.key() + "' can only be set to true if all" + " all Interpretations contains a non-empty subset of the panels used by the case."; - String alternativeExceptionMsgSuffix = "' is using a panel not defined by the case. 'panelLock' can only be set to true if all" + String alternativeExceptionMsgSuffix = "' is using a panel not defined by the case. '" + + ClinicalAnalysisDBAdaptor.QueryParams.PANEL_LOCKED.key() + "' can only be set to true if all" + " all Interpretations contains a non-empty subset of the panels used by the case."; if (clinicalAnalysis.getInterpretation() != null) { if (CollectionUtils.isEmpty(clinicalAnalysis.getInterpretation().getPanels())) { @@ -1591,7 +1659,7 @@ private OpenCGAResult update(Study study, ClinicalAnalysis cli Query query = new Query(PanelDBAdaptor.QueryParams.ID.key(), updateParamsClone.getPanels().stream().map(PanelReferenceParam::getId).collect(Collectors.toList())); OpenCGAResult panelResult = - panelDBAdaptor.get(study.getUid(), query, PanelManager.INCLUDE_PANEL_IDS, userId); + getPanelDBAdaptor(organizationId).get(study.getUid(), query, PanelManager.INCLUDE_PANEL_IDS, userId); if (panelResult.getNumResults() < updateParamsClone.getPanels().size()) { throw new CatalogException("Some panels were not found or user doesn't have permissions to see them"); } @@ -1609,8 +1677,8 @@ private OpenCGAResult update(Study study, ClinicalAnalysis cli if (parameters.containsKey(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_STATUS.key())) { Map status = (Map) parameters.get(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_STATUS.key()); - if (!(status instanceof Map) || StringUtils.isEmpty(String.valueOf(status.get("name"))) - || !ClinicalAnalysisStatus.isValid(String.valueOf(status.get("name")))) { + if (!(status instanceof Map) || StringUtils.isEmpty(String.valueOf(status.get("id"))) + || !InternalStatus.isValid(String.valueOf(status.get("id")))) { throw new CatalogException("Missing or invalid status"); } } @@ -1654,12 +1722,12 @@ private OpenCGAResult update(Study study, ClinicalAnalysis cli parameters.put(ClinicalAnalysisDBAdaptor.QueryParams.CONSENT.key(), clinicalAnalysis.getConsent()); } if (parameters.containsKey(ClinicalAnalysisDBAdaptor.QueryParams.STATUS.key())) { - clinicalAnalysis.setStatus(updateParamsClone.getStatus().toStatus()); - validateStatusParameter(clinicalAnalysis, clinicalConfiguration); + clinicalAnalysis.setStatus(updateParamsClone.getStatus().toClinicalStatus()); + validateStatusParameter(clinicalAnalysis, clinicalConfiguration, userId, false); parameters.put(ClinicalAnalysisDBAdaptor.QueryParams.STATUS.key(), clinicalAnalysis.getStatus()); if (StringUtils.isNotEmpty(updateParamsClone.getStatus().getId())) { - List clinicalStatusValues = clinicalConfiguration.getStatus().get(clinicalAnalysis.getType()); + List clinicalStatusValues = clinicalConfiguration.getStatus(); for (ClinicalStatusValue clinicalStatusValue : clinicalStatusValues) { if (updateParamsClone.getStatus().getId().equals(clinicalStatusValue.getId())) { if (clinicalStatusValue.getType() == ClinicalStatusValue.ClinicalStatusType.CLOSED) { @@ -1675,16 +1743,16 @@ private OpenCGAResult update(Study study, ClinicalAnalysis cli } } - checkUpdateAnnotations(study, clinicalAnalysis, parameters, options, VariableSet.AnnotableDataModels.CLINICAL_ANALYSIS, - clinicalDBAdaptor, userId); + checkUpdateAnnotations(organizationId, study, clinicalAnalysis, parameters, options, + VariableSet.AnnotableDataModels.CLINICAL_ANALYSIS, getClinicalAnalysisDBAdaptor(organizationId), userId); ClinicalAudit clinicalAudit = new ClinicalAudit(userId, ClinicalAudit.Action.UPDATE_CLINICAL_ANALYSIS, "Update ClinicalAnalysis '" + clinicalAnalysis.getId() + "'", TimeUtils.getTime()); - OpenCGAResult update = clinicalDBAdaptor.update(clinicalAnalysis.getUid(), parameters, study.getVariableSets(), - Collections.singletonList(clinicalAudit), options); + OpenCGAResult update = getClinicalAnalysisDBAdaptor(organizationId).update(clinicalAnalysis.getUid(), parameters, + study.getVariableSets(), Collections.singletonList(clinicalAudit), options); update.addEvents(events); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { // Fetch updated clinical analysis - OpenCGAResult result = clinicalDBAdaptor.get(study.getUid(), + OpenCGAResult result = getClinicalAnalysisDBAdaptor(organizationId).get(study.getUid(), new Query(ClinicalAnalysisDBAdaptor.QueryParams.UID.key(), clinicalAnalysis.getUid()), options, userId); update.setResults(result.getResults()); } @@ -1693,8 +1761,8 @@ private OpenCGAResult update(Study study, ClinicalAnalysis cli public OpenCGAResult updateReport(String studyStr, String clinicalAnalysisId, ClinicalReport report, QueryOptions options, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -1707,10 +1775,16 @@ public OpenCGAResult updateReport(String studyStr, String clinic String caseId = clinicalAnalysisId; String caseUuid = ""; + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + String studyId = studyFqn.getStudyId(); + String studyUuid = studyFqn.getStudyUuid(); try { + Study study = catalogManager.getStudyManager().resolveId(studyFqn, StudyManager.INCLUDE_VARIABLE_SET, tokenPayload); options = ParamUtils.defaultObject(options, QueryOptions::new); - ClinicalAnalysis clinicalAnalysis = internalGet(study.getUid(), clinicalAnalysisId, INCLUDE_CLINICAL_IDS, userId).first(); - authorizationManager.checkClinicalAnalysisPermission(study.getUid(), clinicalAnalysis.getUid(), userId, + ClinicalAnalysis clinicalAnalysis = internalGet(organizationId, study.getUid(), clinicalAnalysisId, INCLUDE_CLINICAL_IDS, + userId).first(); + authorizationManager.checkClinicalAnalysisPermission(organizationId, study.getUid(), clinicalAnalysis.getUid(), userId, ClinicalAnalysisPermissions.WRITE); caseId = clinicalAnalysis.getId(); caseUuid = clinicalAnalysis.getUuid(); @@ -1735,11 +1809,11 @@ public OpenCGAResult updateReport(String studyStr, String clinic updateMap.put(ClinicalAnalysisDBAdaptor.ReportQueryParams.COMMENTS.key(), report.getComments()); } if (CollectionUtils.isNotEmpty(report.getFiles())) { - List files = obtainFiles(study, userId, report.getFiles()); + List files = obtainFiles(organizationId, study, userId, report.getFiles()); updateMap.put(ClinicalAnalysisDBAdaptor.ReportQueryParams.FILES.key(), files, false); } if (CollectionUtils.isNotEmpty(report.getSupportingEvidences())) { - List files = obtainFiles(study, userId, report.getSupportingEvidences()); + List files = obtainFiles(organizationId, study, userId, report.getSupportingEvidences()); updateMap.put(ClinicalAnalysisDBAdaptor.ReportQueryParams.SUPPORTING_EVIDENCES.key(), files, false); } ClinicalAudit clinicalAudit = new ClinicalAudit(userId, ClinicalAudit.Action.UPDATE_CLINICAL_ANALYSIS, @@ -1747,13 +1821,13 @@ public OpenCGAResult updateReport(String studyStr, String clinic // Add custom key to ensure it is properly updated updateMap = new ObjectMap(ClinicalAnalysisDBAdaptor.QueryParams.REPORT_UPDATE.key(), updateMap); - OpenCGAResult update = clinicalDBAdaptor.update(clinicalAnalysis.getUid(), updateMap, null, - Collections.singletonList(clinicalAudit), options); + OpenCGAResult update = getClinicalAnalysisDBAdaptor(organizationId) + .update(clinicalAnalysis.getUid(), updateMap, null, Collections.singletonList(clinicalAudit), options); auditManager.auditUpdate(operationId, userId, Enums.Resource.CLINICAL_ANALYSIS, caseId, caseUuid, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { // Fetch updated clinical analysis - OpenCGAResult result = clinicalDBAdaptor.get(study.getUid(), + OpenCGAResult result = getClinicalAnalysisDBAdaptor(organizationId).get(study.getUid(), new Query(ClinicalAnalysisDBAdaptor.QueryParams.UID.key(), clinicalAnalysis.getUid()), options, userId); update.setResults(result.getResults()); } @@ -1767,8 +1841,8 @@ public OpenCGAResult updateReport(String studyStr, String clinic update.getNumMatches(), update.getNumInserted(), update.getNumUpdated(), update.getNumDeleted(), update.getNumErrors(), update.getAttributes(), update.getFederationNode()); } catch (CatalogException e) { - auditManager.auditUpdate(operationId, userId, Enums.Resource.CLINICAL_ANALYSIS, caseId, caseUuid, study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditUpdate(operationId, userId, Enums.Resource.CLINICAL_ANALYSIS, caseId, caseUuid, studyId, studyUuid, + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } @@ -1837,23 +1911,28 @@ public OpenCGAResult search(String studyId, Query query, Query query = ParamUtils.defaultObject(query, Query::new); options = ParamUtils.defaultObject(options, QueryOptions::new); - String userId = catalogManager.getUserManager().getUserId(token); - Study study = catalogManager.getStudyManager().resolveId(studyId, userId, StudyManager.INCLUDE_VARIABLE_SET); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyFqn, StudyManager.INCLUDE_VARIABLE_SET, tokenPayload); - fixQueryObject(study, query, userId, token); + fixQueryObject(organizationId, study, query, userId, token); AnnotationUtils.fixQueryOptionAnnotation(options); - query.append(ClinicalAnalysisDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - return clinicalDBAdaptor.get(study.getUid(), query, options, userId); + return getClinicalAnalysisDBAdaptor(organizationId).get(study.getUid(), query, options, userId); } @Override public OpenCGAResult distinct(String studyId, List fields, Query query, String token) throws CatalogException { query = ParamUtils.defaultObject(query, Query::new); - String userId = userManager.getUserId(token); - Study study = catalogManager.getStudyManager().resolveId(studyId, userId, StudyManager.INCLUDE_VARIABLE_SET); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyFqn, StudyManager.INCLUDE_VARIABLE_SET, tokenPayload); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) @@ -1861,25 +1940,25 @@ public OpenCGAResult distinct(String studyId, List fields, Query quer .append("query", new Query(query)) .append("token", token); try { - fixQueryObject(study, query, userId, token); + fixQueryObject(organizationId, study, query, userId, token); query.append(ClinicalAnalysisDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - OpenCGAResult result = clinicalDBAdaptor.distinct(study.getUid(), fields, query, userId); + OpenCGAResult result = getClinicalAnalysisDBAdaptor(organizationId).distinct(study.getUid(), fields, query, userId); - auditManager.auditDistinct(userId, Enums.Resource.CLINICAL_ANALYSIS, study.getId(), study.getUuid(), auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.auditDistinct(organizationId, userId, Enums.Resource.CLINICAL_ANALYSIS, study.getId(), study.getUuid(), + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return result; } catch (CatalogException e) { - auditManager.auditDistinct(userId, Enums.Resource.CLINICAL_ANALYSIS, study.getId(), study.getUuid(), auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditDistinct(organizationId, userId, Enums.Resource.CLINICAL_ANALYSIS, study.getId(), study.getUuid(), + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } - protected void fixQueryObject(Study study, Query query, String user, String token) throws CatalogException { + protected void fixQueryObject(String organizationId, Study study, Query query, String user, String token) throws CatalogException { // Fix query if it contains any annotation - AnnotationUtils.fixQueryAnnotationSearch(study, query); + AnnotationUtils.fixQueryAnnotationSearch(organizationId, study, query); changeQueryId(query, ParamConstants.CLINICAL_DISORDER_PARAM, ClinicalAnalysisDBAdaptor.QueryParams.DISORDER.key()); changeQueryId(query, ParamConstants.CLINICAL_ANALYST_ID_PARAM, ClinicalAnalysisDBAdaptor.QueryParams.ANALYSTS_ID.key()); @@ -1897,7 +1976,8 @@ protected void fixQueryObject(Study study, Query query, String user, String toke PanelDBAdaptor.QueryParams fieldFilter = catalogManager.getPanelManager().getFieldFilter(panelList); Query tmpQuery = new Query(fieldFilter.key(), panelList); - OpenCGAResult result = panelDBAdaptor.get(study.getUid(), tmpQuery, PanelManager.INCLUDE_PANEL_IDS, user); + OpenCGAResult result = getPanelDBAdaptor(organizationId).get(study.getUid(), tmpQuery, PanelManager.INCLUDE_PANEL_IDS, + user); if (result.getNumResults() > 0) { query.put(ClinicalAnalysisDBAdaptor.QueryParams.PANELS_UID.key(), result.getResults().stream().map(Panel::getUid).collect(Collectors.toList())); @@ -1913,7 +1993,7 @@ protected void fixQueryObject(Study study, Query query, String user, String toke FileDBAdaptor.QueryParams fieldFilter = catalogManager.getFileManager().getFieldFilter(fileList); Query tmpQuery = new Query(fieldFilter.key(), fileList); - OpenCGAResult result = fileDBAdaptor.get(study.getUid(), tmpQuery, FileManager.INCLUDE_FILE_IDS, user); + OpenCGAResult result = getFileDBAdaptor(organizationId).get(study.getUid(), tmpQuery, FileManager.INCLUDE_FILE_IDS, user); if (result.getNumResults() > 0) { query.put(ClinicalAnalysisDBAdaptor.QueryParams.FILES_UID.key(), result.getResults().stream().map(File::getUid).collect(Collectors.toList())); @@ -1929,8 +2009,8 @@ protected void fixQueryObject(Study study, Query query, String user, String toke IndividualDBAdaptor.QueryParams fieldFilter = catalogManager.getIndividualManager().getFieldFilter(probandList); Query tmpQuery = new Query(fieldFilter.key(), probandList); - OpenCGAResult result = individualDBAdaptor.get(study.getUid(), tmpQuery, IndividualManager.INCLUDE_INDIVIDUAL_IDS, - user); + OpenCGAResult result = getIndividualDBAdaptor(organizationId).get(study.getUid(), tmpQuery, + IndividualManager.INCLUDE_INDIVIDUAL_IDS, user); if (result.getNumResults() > 0) { query.put(ClinicalAnalysisDBAdaptor.QueryParams.PROBAND_UID.key(), result.getResults().stream().map(Individual::getUid).collect(Collectors.toList())); @@ -1945,7 +2025,8 @@ protected void fixQueryObject(Study study, Query query, String user, String toke SampleDBAdaptor.QueryParams fieldFilter = catalogManager.getSampleManager().getFieldFilter(sampleList); Query tmpQuery = new Query(fieldFilter.key(), sampleList); - OpenCGAResult result = sampleDBAdaptor.get(study.getUid(), tmpQuery, SampleManager.INCLUDE_SAMPLE_IDS, user); + OpenCGAResult result = getSampleDBAdaptor(organizationId).get(study.getUid(), tmpQuery, + SampleManager.INCLUDE_SAMPLE_IDS, user); if (result.getNumResults() > 0) { query.put(ClinicalAnalysisDBAdaptor.QueryParams.PROBAND_SAMPLES_UID.key(), result.getResults().stream().map(Sample::getUid).collect(Collectors.toList())); @@ -1960,7 +2041,8 @@ protected void fixQueryObject(Study study, Query query, String user, String toke FamilyDBAdaptor.QueryParams fieldFilter = catalogManager.getFamilyManager().getFieldFilter(familyList); Query tmpQuery = new Query(fieldFilter.key(), familyList); - OpenCGAResult result = familyDBAdaptor.get(study.getUid(), tmpQuery, FamilyManager.INCLUDE_FAMILY_IDS, user); + OpenCGAResult result = getFamilyDBAdaptor(organizationId).get(study.getUid(), tmpQuery, + FamilyManager.INCLUDE_FAMILY_IDS, user); if (result.getNumResults() > 0) { query.put(ClinicalAnalysisDBAdaptor.QueryParams.FAMILY_UID.key(), result.getResults().stream().map(Family::getUid).collect(Collectors.toList())); @@ -1975,7 +2057,8 @@ protected void fixQueryObject(Study study, Query query, String user, String toke IndividualDBAdaptor.QueryParams fieldFilter = catalogManager.getIndividualManager().getFieldFilter(probandList); Query tmpQuery = new Query(fieldFilter.key(), probandList); - OpenCGAResult result = individualDBAdaptor.get(study.getUid(), tmpQuery, IndividualManager.INCLUDE_INDIVIDUAL_IDS, + OpenCGAResult result = getIndividualDBAdaptor(organizationId).get(study.getUid(), tmpQuery, + IndividualManager.INCLUDE_INDIVIDUAL_IDS, user); if (result.getNumResults() > 0) { query.put(ClinicalAnalysisDBAdaptor.QueryParams.FAMILY_MEMBERS_UID.key(), @@ -1991,7 +2074,8 @@ protected void fixQueryObject(Study study, Query query, String user, String toke SampleDBAdaptor.QueryParams fieldFilter = catalogManager.getSampleManager().getFieldFilter(sampleList); Query tmpQuery = new Query(fieldFilter.key(), sampleList); - OpenCGAResult result = sampleDBAdaptor.get(study.getUid(), tmpQuery, SampleManager.INCLUDE_SAMPLE_IDS, user); + OpenCGAResult result = getSampleDBAdaptor(organizationId).get(study.getUid(), tmpQuery, + SampleManager.INCLUDE_SAMPLE_IDS, user); if (result.getNumResults() > 0) { query.put(ClinicalAnalysisDBAdaptor.QueryParams.FAMILY_MEMBERS_SAMPLES_UID.key(), result.getResults().stream().map(Sample::getUid).collect(Collectors.toList())); @@ -2008,7 +2092,8 @@ protected void fixQueryObject(Study study, Query query, String user, String toke IndividualDBAdaptor.QueryParams fieldFilter = catalogManager.getIndividualManager().getFieldFilter(individualList); Query tmpQuery = new Query(fieldFilter.key(), individualList); - OpenCGAResult result = individualDBAdaptor.get(study.getUid(), tmpQuery, IndividualManager.INCLUDE_INDIVIDUAL_IDS, + OpenCGAResult result = getIndividualDBAdaptor(organizationId).get(study.getUid(), tmpQuery, + IndividualManager.INCLUDE_INDIVIDUAL_IDS, user); if (result.getNumResults() > 0) { query.put(ClinicalAnalysisDBAdaptor.QueryParams.INDIVIDUAL.key(), @@ -2024,7 +2109,8 @@ protected void fixQueryObject(Study study, Query query, String user, String toke SampleDBAdaptor.QueryParams fieldFilter = catalogManager.getSampleManager().getFieldFilter(sampleList); Query tmpQuery = new Query(fieldFilter.key(), sampleList); - OpenCGAResult result = sampleDBAdaptor.get(study.getUid(), tmpQuery, SampleManager.INCLUDE_SAMPLE_IDS, user); + OpenCGAResult result = getSampleDBAdaptor(organizationId).get(study.getUid(), tmpQuery, + SampleManager.INCLUDE_SAMPLE_IDS, user); if (result.getNumResults() > 0) { query.put(ClinicalAnalysisDBAdaptor.QueryParams.SAMPLE.key(), result.getResults().stream().map(Sample::getUid).collect(Collectors.toList())); @@ -2035,28 +2121,30 @@ protected void fixQueryObject(Study study, Query query, String user, String toke } } - public OpenCGAResult count(String studyId, Query query, String token) - throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(token); - Study study = catalogManager.getStudyManager().resolveId(studyId, userId, StudyManager.INCLUDE_VARIABLE_SET); + public OpenCGAResult count(String studyId, Query query, String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyFqn, StudyManager.INCLUDE_VARIABLE_SET, tokenPayload); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) .append("query", new Query(query)) .append("token", token); try { - fixQueryObject(study, query, userId, token); + fixQueryObject(organizationId, study, query, userId, token); query.append(ClinicalAnalysisDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - OpenCGAResult queryResultAux = clinicalDBAdaptor.count(query, userId); + OpenCGAResult queryResultAux = getClinicalAnalysisDBAdaptor(organizationId).count(query, userId); - auditManager.auditCount(userId, Enums.Resource.CLINICAL_ANALYSIS, study.getId(), study.getUuid(), auditParams, + auditManager.auditCount(organizationId, userId, Enums.Resource.CLINICAL_ANALYSIS, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return new OpenCGAResult<>(queryResultAux.getTime(), queryResultAux.getEvents(), 0, Collections.emptyList(), queryResultAux.getNumMatches()); } catch (CatalogException e) { - auditManager.auditCount(userId, Enums.Resource.CLINICAL_ANALYSIS, study.getId(), study.getUuid(), auditParams, + auditManager.auditCount(organizationId, userId, Enums.Resource.CLINICAL_ANALYSIS, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -2074,8 +2162,11 @@ public OpenCGAResult delete(String studyStr, List clinicalAnalysisIds, Q throw new CatalogException("Missing list of Clinical Analysis ids"); } - String userId = catalogManager.getUserManager().getUserId(token); - Study study = studyManager.resolveId(studyStr, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, userId, organizationId); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -2091,10 +2182,11 @@ public OpenCGAResult delete(String studyStr, List clinicalAnalysisIds, Q boolean checkPermissions; try { // If the user is the owner or the admin, we won't check if he has permissions for every single entry - checkPermissions = !authorizationManager.isOwnerOrAdmin(study.getUid(), userId); + long studyId = study.getUid(); + checkPermissions = !authorizationManager.isAtLeastStudyAdministrator(organizationId, studyId, userId); } catch (CatalogException e) { - auditManager.auditDelete(operationId, userId, Enums.Resource.CLINICAL_ANALYSIS, "", "", study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditDelete(organizationId, operationId, userId, Enums.Resource.CLINICAL_ANALYSIS, "", "", study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -2104,7 +2196,7 @@ public OpenCGAResult delete(String studyStr, List clinicalAnalysisIds, Q String clinicalId = id; String clinicalUuid = ""; try { - OpenCGAResult internalResult = internalGet(study.getUid(), id, + OpenCGAResult internalResult = internalGet(organizationId, study.getUid(), id, keepFieldsInQueryOptions(INCLUDE_CLINICAL_INTERPRETATION_IDS, Arrays.asList( ClinicalAnalysisDBAdaptor.QueryParams.INTERPRETATION.key() + "." + InterpretationDBAdaptor.QueryParams.PRIMARY_FINDINGS_ID.key(), @@ -2121,7 +2213,7 @@ public OpenCGAResult delete(String studyStr, List clinicalAnalysisIds, Q clinicalUuid = clinicalAnalysis.getUuid(); if (checkPermissions) { - authorizationManager.checkClinicalAnalysisPermission(study.getUid(), clinicalAnalysis.getUid(), userId, + authorizationManager.checkClinicalAnalysisPermission(organizationId, study.getUid(), clinicalAnalysis.getUid(), userId, ClinicalAnalysisPermissions.DELETE); } @@ -2131,9 +2223,10 @@ public OpenCGAResult delete(String studyStr, List clinicalAnalysisIds, Q ClinicalAudit clinicalAudit = new ClinicalAudit(userId, ClinicalAudit.Action.DELETE_CLINICAL_ANALYSIS, "Delete Clinical Analysis '" + clinicalId + "'", TimeUtils.getTime()); - result.append(clinicalDBAdaptor.delete(clinicalAnalysis, Collections.singletonList(clinicalAudit))); + result.append(getClinicalAnalysisDBAdaptor(organizationId).delete(clinicalAnalysis, + Collections.singletonList(clinicalAudit))); - auditManager.auditDelete(operationId, userId, Enums.Resource.CLINICAL_ANALYSIS, clinicalAnalysis.getId(), + auditManager.auditDelete(organizationId, operationId, userId, Enums.Resource.CLINICAL_ANALYSIS, clinicalAnalysis.getId(), clinicalAnalysis.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { @@ -2144,11 +2237,11 @@ public OpenCGAResult delete(String studyStr, List clinicalAnalysisIds, Q result.setNumErrors(result.getNumErrors() + 1); logger.error(errorMsg); - auditManager.auditDelete(operationId, userId, Enums.Resource.CLINICAL_ANALYSIS, clinicalId, clinicalUuid, + auditManager.auditDelete(organizationId, operationId, userId, Enums.Resource.CLINICAL_ANALYSIS, clinicalId, clinicalUuid, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); } } - auditManager.finishAuditBatch(operationId); + auditManager.finishAuditBatch(organizationId, operationId); return endResult(result, ignoreException); } @@ -2185,8 +2278,11 @@ public OpenCGAResult delete(String studyStr, Query query, QueryOptions options, OpenCGAResult result = OpenCGAResult.empty(); - String userId = catalogManager.getUserManager().getUserId(token); - Study study = catalogManager.getStudyManager().resolveId(studyStr, userId, StudyManager.INCLUDE_VARIABLE_SET); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyFqn, StudyManager.INCLUDE_VARIABLE_SET, tokenPayload); String operationUuid = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -2203,17 +2299,19 @@ public OpenCGAResult delete(String studyStr, Query query, QueryOptions options, // We try to get an iterator containing all the ClinicalAnalyses to be deleted DBIterator iterator; try { - fixQueryObject(study, finalQuery, userId, token); + fixQueryObject(organizationId, study, finalQuery, userId, token); AnnotationUtils.fixQueryOptionAnnotation(options); finalQuery.append(ClinicalAnalysisDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - iterator = clinicalDBAdaptor.iterator(study.getUid(), finalQuery, INCLUDE_CLINICAL_INTERPRETATION_IDS, userId); + iterator = getClinicalAnalysisDBAdaptor(organizationId).iterator(study.getUid(), finalQuery, + INCLUDE_CLINICAL_INTERPRETATION_IDS, userId); // If the user is the owner or the admin, we won't check if he has permissions for every single entry - checkPermissions = !authorizationManager.isOwnerOrAdmin(study.getUid(), userId); + long studyId = study.getUid(); + checkPermissions = !authorizationManager.isAtLeastStudyAdministrator(organizationId, studyId, userId); } catch (CatalogException e) { - auditManager.auditDelete(operationUuid, userId, Enums.Resource.CLINICAL_ANALYSIS, "", "", study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.CLINICAL_ANALYSIS, "", "", study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -2223,7 +2321,7 @@ public OpenCGAResult delete(String studyStr, Query query, QueryOptions options, try { if (checkPermissions) { - authorizationManager.checkClinicalAnalysisPermission(study.getUid(), clinicalAnalysis.getUid(), userId, + authorizationManager.checkClinicalAnalysisPermission(organizationId, study.getUid(), clinicalAnalysis.getUid(), userId, ClinicalAnalysisPermissions.DELETE); } @@ -2233,9 +2331,10 @@ public OpenCGAResult delete(String studyStr, Query query, QueryOptions options, ClinicalAudit clinicalAudit = new ClinicalAudit(userId, ClinicalAudit.Action.DELETE_CLINICAL_ANALYSIS, "Delete Clinical Analysis '" + clinicalAnalysis.getId() + "'", TimeUtils.getTime()); - result.append(clinicalDBAdaptor.delete(clinicalAnalysis, Collections.singletonList(clinicalAudit))); + result.append(getClinicalAnalysisDBAdaptor(organizationId).delete(clinicalAnalysis, + Collections.singletonList(clinicalAudit))); - auditManager.auditDelete(operationUuid, userId, Enums.Resource.CLINICAL_ANALYSIS, clinicalAnalysis.getId(), + auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.CLINICAL_ANALYSIS, clinicalAnalysis.getId(), clinicalAnalysis.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { @@ -2246,47 +2345,50 @@ public OpenCGAResult delete(String studyStr, Query query, QueryOptions options, result.setNumErrors(result.getNumErrors() + 1); logger.error(errorMsg); - auditManager.auditDelete(operationUuid, userId, Enums.Resource.CLINICAL_ANALYSIS, clinicalAnalysis.getId(), + auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.CLINICAL_ANALYSIS, clinicalAnalysis.getId(), clinicalAnalysis.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); } } - auditManager.finishAuditBatch(operationUuid); + auditManager.finishAuditBatch(organizationId, operationUuid); return endResult(result, ignoreException); } @Override - public OpenCGAResult rank(String studyStr, Query query, String field, int numResults, boolean asc, String sessionId) - throws CatalogException { + public OpenCGAResult rank(String studyStr, Query query, String field, int numResults, boolean asc, + String sessionId) throws CatalogException { return null; } @Override - public OpenCGAResult groupBy(@Nullable String studyStr, Query query, List fields, QueryOptions options, String sessionId) - throws CatalogException { + public OpenCGAResult groupBy(@Nullable String studyStr, Query query, List fields, QueryOptions options, + String sessionId) throws CatalogException { query = ParamUtils.defaultObject(query, Query::new); options = ParamUtils.defaultObject(options, QueryOptions::new); if (fields == null || fields.size() == 0) { throw new CatalogException("Empty fields parameter."); } - String userId = catalogManager.getUserManager().getUserId(sessionId); - Study study = catalogManager.getStudyManager().resolveId(studyStr, userId, StudyManager.INCLUDE_VARIABLE_SET); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(sessionId); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyFqn, StudyManager.INCLUDE_VARIABLE_SET, tokenPayload); - fixQueryObject(study, query, userId, sessionId); + fixQueryObject(organizationId, study, query, userId, sessionId); AnnotationUtils.fixQueryOptionAnnotation(options); // Add study id to the query query.put(FamilyDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - OpenCGAResult queryResult = clinicalDBAdaptor.groupBy(query, fields, options, userId); + OpenCGAResult queryResult = getClinicalAnalysisDBAdaptor(organizationId).groupBy(query, fields, options, userId); return ParamUtils.defaultObject(queryResult, OpenCGAResult::new); } public OpenCGAResult updateAnnotations(String studyStr, String clinicalStr, String annotationSetId, - Map annotations, ParamUtils.CompleteUpdateAction action, - QueryOptions options, String token) throws CatalogException { + Map annotations, ParamUtils.CompleteUpdateAction action, + QueryOptions options, String token) throws CatalogException { if (annotations == null || annotations.isEmpty()) { throw new CatalogException("Missing array of annotations."); } @@ -2299,16 +2401,19 @@ public OpenCGAResult updateAnnotations(String studyStr, String } // ************************** ACLs ******************************** // - public OpenCGAResult> getAcls( - String studyStr, List clinicalList, String member, boolean ignoreException, String token) throws CatalogException { - return getAcls(studyStr, clinicalList, StringUtils.isNotEmpty(member) ? Collections.singletonList(member) : Collections.emptyList(), - ignoreException, token); + public OpenCGAResult> getAcls(String studyStr, List clinicalList, String member, + boolean ignoreException, String token) throws CatalogException { + return getAcls(studyStr, clinicalList, + StringUtils.isNotEmpty(member) ? Collections.singletonList(member) : Collections.emptyList(), ignoreException, token); } public OpenCGAResult> getAcls(String studyId, List clinicalList, List members, boolean ignoreException, String token) throws CatalogException { - String user = userManager.getUserId(token); - Study study = studyManager.resolveId(studyId, user); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyId, userId, organizationId); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); ObjectMap auditParams = new ObjectMap() @@ -2322,8 +2427,8 @@ public OpenCGAResult> getAcls(String s Map missingMap = new HashMap<>(); try { auditManager.initAuditBatch(operationId); - InternalGetDataResult queryResult = internalGet(study.getUid(), clinicalList, INCLUDE_CLINICAL_IDS, user, - ignoreException); + InternalGetDataResult queryResult = internalGet(organizationId, study.getUid(), clinicalList, + INCLUDE_CLINICAL_IDS, userId, ignoreException); if (queryResult.getMissing() != null) { missingMap = queryResult.getMissing().stream() @@ -2332,11 +2437,11 @@ public OpenCGAResult> getAcls(String s List clinicalUids = queryResult.getResults().stream().map(ClinicalAnalysis::getUid).collect(Collectors.toList()); if (CollectionUtils.isNotEmpty(members)) { - clinicalAcls = authorizationManager.getAcl(user, study.getUid(), clinicalUids, members, Enums.Resource.CLINICAL_ANALYSIS, - ClinicalAnalysisPermissions.class); + clinicalAcls = authorizationManager.getAcl(organizationId, study.getUid(), clinicalUids, members, + Enums.Resource.CLINICAL_ANALYSIS, ClinicalAnalysisPermissions.class, userId); } else { - clinicalAcls = authorizationManager.getAcl(user, study.getUid(), clinicalUids, Enums.Resource.CLINICAL_ANALYSIS, - ClinicalAnalysisPermissions.class); + clinicalAcls = authorizationManager.getAcl(organizationId, study.getUid(), clinicalUids, Enums.Resource.CLINICAL_ANALYSIS, + ClinicalAnalysisPermissions.class, userId); } // Include non-existing samples to the result list @@ -2347,16 +2452,17 @@ public OpenCGAResult> getAcls(String s if (!missingMap.containsKey(clinicalId)) { ClinicalAnalysis clinical = queryResult.getResults().get(counter); resultList.add(clinicalAcls.getResults().get(counter)); - auditManager.audit(operationId, user, Enums.Action.FETCH_ACLS, Enums.Resource.CLINICAL_ANALYSIS, clinical.getId(), - clinical.getUuid(), study.getId(), study.getUuid(), auditParams, + auditManager.audit(organizationId, operationId, userId, Enums.Action.FETCH_ACLS, Enums.Resource.CLINICAL_ANALYSIS, + clinical.getId(), clinical.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS), new ObjectMap()); counter++; } else { resultList.add(new AclEntryList<>()); eventList.add(new Event(Event.Type.ERROR, clinicalId, missingMap.get(clinicalId).getErrorMsg())); - auditManager.audit(operationId, user, Enums.Action.FETCH_ACLS, Enums.Resource.CLINICAL_ANALYSIS, clinicalId, "", - study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, - new Error(0, "", missingMap.get(clinicalId).getErrorMsg())), new ObjectMap()); + auditManager.audit(organizationId, operationId, userId, Enums.Action.FETCH_ACLS, Enums.Resource.CLINICAL_ANALYSIS, + clinicalId, "", study.getId(), study.getUuid(), auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(0, "", + missingMap.get(clinicalId).getErrorMsg())), new ObjectMap()); } } for (int i = 0; i < queryResult.getResults().size(); i++) { @@ -2366,9 +2472,9 @@ public OpenCGAResult> getAcls(String s clinicalAcls.setEvents(eventList); } catch (CatalogException e) { for (String caseId : clinicalList) { - auditManager.audit(operationId, user, Enums.Action.FETCH_ACLS, Enums.Resource.CLINICAL_ANALYSIS, caseId, "", - study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError()), - new ObjectMap()); + auditManager.audit(organizationId, operationId, userId, Enums.Action.FETCH_ACLS, Enums.Resource.CLINICAL_ANALYSIS, caseId, + "", study.getId(), study.getUuid(), auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError()), new ObjectMap()); } if (!ignoreException) { throw e; @@ -2379,17 +2485,20 @@ public OpenCGAResult> getAcls(String s } } } finally { - auditManager.finishAuditBatch(operationId); + auditManager.finishAuditBatch(organizationId, operationId); } return clinicalAcls; } public OpenCGAResult> updateAcl( - String studyStr, List clinicalList, String memberIds, AclParams clinicalAclParams, ParamUtils.AclAction action, - boolean propagate, String token) throws CatalogException { - String user = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, user); + String studyStr, List clinicalList, String memberIds, AclParams clinicalAclParams, + ParamUtils.AclAction action, boolean propagate, String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("studyId", studyStr) @@ -2418,9 +2527,10 @@ public OpenCGAResult> updateAcl( checkPermissions(permissions, ClinicalAnalysisPermissions::valueOf); } - OpenCGAResult queryResult = internalGet(study.getUid(), clinicalList, INCLUDE_CATALOG_DATA, user, false); + OpenCGAResult queryResult = internalGet(organizationId, study.getUid(), clinicalList, INCLUDE_CATALOG_DATA, + userId, false); - authorizationManager.checkCanAssignOrSeePermissions(study.getUid(), user); + authorizationManager.checkCanAssignOrSeePermissions(organizationId, study.getUid(), userId); // Validate that the members are actually valid members List members; @@ -2430,7 +2540,7 @@ public OpenCGAResult> updateAcl( members = Collections.emptyList(); } authorizationManager.checkNotAssigningPermissionsToAdminsGroup(members); - checkMembers(study.getUid(), members); + checkMembers(organizationId, study.getUid(), members); List clinicalUidList = queryResult.getResults().stream().map(ClinicalAnalysis::getUid).collect(Collectors.toList()); List aclParamsList = new LinkedList<>(); @@ -2500,31 +2610,31 @@ public OpenCGAResult> updateAcl( OpenCGAResult> queryResults; switch (action) { case SET: - authorizationManager.setAcls(study.getUid(), members, aclParamsList); + authorizationManager.setAcls(organizationId, study.getUid(), members, aclParamsList); break; case ADD: - authorizationManager.addAcls(study.getUid(), members, aclParamsList); + authorizationManager.addAcls(organizationId, study.getUid(), members, aclParamsList); break; case REMOVE: - authorizationManager.removeAcls(members, aclParamsList); + authorizationManager.removeAcls(organizationId, members, aclParamsList); break; case RESET: for (AuthorizationManager.CatalogAclParams aclParams : aclParamsList) { aclParams.setPermissions(null); } - authorizationManager.removeAcls(members, aclParamsList); + authorizationManager.removeAcls(organizationId, members, aclParamsList); break; default: throw new CatalogException("Unexpected error occurred. No valid action found."); } - queryResults = authorizationManager.getAcls(study.getUid(), clinicalUidList, members, Enums.Resource.CLINICAL_ANALYSIS, - ClinicalAnalysisPermissions.class); + queryResults = authorizationManager.getAcls(organizationId, study.getUid(), clinicalUidList, members, + Enums.Resource.CLINICAL_ANALYSIS, ClinicalAnalysisPermissions.class); for (int i = 0; i < queryResults.getResults().size(); i++) { queryResults.getResults().get(i).setId(queryResult.getResults().get(i).getId()); } for (ClinicalAnalysis clinicalAnalysis : queryResult.getResults()) { - auditManager.audit(operationUuid, user, Enums.Action.UPDATE_ACLS, Enums.Resource.CLINICAL_ANALYSIS, + auditManager.audit(organizationId, operationUuid, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.CLINICAL_ANALYSIS, clinicalAnalysis.getId(), clinicalAnalysis.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS), new ObjectMap()); } @@ -2533,21 +2643,24 @@ public OpenCGAResult> updateAcl( } catch (CatalogException e) { if (clinicalList != null) { for (String clinicalId : clinicalList) { - auditManager.audit(operationUuid, user, Enums.Action.UPDATE_ACLS, Enums.Resource.CLINICAL_ANALYSIS, clinicalId, "", - study.getId(), study.getUuid(), auditParams, + auditManager.audit(organizationId, operationUuid, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.CLINICAL_ANALYSIS, + clinicalId, "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError()), new ObjectMap()); } } throw e; } finally { - auditManager.finishAuditBatch(operationUuid); + auditManager.finishAuditBatch(organizationId, operationUuid); } } public OpenCGAResult configureStudy(String studyStr, ClinicalAnalysisStudyConfiguration clinicalConfiguration, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("studyId", studyStr) @@ -2555,7 +2668,7 @@ public OpenCGAResult configureStudy(String studyStr, ClinicalAnalysisStudyConfig .append("token", token); try { - authorizationManager.checkIsOwnerOrAdmin(study.getUid(), userId); + authorizationManager.checkIsAtLeastStudyAdministrator(organizationId, study.getUid(), userId); ParamUtils.checkObj(clinicalConfiguration, "ClinicalConfiguration"); ParamUtils.checkObj(clinicalConfiguration.getFlags(), "flags"); ParamUtils.checkObj(clinicalConfiguration.getPriorities(), "priorities"); @@ -2574,35 +2687,41 @@ public OpenCGAResult configureStudy(String studyStr, ClinicalAnalysisStudyConfig throw new CatalogException("Jackson casting error: " + e.getMessage(), e); } - OpenCGAResult updateResult = studyDBAdaptor.update(study.getUid(), update, QueryOptions.empty()); - auditManager.auditUpdate(userId, Enums.Resource.STUDY, study.getId(), study.getUuid(), study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + OpenCGAResult updateResult = getStudyDBAdaptor(organizationId).update(study.getUid(), update, QueryOptions.empty()); + auditManager.auditUpdate(organizationId, userId, Enums.Resource.STUDY, study.getId(), study.getUuid(), study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return updateResult; } catch (CatalogException e) { - auditManager.auditUpdate(userId, Enums.Resource.STUDY, study.getId(), study.getUuid(), study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditUpdate(organizationId, userId, Enums.Resource.STUDY, study.getId(), study.getUuid(), study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } - private void validateClinicalStatus(Map> status, String field) + private void validateClinicalStatus(List status, String field) throws CatalogException { - if (status == null) { + if (CollectionUtils.isEmpty(status)) { throw CatalogParameterException.isNull(field); } - for (ClinicalAnalysis.Type value : ClinicalAnalysis.Type.values()) { - if (!status.containsKey(value)) { - throw new CatalogParameterException(field + ": Missing status values for ClinicalAnalysis type '" + value + "'"); + // Ensure there's at least one status id per status type + Map presentMap = new HashMap<>(); + for (ClinicalStatusValue.ClinicalStatusType value : ClinicalStatusValue.ClinicalStatusType.values()) { + // Init map + presentMap.put(value, false); + } + for (ClinicalStatusValue clinicalStatusValue : status) { + if (StringUtils.isEmpty(clinicalStatusValue.getId())) { + throw CatalogParameterException.isNull(field + ".id"); } - List statuses = status.get(value); - for (ClinicalStatusValue clinicalStatusValue : statuses) { - if (StringUtils.isEmpty(clinicalStatusValue.getId())) { - throw CatalogParameterException.isNull(field + ".{" + value + "}.id"); - } - if (clinicalStatusValue.getType() == null) { - throw CatalogParameterException.isNull(field + ".{" + value + "}.type"); - } + if (clinicalStatusValue.getType() == null) { + throw CatalogParameterException.isNull(field + ".type"); + } + presentMap.put(clinicalStatusValue.getType(), true); + } + for (Map.Entry entry : presentMap.entrySet()) { + if (!entry.getValue()) { + throw new CatalogException("Missing status values for ClinicalStatus type '" + entry.getKey() + "'"); } } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CohortManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CohortManager.java index 36c14a4adc4..b7971c76106 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CohortManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CohortManager.java @@ -22,7 +22,10 @@ import org.apache.commons.lang3.time.StopWatch; import org.opencb.biodata.models.common.Status; import org.opencb.biodata.models.variant.StudyEntry; -import org.opencb.commons.datastore.core.*; +import org.opencb.commons.datastore.core.Event; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.result.Error; import org.opencb.commons.utils.ListUtils; import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager; @@ -32,15 +35,12 @@ import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.models.InternalGetDataResult; -import org.opencb.opencga.catalog.stats.solr.CatalogSolrManager; -import org.opencb.opencga.catalog.utils.AnnotationUtils; -import org.opencb.opencga.catalog.utils.Constants; -import org.opencb.opencga.catalog.utils.ParamUtils; -import org.opencb.opencga.catalog.utils.UuidUtils; +import org.opencb.opencga.catalog.utils.*; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.models.AclEntryList; import org.opencb.opencga.core.models.AclParams; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.audit.AuditRecord; import org.opencb.opencga.core.models.cohort.*; import org.opencb.opencga.core.models.common.AnnotationSet; @@ -56,7 +56,6 @@ import org.slf4j.LoggerFactory; import javax.annotation.Nullable; -import java.io.IOException; import java.util.*; import java.util.function.Function; import java.util.stream.Collectors; @@ -94,8 +93,8 @@ Enums.Resource getEntity() { } @Override - InternalGetDataResult internalGet(long studyUid, List entryList, @Nullable Query query, QueryOptions options, - String user, boolean ignoreException) throws CatalogException { + InternalGetDataResult internalGet(String organizationId, long studyUid, List entryList, @Nullable Query query, + QueryOptions options, String user, boolean ignoreException) throws CatalogException { if (ListUtils.isEmpty(entryList)) { throw new CatalogException("Missing cohort entries."); } @@ -125,13 +124,13 @@ InternalGetDataResult internalGet(long studyUid, List entryList, // Ensure the field by which we are querying for will be kept in the results queryOptions = keepFieldInQueryOptions(queryOptions, idQueryParam.key()); - OpenCGAResult cohortDataResult = cohortDBAdaptor.get(studyUid, queryCopy, queryOptions, user); + OpenCGAResult cohortDataResult = getCohortDBAdaptor(organizationId).get(studyUid, queryCopy, queryOptions, user); if (ignoreException || cohortDataResult.getNumResults() == uniqueList.size()) { return keepOriginalOrder(uniqueList, cohortStringFunction, cohortDataResult, ignoreException, false); } // Query without adding the user check - OpenCGAResult resultsNoCheck = cohortDBAdaptor.get(queryCopy, queryOptions); + OpenCGAResult resultsNoCheck = getCohortDBAdaptor(organizationId).get(queryCopy, queryOptions); if (resultsNoCheck.getNumResults() == cohortDataResult.getNumResults()) { throw CatalogException.notFound("cohorts", @@ -141,21 +140,25 @@ InternalGetDataResult internalGet(long studyUid, List entryList, } } - private OpenCGAResult getCohort(long studyUid, String cohortUuid, QueryOptions options) throws CatalogException { + private OpenCGAResult getCohort(String organizationId, long studyUid, String cohortUuid, QueryOptions options) + throws CatalogException { Query query = new Query() .append(CohortDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(CohortDBAdaptor.QueryParams.UUID.key(), cohortUuid); - return cohortDBAdaptor.get(query, options); + return getCohortDBAdaptor(organizationId).get(query, options); } - public OpenCGAResult create(String studyStr, CohortCreateParams cohortParams, String variableSetId, String variableId, - QueryOptions options, String token) throws CatalogException { + public OpenCGAResult create(String studyStr, CohortCreateParams cohortParams, String variableSetId, + String variableId, QueryOptions options, String token) throws CatalogException { ParamUtils.checkObj(cohortParams, "CohortCreateParams"); options = ParamUtils.defaultObject(options, QueryOptions::new); - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId, StudyManager.INCLUDE_VARIABLE_SET); - authorizationManager.checkStudyPermission(study.getUid(), userId, StudyPermissions.Permissions.WRITE_COHORTS); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, StudyManager.INCLUDE_VARIABLE_SET, userId, organizationId); + authorizationManager.checkStudyPermission(organizationId, study.getUid(), userId, StudyPermissions.Permissions.WRITE_COHORTS); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -186,7 +189,7 @@ public OpenCGAResult create(String studyStr, CohortCreateParams cohortPa throw new CatalogParameterException("Found samples with missing id and uuid."); } } - List sampleList = catalogManager.getSampleManager().internalGet(study.getUid(), sampleIds, + List sampleList = catalogManager.getSampleManager().internalGet(organizationId, study.getUid(), sampleIds, SampleManager.INCLUDE_SAMPLE_IDS, userId, false).getResults(); cohorts.add(new Cohort(cohortParams.getId(), cohortParams.getName(), cohortParams.getType(), cohortParams.getCreationDate(), cohortParams.getModificationDate(), cohortParams.getDescription(), sampleList, 0, @@ -241,7 +244,7 @@ public OpenCGAResult create(String studyStr, CohortCreateParams cohortPa Collections.emptyList(), cohortParams.getAnnotationSets(), -1, null)); } } catch (CatalogException e) { - auditManager.audit(operationId, userId, Enums.Action.CREATE, Enums.Resource.COHORT, "", "", study.getId(), + auditManager.audit(organizationId, operationId, userId, Enums.Action.CREATE, Enums.Resource.COHORT, "", "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -252,22 +255,25 @@ public OpenCGAResult create(String studyStr, CohortCreateParams cohortPa stopWatch.start(); for (Cohort cohort : cohorts) { try { - validateNewCohort(study, cohort); - OpenCGAResult tmpResult = cohortDBAdaptor.insert(study.getUid(), cohort, study.getVariableSets(), options); + validateNewCohort(organizationId, study, cohort); + OpenCGAResult tmpResult = getCohortDBAdaptor(organizationId).insert(study.getUid(), cohort, study.getVariableSets(), + options); insertResult.append(tmpResult); - auditManager.audit(operationId, userId, Enums.Action.CREATE, Enums.Resource.COHORT, cohort.getId(), cohort.getUuid(), - study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.audit(organizationId, operationId, userId, Enums.Action.CREATE, Enums.Resource.COHORT, cohort.getId(), + cohort.getUuid(), study.getId(), study.getUuid(), auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { Event event = new Event(Event.Type.ERROR, cohort.getId(), e.getMessage()); insertResult.getEvents().add(event); logger.error("Could not create cohort {}: {}", cohort.getId(), e.getMessage(), e); - auditManager.audit(operationId, userId, Enums.Action.CREATE, Enums.Resource.COHORT, cohort.getId(), cohort.getUuid(), - study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.audit(organizationId, operationId, userId, Enums.Action.CREATE, Enums.Resource.COHORT, cohort.getId(), + cohort.getUuid(), study.getId(), study.getUuid(), auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); } } - auditManager.finishAuditBatch(operationId); + auditManager.finishAuditBatch(organizationId, operationId); stopWatch.stop(); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { @@ -277,7 +283,7 @@ public OpenCGAResult create(String studyStr, CohortCreateParams cohortPa .map(Cohort::getUid) .filter(uid -> uid > 0) .collect(Collectors.toList())); - OpenCGAResult result = cohortDBAdaptor.get(study.getUid(), query, options, userId); + OpenCGAResult result = getCohortDBAdaptor(organizationId).get(study.getUid(), query, options, userId); insertResult.setResults(result.getResults()); } @@ -286,9 +292,12 @@ public OpenCGAResult create(String studyStr, CohortCreateParams cohortPa public OpenCGAResult generate(String studyStr, Query sampleQuery, Cohort cohort, QueryOptions options, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId, StudyManager.INCLUDE_VARIABLE_SET); - authorizationManager.checkStudyPermission(study.getUid(), userId, StudyPermissions.Permissions.WRITE_COHORTS); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, StudyManager.INCLUDE_VARIABLE_SET, userId, organizationId); + authorizationManager.checkStudyPermission(organizationId, study.getUid(), userId, StudyPermissions.Permissions.WRITE_COHORTS); ObjectMap auditParams = new ObjectMap() .append("studyId", studyStr) @@ -300,22 +309,22 @@ public OpenCGAResult generate(String studyStr, Query sampleQuery, Cohort options = ParamUtils.defaultObject(options, QueryOptions::new); try { // Fix sample query object and search for samples - catalogManager.getSampleManager().fixQueryObject(study, sampleQuery, userId); + catalogManager.getSampleManager().fixQueryObject(organizationId, study, sampleQuery, userId); AnnotationUtils.fixQueryOptionAnnotation(options); sampleQuery.append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - OpenCGAResult result = sampleDBAdaptor.get(study.getUid(), sampleQuery, options, userId); + OpenCGAResult result = getSampleDBAdaptor(organizationId).get(study.getUid(), sampleQuery, options, userId); // Add samples to provided cohort object cohort.setSamples(result.getResults()); - OpenCGAResult cohortResult = privateCreate(study, cohort, options, userId); - auditManager.audit(userId, Enums.Action.GENERATE, Enums.Resource.COHORT, cohort.getId(), cohort.getUuid(), study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + OpenCGAResult cohortResult = privateCreate(organizationId, study, cohort, options, userId); + auditManager.audit(organizationId, userId, Enums.Action.GENERATE, Enums.Resource.COHORT, cohort.getId(), cohort.getUuid(), + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return cohortResult; } catch (CatalogException e) { - auditManager.audit(userId, Enums.Action.GENERATE, Enums.Resource.COHORT, cohort.getId(), "", study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.audit(organizationId, userId, Enums.Action.GENERATE, Enums.Resource.COHORT, cohort.getId(), "", study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } @@ -324,11 +333,15 @@ public OpenCGAResult generate(String studyStr, Query sampleQuery, Cohort public OpenCGAResult create(String studyStr, Cohort cohort, QueryOptions options, String token) throws CatalogException { options = ParamUtils.defaultObject(options, QueryOptions::new); - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId, StudyManager.INCLUDE_VARIABLE_SET); - authorizationManager.checkStudyPermission(study.getUid(), userId, StudyPermissions.Permissions.WRITE_COHORTS); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, StudyManager.INCLUDE_VARIABLE_SET, userId, organizationId); + authorizationManager.checkStudyPermission(organizationId, study.getUid(), userId, StudyPermissions.Permissions.WRITE_COHORTS); ObjectMap auditParams = new ObjectMap() + .append("organizationId", organizationId) .append("study", studyStr) .append("cohort", cohort) .append("options", options) @@ -336,38 +349,39 @@ public OpenCGAResult create(String studyStr, Cohort cohort, QueryOptions try { if (CollectionUtils.isNotEmpty(cohort.getSamples())) { // Look for the samples - InternalGetDataResult sampleResult = catalogManager.getSampleManager().internalGet(study.getUid(), + InternalGetDataResult sampleResult = catalogManager.getSampleManager().internalGet(organizationId, study.getUid(), cohort.getSamples().stream().map(Sample::getId).collect(Collectors.toList()), SampleManager.INCLUDE_SAMPLE_IDS, userId, false); cohort.setSamples(sampleResult.getResults()); } - OpenCGAResult cohortResult = privateCreate(study, cohort, options, userId); - auditManager.auditCreate(userId, Enums.Resource.COHORT, cohort.getId(), cohort.getUuid(), study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + OpenCGAResult cohortResult = privateCreate(organizationId, study, cohort, options, userId); + auditManager.auditCreate(organizationId, userId, Enums.Resource.COHORT, cohort.getId(), cohort.getUuid(), study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return cohortResult; } catch (CatalogException e) { - auditManager.auditCreate(userId, Enums.Resource.COHORT, cohort.getId(), "", study.getId(), study.getUuid(), auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditCreate(organizationId, userId, Enums.Resource.COHORT, cohort.getId(), "", study.getId(), study.getUuid(), + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } - OpenCGAResult privateCreate(Study study, Cohort cohort, QueryOptions options, String userId) throws CatalogException { - authorizationManager.checkStudyPermission(study.getUid(), userId, StudyPermissions.Permissions.WRITE_COHORTS); - validateNewCohort(study, cohort); + OpenCGAResult privateCreate(String organizationId, Study study, Cohort cohort, QueryOptions options, String userId) + throws CatalogException { + authorizationManager.checkStudyPermission(organizationId, study.getUid(), userId, StudyPermissions.Permissions.WRITE_COHORTS); + validateNewCohort(organizationId, study, cohort); - OpenCGAResult insert = cohortDBAdaptor.insert(study.getUid(), cohort, study.getVariableSets(), options); + OpenCGAResult insert = getCohortDBAdaptor(organizationId).insert(study.getUid(), cohort, study.getVariableSets(), options); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { // Fetch created cohort - OpenCGAResult result = getCohort(study.getUid(), cohort.getUuid(), options); + OpenCGAResult result = getCohort(organizationId, study.getUid(), cohort.getUuid(), options); insert.setResults(result.getResults()); } return insert; } - void validateNewCohort(Study study, Cohort cohort) throws CatalogException { + void validateNewCohort(String organizationId, Study study, Cohort cohort) throws CatalogException { ParamUtils.checkObj(cohort, "Cohort"); ParamUtils.checkParameter(cohort.getId(), "id"); ParamUtils.checkObj(cohort.getSamples(), "Sample list"); @@ -403,23 +417,27 @@ void validateNewCohort(Study study, Cohort cohort) throws CatalogException { cohort.setNumSamples(cohort.getSamples().size()); } - public Long getStudyId(long cohortId) throws CatalogException { - return cohortDBAdaptor.getStudyId(cohortId); + public Long getStudyId(String organizationId, long cohortId) throws CatalogException { + return getCohortDBAdaptor(organizationId).getStudyId(cohortId); } /** * Fetch all the samples from a cohort. * - * @param studyStr Study id in string format. Could be one of [id|user@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy]. + * @param studyStr Study id in string format. Could be one of + * [id|organization@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy] * @param cohortStr Cohort id or name. - * @param sessionId Token of the user logged in. + * @param token Token of the user logged in. * @return a OpenCGAResult containing all the samples belonging to the cohort. * @throws CatalogException if there is any kind of error (permissions or invalid ids). */ - public OpenCGAResult getSamples(String studyStr, String cohortStr, String sessionId) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(sessionId); - Study study = catalogManager.getStudyManager().resolveId(studyStr, userId); - OpenCGAResult cohortDataResult = internalGet(study.getUid(), cohortStr, + public OpenCGAResult getSamples(String studyStr, String cohortStr, String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyStr, userId, organizationId); + OpenCGAResult cohortDataResult = internalGet(organizationId, study.getUid(), cohortStr, new QueryOptions(QueryOptions.INCLUDE, CohortDBAdaptor.QueryParams.SAMPLES.key()), userId); if (cohortDataResult == null || cohortDataResult.getNumResults() == 0) { @@ -438,17 +456,20 @@ public DBIterator iterator(String studyStr, Query query, QueryOptions op options = ParamUtils.defaultObject(options, QueryOptions::new); query = ParamUtils.defaultObject(query, Query::new); - String userId = userManager.getUserId(sessionId); - Study study = studyManager.resolveId(studyStr, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(sessionId); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, userId, organizationId); // Fix query if it contains any annotation Query finalQuery = new Query(query); - AnnotationUtils.fixQueryAnnotationSearch(study, finalQuery); + AnnotationUtils.fixQueryAnnotationSearch(organizationId, study, finalQuery); AnnotationUtils.fixQueryOptionAnnotation(options); - fixQueryObject(study, finalQuery, userId); + fixQueryObject(organizationId, study, finalQuery, userId); finalQuery.append(CohortDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - return cohortDBAdaptor.iterator(study.getUid(), finalQuery, options, userId); + return getCohortDBAdaptor(organizationId).iterator(study.getUid(), finalQuery, options, userId); } @Override @@ -456,9 +477,12 @@ public OpenCGAResult search(String studyId, Query query, QueryOptions op query = ParamUtils.defaultObject(query, Query::new); options = ParamUtils.defaultObject(options, QueryOptions::new); - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyId, userId, new QueryOptions(QueryOptions.INCLUDE, - StudyDBAdaptor.QueryParams.VARIABLE_SET.key())); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyId, new QueryOptions(QueryOptions.INCLUDE, + StudyDBAdaptor.QueryParams.VARIABLE_SET.key()), userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) @@ -467,19 +491,19 @@ public OpenCGAResult search(String studyId, Query query, QueryOptions op .append("token", token); try { // Fix query if it contains any annotation - AnnotationUtils.fixQueryAnnotationSearch(study, query); + AnnotationUtils.fixQueryAnnotationSearch(organizationId, study, query); AnnotationUtils.fixQueryOptionAnnotation(options); - fixQueryObject(study, query, userId); + fixQueryObject(organizationId, study, query, userId); query.append(CohortDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - OpenCGAResult queryResult = cohortDBAdaptor.get(study.getUid(), query, options, userId); + OpenCGAResult queryResult = getCohortDBAdaptor(organizationId).get(study.getUid(), query, options, userId); - auditManager.auditSearch(userId, Enums.Resource.COHORT, study.getId(), study.getUuid(), auditParams, + auditManager.auditSearch(organizationId, userId, Enums.Resource.COHORT, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return queryResult; } catch (CatalogException e) { - auditManager.auditSearch(userId, Enums.Resource.COHORT, study.getId(), study.getUuid(), auditParams, + auditManager.auditSearch(organizationId, userId, Enums.Resource.COHORT, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -489,9 +513,12 @@ public OpenCGAResult search(String studyId, Query query, QueryOptions op public OpenCGAResult distinct(String studyId, List fields, Query query, String token) throws CatalogException { query = ParamUtils.defaultObject(query, Query::new); - String userId = userManager.getUserId(token); - Study study = catalogManager.getStudyManager().resolveId(studyId, userId, new QueryOptions(QueryOptions.INCLUDE, - StudyDBAdaptor.QueryParams.VARIABLE_SET.key())); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyId, new QueryOptions(QueryOptions.INCLUDE, + StudyDBAdaptor.QueryParams.VARIABLE_SET.key()), userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) @@ -499,25 +526,25 @@ public OpenCGAResult distinct(String studyId, List fields, Query quer .append("query", new Query(query)) .append("token", token); try { - fixQueryObject(study, query, userId); + fixQueryObject(organizationId, study, query, userId); // Fix query if it contains any annotation - AnnotationUtils.fixQueryAnnotationSearch(study, query); + AnnotationUtils.fixQueryAnnotationSearch(organizationId, study, query); query.append(CohortDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - OpenCGAResult result = cohortDBAdaptor.distinct(study.getUid(), fields, query, userId); + OpenCGAResult result = getCohortDBAdaptor(organizationId).distinct(study.getUid(), fields, query, userId); - auditManager.auditDistinct(userId, Enums.Resource.COHORT, study.getId(), study.getUuid(), auditParams, + auditManager.auditDistinct(organizationId, userId, Enums.Resource.COHORT, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return result; } catch (CatalogException e) { - auditManager.auditDistinct(userId, Enums.Resource.COHORT, study.getId(), study.getUuid(), auditParams, + auditManager.auditDistinct(organizationId, userId, Enums.Resource.COHORT, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } - private void fixQueryObject(Study study, Query query, String userId) throws CatalogException { + private void fixQueryObject(String organizationId, Study study, Query query, String userId) throws CatalogException { super.fixQueryObject(query); changeQueryId(query, ParamConstants.COHORT_INTERNAL_STATUS_PARAM, CohortDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key()); @@ -525,7 +552,7 @@ private void fixQueryObject(Study study, Query query, String userId) throws Cata QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.UID.key()); // First look for the sample ids. - List sampleList = catalogManager.getSampleManager().internalGet(study.getUid(), + List sampleList = catalogManager.getSampleManager().internalGet(organizationId, study.getUid(), query.getAsStringList(ParamConstants.COHORT_SAMPLES_PARAM), SampleManager.INCLUDE_SAMPLE_IDS, userId, true) .getResults(); if (CollectionUtils.isNotEmpty(sampleList)) { @@ -544,9 +571,12 @@ private void fixQueryObject(Study study, Query query, String userId) throws Cata public OpenCGAResult count(String studyId, Query query, String token) throws CatalogException { query = ParamUtils.defaultObject(query, Query::new); - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyId, userId, new QueryOptions(QueryOptions.INCLUDE, - StudyDBAdaptor.QueryParams.VARIABLE_SET.key())); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyId, new QueryOptions(QueryOptions.INCLUDE, + StudyDBAdaptor.QueryParams.VARIABLE_SET.key()), userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) @@ -554,19 +584,19 @@ public OpenCGAResult count(String studyId, Query query, String token) th .append("token", token); try { // Fix query if it contains any annotation - AnnotationUtils.fixQueryAnnotationSearch(study, query); - fixQueryObject(study, query, userId); + AnnotationUtils.fixQueryAnnotationSearch(organizationId, study, query); + fixQueryObject(organizationId, study, query, userId); query.append(CohortDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - OpenCGAResult queryResultAux = cohortDBAdaptor.count(query, userId); + OpenCGAResult queryResultAux = getCohortDBAdaptor(organizationId).count(query, userId); - auditManager.auditCount(userId, Enums.Resource.COHORT, study.getId(), study.getUuid(), auditParams, + auditManager.auditCount(organizationId, userId, Enums.Resource.COHORT, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return new OpenCGAResult<>(queryResultAux.getTime(), queryResultAux.getEvents(), 0, Collections.emptyList(), queryResultAux.getNumMatches()); } catch (CatalogException e) { - auditManager.auditCount(userId, Enums.Resource.COHORT, study.getId(), study.getUuid(), auditParams, + auditManager.auditCount(organizationId, userId, Enums.Resource.COHORT, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -583,9 +613,12 @@ public OpenCGAResult delete(String studyStr, List cohortIds, ObjectMap p throw new CatalogException("Missing list of cohort ids"); } - String userId = catalogManager.getUserManager().getUserId(token); - Study study = studyManager.resolveId(studyStr, userId, new QueryOptions(QueryOptions.INCLUDE, - StudyDBAdaptor.QueryParams.VARIABLE_SET.key())); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, new QueryOptions(QueryOptions.INCLUDE, + StudyDBAdaptor.QueryParams.VARIABLE_SET.key()), userId, organizationId); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -599,9 +632,10 @@ public OpenCGAResult delete(String studyStr, List cohortIds, ObjectMap p boolean checkPermissions; try { // If the user is the owner or the admin, we won't check if he has permissions for every single entry - checkPermissions = !authorizationManager.isOwnerOrAdmin(study.getUid(), userId); + long studyId = study.getUid(); + checkPermissions = !authorizationManager.isAtLeastStudyAdministrator(organizationId, studyId, userId); } catch (CatalogException e) { - auditManager.auditDelete(operationId, userId, Enums.Resource.COHORT, "", "", study.getId(), study.getUuid(), + auditManager.auditDelete(organizationId, operationId, userId, Enums.Resource.COHORT, "", "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -613,7 +647,7 @@ public OpenCGAResult delete(String studyStr, List cohortIds, ObjectMap p String cohortId = id; String cohortUuid = ""; try { - OpenCGAResult internalResult = internalGet(study.getUid(), id, INCLUDE_COHORT_IDS, userId); + OpenCGAResult internalResult = internalGet(organizationId, study.getUid(), id, INCLUDE_COHORT_IDS, userId); if (internalResult.getNumResults() == 0) { throw new CatalogException("Cohort '" + id + "' not found"); } @@ -623,13 +657,13 @@ public OpenCGAResult delete(String studyStr, List cohortIds, ObjectMap p cohortUuid = internalResult.first().getUuid(); if (checkPermissions) { - authorizationManager.checkCohortPermission(study.getUid(), internalResult.first().getUid(), userId, + authorizationManager.checkCohortPermission(organizationId, study.getUid(), internalResult.first().getUid(), userId, CohortPermissions.DELETE); } - OpenCGAResult deleteResult = cohortDBAdaptor.delete(internalResult.first()); + OpenCGAResult deleteResult = getCohortDBAdaptor(organizationId).delete(internalResult.first()); result.append(deleteResult); - auditManager.auditDelete(operationId, userId, Enums.Resource.COHORT, internalResult.first().getId(), + auditManager.auditDelete(organizationId, operationId, userId, Enums.Resource.COHORT, internalResult.first().getId(), internalResult.first().getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { @@ -638,11 +672,11 @@ public OpenCGAResult delete(String studyStr, List cohortIds, ObjectMap p result.setNumErrors(result.getNumErrors() + 1); logger.error("Cannot delete cohort {}: {}", cohortId, e.getMessage()); - auditManager.auditDelete(operationId, userId, Enums.Resource.COHORT, cohortId, cohortUuid, study.getId(), + auditManager.auditDelete(organizationId, operationId, userId, Enums.Resource.COHORT, cohortId, cohortUuid, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); } } - auditManager.finishAuditBatch(operationId); + auditManager.finishAuditBatch(organizationId, operationId); return endResult(result, ignoreException); } @@ -657,9 +691,12 @@ public OpenCGAResult delete(String studyStr, Query query, ObjectMap params, bool Query finalQuery = new Query(ParamUtils.defaultObject(query, Query::new)); OpenCGAResult result = OpenCGAResult.empty(); - String userId = catalogManager.getUserManager().getUserId(token); - Study study = studyManager.resolveId(studyStr, userId, new QueryOptions(QueryOptions.INCLUDE, - StudyDBAdaptor.QueryParams.VARIABLE_SET.key())); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, new QueryOptions(QueryOptions.INCLUDE, + StudyDBAdaptor.QueryParams.VARIABLE_SET.key()), userId, organizationId); String operationUuid = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -676,16 +713,17 @@ public OpenCGAResult delete(String studyStr, Query query, ObjectMap params, bool // We try to get an iterator containing all the cohorts to be deleted DBIterator iterator; try { - AnnotationUtils.fixQueryAnnotationSearch(study, finalQuery); - fixQueryObject(study, finalQuery, userId); + AnnotationUtils.fixQueryAnnotationSearch(organizationId, study, finalQuery); + fixQueryObject(organizationId, study, finalQuery, userId); finalQuery.append(CohortDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - iterator = cohortDBAdaptor.iterator(study.getUid(), finalQuery, INCLUDE_COHORT_IDS, userId); + iterator = getCohortDBAdaptor(organizationId).iterator(study.getUid(), finalQuery, INCLUDE_COHORT_IDS, userId); // If the user is the owner or the admin, we won't check if he has permissions for every single entry - checkPermissions = !authorizationManager.isOwnerOrAdmin(study.getUid(), userId); + long studyId = study.getUid(); + checkPermissions = !authorizationManager.isAtLeastStudyAdministrator(organizationId, studyId, userId); } catch (CatalogException e) { - auditManager.auditDelete(operationUuid, userId, Enums.Resource.COHORT, "", "", study.getId(), study.getUuid(), + auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.COHORT, "", "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -695,13 +733,13 @@ public OpenCGAResult delete(String studyStr, Query query, ObjectMap params, bool Cohort cohort = iterator.next(); try { if (checkPermissions) { - authorizationManager.checkCohortPermission(study.getUid(), cohort.getUid(), userId, + authorizationManager.checkCohortPermission(organizationId, study.getUid(), cohort.getUid(), userId, CohortPermissions.DELETE); } - OpenCGAResult tmpResult = cohortDBAdaptor.delete(cohort); + OpenCGAResult tmpResult = getCohortDBAdaptor(organizationId).delete(cohort); result.append(tmpResult); - auditManager.auditDelete(operationUuid, userId, Enums.Resource.COHORT, cohort.getId(), cohort.getUuid(), + auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.COHORT, cohort.getId(), cohort.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { Event event = new Event(Event.Type.ERROR, cohort.getId(), e.getMessage()); @@ -709,11 +747,11 @@ public OpenCGAResult delete(String studyStr, Query query, ObjectMap params, bool result.setNumErrors(result.getNumErrors() + 1); logger.error("Cannot delete cohort {}: {}", cohort.getId(), e.getMessage()); - auditManager.auditDelete(operationUuid, userId, Enums.Resource.COHORT, cohort.getId(), cohort.getUuid(), + auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.COHORT, cohort.getId(), cohort.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); } } - auditManager.finishAuditBatch(operationUuid); + auditManager.finishAuditBatch(organizationId, operationUuid); return endResult(result, ignoreException); } @@ -736,7 +774,8 @@ public OpenCGAResult addAnnotationSet(String studyStr, String cohortStr, public OpenCGAResult addAnnotationSets(String studyStr, String cohortStr, List annotationSetList, QueryOptions options, String token) throws CatalogException { - return updateAnnotationSet(studyStr, cohortStr, annotationSetList, ParamUtils.BasicUpdateAction.ADD, options, token); + return updateAnnotationSet(studyStr, cohortStr, annotationSetList, ParamUtils.BasicUpdateAction.ADD, options, + token); } public OpenCGAResult removeAnnotationSet(String studyStr, String cohortStr, String annotationSetId, QueryOptions options, @@ -750,7 +789,8 @@ public OpenCGAResult removeAnnotationSets(String studyStr, String cohort .stream() .map(id -> new AnnotationSet().setId(id)) .collect(Collectors.toList()); - return updateAnnotationSet(studyStr, cohortStr, annotationSetList, ParamUtils.BasicUpdateAction.REMOVE, options, token); + return updateAnnotationSet(studyStr, cohortStr, annotationSetList, ParamUtils.BasicUpdateAction.REMOVE, options, + token); } public OpenCGAResult updateAnnotations(String studyStr, String cohortStr, String annotationSetId, @@ -769,14 +809,14 @@ public OpenCGAResult updateAnnotations(String studyStr, String cohortStr public OpenCGAResult removeAnnotations(String studyStr, String cohortStr, String annotationSetId, List annotations, QueryOptions options, String token) throws CatalogException { - return updateAnnotations(studyStr, cohortStr, annotationSetId, new ObjectMap("remove", StringUtils.join(annotations, ",")), - ParamUtils.CompleteUpdateAction.REMOVE, options, token); + return updateAnnotations(studyStr, cohortStr, annotationSetId, + new ObjectMap("remove", StringUtils.join(annotations, ",")), ParamUtils.CompleteUpdateAction.REMOVE, options, token); } - public OpenCGAResult resetAnnotations(String studyStr, String cohortStr, String annotationSetId, List annotations, - QueryOptions options, String token) throws CatalogException { - return updateAnnotations(studyStr, cohortStr, annotationSetId, new ObjectMap("reset", StringUtils.join(annotations, ",")), - ParamUtils.CompleteUpdateAction.RESET, options, token); + public OpenCGAResult resetAnnotations(String studyStr, String cohortStr, String annotationSetId, + List annotations, QueryOptions options, String token) throws CatalogException { + return updateAnnotations(studyStr, cohortStr, annotationSetId, + new ObjectMap("reset", StringUtils.join(annotations, ",")), ParamUtils.CompleteUpdateAction.RESET, options, token); } public OpenCGAResult update(String studyStr, String cohortId, CohortUpdateParams updateParams, QueryOptions options, @@ -784,10 +824,13 @@ public OpenCGAResult update(String studyStr, String cohortId, CohortUpda return update(studyStr, cohortId, updateParams, false, options, token); } - public OpenCGAResult update(String studyStr, String cohortId, CohortUpdateParams updateParams, boolean allowModifyCohortAll, - QueryOptions options, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId, StudyManager.INCLUDE_VARIABLE_SET); + public OpenCGAResult update(String studyStr, String cohortId, CohortUpdateParams updateParams, + boolean allowModifyCohortAll, QueryOptions options, String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, StudyManager.INCLUDE_VARIABLE_SET, userId, organizationId); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -810,7 +853,7 @@ public OpenCGAResult update(String studyStr, String cohortId, CohortUpda String cohortUuid = ""; try { - OpenCGAResult internalResult = internalGet(study.getUid(), cohortId, INCLUDE_COHORT_STATUS, userId); + OpenCGAResult internalResult = internalGet(organizationId, study.getUid(), cohortId, INCLUDE_COHORT_STATUS, userId); if (internalResult.getNumResults() == 0) { throw new CatalogException("Cohort '" + cohortId + "' not found"); } @@ -820,18 +863,18 @@ public OpenCGAResult update(String studyStr, String cohortId, CohortUpda cohortId = cohort.getId(); cohortUuid = cohort.getUuid(); - OpenCGAResult updateResult = update(study, cohort, updateParams, allowModifyCohortAll, options, userId); + OpenCGAResult updateResult = update(organizationId, study, cohort, updateParams, allowModifyCohortAll, options, userId); result.append(updateResult); - auditManager.auditUpdate(operationId, userId, Enums.Resource.COHORT, cohort.getId(), cohort.getUuid(), study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.COHORT, cohort.getId(), cohort.getUuid(), + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { Event event = new Event(Event.Type.ERROR, cohortId, e.getMessage()); result.getEvents().add(event); result.setNumErrors(result.getNumErrors() + 1); logger.error("Could not update cohort {}: {}", cohortId, e.getMessage(), e); - auditManager.auditUpdate(operationId, userId, Enums.Resource.COHORT, cohortId, cohortUuid, study.getId(), + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.COHORT, cohortId, cohortUuid, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -839,15 +882,15 @@ public OpenCGAResult update(String studyStr, String cohortId, CohortUpda return result; } - public OpenCGAResult update(String studyStr, List cohortIds, CohortUpdateParams updateParams, QueryOptions options, - String token) throws CatalogException { + public OpenCGAResult update(String studyStr, List cohortIds, CohortUpdateParams updateParams, + QueryOptions options, String token) throws CatalogException { return update(studyStr, cohortIds, updateParams, false, false, options, token); } /** * Update a Cohort from catalog. * - * @param studyStr Study id in string format. Could be one of [studyId|projectId:studyId|user@projectId:studyId]. + * @param studyStr Study id in string format. Could be one of [studyId|projectId:studyId|organizationId@projectId:studyId]. * @param cohortIds List of cohort ids. Could be either the id or uuid. * @param updateParams Data model filled only with the parameters to be updated. * @param allowModifyCohortAll Boolean indicating whether we should not raise an exception if the cohort ALL is to be updated. @@ -861,8 +904,11 @@ public OpenCGAResult update(String studyStr, List cohortIds, Coh public OpenCGAResult update(String studyStr, List cohortIds, CohortUpdateParams updateParams, boolean allowModifyCohortAll, boolean ignoreException, QueryOptions options, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId, StudyManager.INCLUDE_VARIABLE_SET); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, StudyManager.INCLUDE_VARIABLE_SET, userId, organizationId); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -889,7 +935,7 @@ public OpenCGAResult update(String studyStr, List cohortIds, Coh String cohortUuid = ""; try { - OpenCGAResult internalResult = internalGet(study.getUid(), id, INCLUDE_COHORT_STATUS, userId); + OpenCGAResult internalResult = internalGet(organizationId, study.getUid(), id, INCLUDE_COHORT_STATUS, userId); if (internalResult.getNumResults() == 0) { throw new CatalogException("Cohort '" + id + "' not found"); } @@ -899,22 +945,23 @@ public OpenCGAResult update(String studyStr, List cohortIds, Coh cohortId = cohort.getId(); cohortUuid = cohort.getUuid(); - OpenCGAResult updateResult = update(study, cohort, updateParams, allowModifyCohortAll, options, userId); + OpenCGAResult updateResult = update(organizationId, study, cohort, updateParams, allowModifyCohortAll, options, + userId); result.append(updateResult); - auditManager.auditUpdate(operationId, userId, Enums.Resource.COHORT, cohort.getId(), cohort.getUuid(), study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.COHORT, cohort.getId(), cohort.getUuid(), + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { Event event = new Event(Event.Type.ERROR, cohortId, e.getMessage()); result.getEvents().add(event); result.setNumErrors(result.getNumErrors() + 1); logger.error("Could not update cohort {}: {}", cohortId, e.getMessage(), e); - auditManager.auditUpdate(operationId, userId, Enums.Resource.COHORT, cohortId, cohortUuid, study.getId(), + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.COHORT, cohortId, cohortUuid, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); } } - auditManager.finishAuditBatch(operationId); + auditManager.finishAuditBatch(organizationId, operationId); return endResult(result, ignoreException); } @@ -922,7 +969,8 @@ public OpenCGAResult update(String studyStr, List cohortIds, Coh /** * Update a Cohort from catalog. * - * @param studyStr Study id in string format. Could be one of [id|user@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy]. + * @param studyStr Study id in string format. Could be one of + * [id|organization@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy] * @param query Query object. * @param updateParams Data model filled only with the parameters to be updated. * @param options QueryOptions object. @@ -940,8 +988,11 @@ public OpenCGAResult update(String studyStr, Query query, CohortUpdatePa boolean ignoreException, QueryOptions options, String token) throws CatalogException { Query finalQuery = new Query(ParamUtils.defaultObject(query, Query::new)); - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId, StudyManager.INCLUDE_VARIABLE_SET); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, StudyManager.INCLUDE_VARIABLE_SET, userId, organizationId); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -962,13 +1013,13 @@ public OpenCGAResult update(String studyStr, Query query, CohortUpdatePa DBIterator iterator; try { - AnnotationUtils.fixQueryAnnotationSearch(study, finalQuery); - fixQueryObject(study, finalQuery, userId); + AnnotationUtils.fixQueryAnnotationSearch(organizationId, study, finalQuery); + fixQueryObject(organizationId, study, finalQuery, userId); finalQuery.append(CohortDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - iterator = cohortDBAdaptor.iterator(study.getUid(), finalQuery, INCLUDE_COHORT_STATUS, userId); + iterator = getCohortDBAdaptor(organizationId).iterator(study.getUid(), finalQuery, INCLUDE_COHORT_STATUS, userId); } catch (CatalogException e) { - auditManager.auditUpdate(operationId, userId, Enums.Resource.COHORT, "", "", study.getId(), study.getUuid(), + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.COHORT, "", "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -978,10 +1029,11 @@ public OpenCGAResult update(String studyStr, Query query, CohortUpdatePa while (iterator.hasNext()) { Cohort cohort = iterator.next(); try { - OpenCGAResult queryResult = update(study, cohort, updateParams, allowModifyCohortAll, options, userId); + OpenCGAResult queryResult = update(organizationId, study, cohort, updateParams, allowModifyCohortAll, options, + userId); result.append(queryResult); - auditManager.auditUpdate(operationId, userId, Enums.Resource.COHORT, cohort.getId(), cohort.getUuid(), + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.COHORT, cohort.getId(), cohort.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { Event event = new Event(Event.Type.ERROR, cohort.getId(), e.getMessage()); @@ -989,17 +1041,17 @@ public OpenCGAResult update(String studyStr, Query query, CohortUpdatePa result.setNumErrors(result.getNumErrors() + 1); logger.error("Could not update cohort {}: {}", cohort.getId(), e.getMessage(), e); - auditManager.auditUpdate(operationId, userId, Enums.Resource.COHORT, cohort.getId(), cohort.getUuid(), + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.COHORT, cohort.getId(), cohort.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); } } - auditManager.finishAuditBatch(operationId); + auditManager.finishAuditBatch(organizationId, operationId); return endResult(result, ignoreException); } - private OpenCGAResult update(Study study, Cohort cohort, CohortUpdateParams updateParams, boolean allowModifyCohortAll, - QueryOptions options, String userId) throws CatalogException { + private OpenCGAResult update(String organizationId, Study study, Cohort cohort, CohortUpdateParams updateParams, + boolean allowModifyCohortAll, QueryOptions options, String userId) throws CatalogException { options = ParamUtils.defaultObject(options, QueryOptions::new); if (StringUtils.isNotEmpty(updateParams.getCreationDate())) { @@ -1031,13 +1083,13 @@ private OpenCGAResult update(Study study, Cohort cohort, CohortUpdatePar // Check permissions... // Only check write annotation permissions if the user wants to update the annotation sets if (updateParams != null && updateParams.getAnnotationSets() != null) { - authorizationManager.checkCohortPermission(study.getUid(), cohort.getUid(), userId, + authorizationManager.checkCohortPermission(organizationId, study.getUid(), cohort.getUid(), userId, CohortPermissions.WRITE_ANNOTATIONS); } // Only check update permissions if the user wants to update anything apart from the annotation sets if ((parameters.size() == 1 && !parameters.containsKey(CohortDBAdaptor.QueryParams.ANNOTATION_SETS.key())) || parameters.size() > 1) { - authorizationManager.checkCohortPermission(study.getUid(), cohort.getUid(), userId, + authorizationManager.checkCohortPermission(organizationId, study.getUid(), cohort.getUid(), userId, CohortPermissions.WRITE); } @@ -1079,7 +1131,7 @@ private OpenCGAResult update(Study study, Cohort cohort, CohortUpdatePar } } - InternalGetDataResult sampleResult = catalogManager.getSampleManager().internalGet(study.getUid(), + InternalGetDataResult sampleResult = catalogManager.getSampleManager().internalGet(organizationId, study.getUid(), new ArrayList<>(sampleIds), SampleManager.INCLUDE_SAMPLE_IDS, userId, false); if (sampleResult.getNumResults() != sampleIds.size()) { @@ -1091,11 +1143,13 @@ private OpenCGAResult update(Study study, Cohort cohort, CohortUpdatePar } } - checkUpdateAnnotations(study, cohort, parameters, options, VariableSet.AnnotableDataModels.COHORT, cohortDBAdaptor, userId); - OpenCGAResult update = cohortDBAdaptor.update(cohort.getUid(), parameters, study.getVariableSets(), options); + checkUpdateAnnotations(organizationId, study, cohort, parameters, options, VariableSet.AnnotableDataModels.COHORT, + getCohortDBAdaptor(organizationId), userId); + OpenCGAResult update = getCohortDBAdaptor(organizationId).update(cohort.getUid(), parameters, study.getVariableSets(), + options); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { // Fetch updated cohort - OpenCGAResult result = cohortDBAdaptor.get(study.getUid(), + OpenCGAResult result = getCohortDBAdaptor(organizationId).get(study.getUid(), new Query(CohortDBAdaptor.QueryParams.UID.key(), cohort.getUid()), options, userId); update.setResults(result.getResults()); } @@ -1109,24 +1163,30 @@ public OpenCGAResult groupBy(@Nullable String studyStr, Query query, List> getAcls(String studyId, List cohortList, String member, boolean ignoreException, String token) throws CatalogException { - return getAcls(studyId, cohortList, StringUtils.isNotEmpty(member) ? Collections.singletonList(member) : Collections.emptyList(), + return getAcls(studyId, cohortList, + StringUtils.isNotEmpty(member) ? Collections.singletonList(member) : Collections.emptyList(), ignoreException, token); } public OpenCGAResult> getAcls(String studyId, List cohortList, List members, boolean ignoreException, String token) throws CatalogException { - String user = userManager.getUserId(token); - Study study = studyManager.resolveId(studyId, user); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyId, userId, organizationId); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); ObjectMap auditParams = new ObjectMap() @@ -1212,7 +1279,8 @@ public OpenCGAResult> getAcls(String studyId, Li Map missingMap = new HashMap<>(); try { auditManager.initAuditBatch(operationId); - InternalGetDataResult queryResult = internalGet(study.getUid(), cohortList, INCLUDE_COHORT_IDS, user, ignoreException); + InternalGetDataResult queryResult = internalGet(organizationId, study.getUid(), cohortList, INCLUDE_COHORT_IDS, userId, + ignoreException); if (queryResult.getMissing() != null) { missingMap = queryResult.getMissing().stream() @@ -1221,10 +1289,11 @@ public OpenCGAResult> getAcls(String studyId, Li List cohortUids = queryResult.getResults().stream().map(Cohort::getUid).collect(Collectors.toList()); if (CollectionUtils.isNotEmpty(members)) { - cohortAcls = authorizationManager.getAcl(user, study.getUid(), cohortUids, members, Enums.Resource.COHORT, - CohortPermissions.class); + cohortAcls = authorizationManager.getAcl(organizationId, study.getUid(), cohortUids, members, Enums.Resource.COHORT, + CohortPermissions.class, userId); } else { - cohortAcls = authorizationManager.getAcl(user, study.getUid(), cohortUids, Enums.Resource.COHORT, CohortPermissions.class); + cohortAcls = authorizationManager.getAcl(organizationId, study.getUid(), cohortUids, Enums.Resource.COHORT, + CohortPermissions.class, userId); } // Include non-existing cohorts to the result list @@ -1235,15 +1304,15 @@ public OpenCGAResult> getAcls(String studyId, Li if (!missingMap.containsKey(cohortId)) { Cohort cohort = queryResult.getResults().get(counter); resultList.add(cohortAcls.getResults().get(counter)); - auditManager.audit(operationId, user, Enums.Action.FETCH_ACLS, Enums.Resource.COHORT, cohort.getId(), cohort.getUuid(), - study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS), - new ObjectMap()); + auditManager.audit(organizationId, operationId, userId, Enums.Action.FETCH_ACLS, Enums.Resource.COHORT, cohort.getId(), + cohort.getUuid(), study.getId(), study.getUuid(), auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS), new ObjectMap()); counter++; } else { resultList.add(new AclEntryList<>()); eventList.add(new Event(Event.Type.ERROR, cohortId, missingMap.get(cohortId).getErrorMsg())); - auditManager.audit(operationId, user, Enums.Action.FETCH_ACLS, Enums.Resource.COHORT, cohortId, "", study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, + auditManager.audit(organizationId, operationId, userId, Enums.Action.FETCH_ACLS, Enums.Resource.COHORT, cohortId, "", + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(0, "", missingMap.get(cohortId).getErrorMsg())), new ObjectMap()); } } @@ -1254,7 +1323,7 @@ public OpenCGAResult> getAcls(String studyId, Li cohortAcls.setEvents(eventList); } catch (CatalogException e) { for (String cohortId : cohortList) { - auditManager.audit(operationId, user, Enums.Action.FETCH_ACLS, Enums.Resource.COHORT, cohortId, "", + auditManager.audit(organizationId, operationId, userId, Enums.Action.FETCH_ACLS, Enums.Resource.COHORT, cohortId, "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError()), new ObjectMap()); } @@ -1267,18 +1336,20 @@ public OpenCGAResult> getAcls(String studyId, Li } } } finally { - auditManager.finishAuditBatch(operationId); + auditManager.finishAuditBatch(organizationId, operationId); } return cohortAcls; } - public OpenCGAResult> updateAcl(String studyId, List cohortStrList, - String memberList, AclParams aclParams, - ParamUtils.AclAction action, String token) + public OpenCGAResult> updateAcl(String studyId, List cohortStrList, String memberList, + AclParams aclParams, ParamUtils.AclAction action, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyId, userId, StudyManager.INCLUDE_STUDY_UID); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyId, StudyManager.INCLUDE_STUDY_UID, userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) @@ -1306,9 +1377,10 @@ public OpenCGAResult> updateAcl(String studyId, checkPermissions(permissions, CohortPermissions::valueOf); } - List cohortList = internalGet(study.getUid(), cohortStrList, INCLUDE_COHORT_IDS, userId, false).getResults(); + List cohortList = internalGet(organizationId, study.getUid(), cohortStrList, INCLUDE_COHORT_IDS, userId, false) + .getResults(); - authorizationManager.checkCanAssignOrSeePermissions(study.getUid(), userId); + authorizationManager.checkCanAssignOrSeePermissions(organizationId, study.getUid(), userId); // Validate that the members are actually valid members List members; @@ -1318,7 +1390,7 @@ public OpenCGAResult> updateAcl(String studyId, members = Collections.emptyList(); } authorizationManager.checkNotAssigningPermissionsToAdminsGroup(members); - checkMembers(study.getUid(), members); + checkMembers(organizationId, study.getUid(), members); List cohortUids = cohortList.stream().map(Cohort::getUid).collect(Collectors.toList()); @@ -1328,29 +1400,29 @@ public OpenCGAResult> updateAcl(String studyId, OpenCGAResult> queryResultList; switch (action) { case SET: - authorizationManager.setAcls(study.getUid(), members, catalogAclParams); + authorizationManager.setAcls(organizationId, study.getUid(), members, catalogAclParams); break; case ADD: - authorizationManager.addAcls(study.getUid(), members, catalogAclParams); + authorizationManager.addAcls(organizationId, study.getUid(), members, catalogAclParams); break; case REMOVE: - authorizationManager.removeAcls(members, catalogAclParams); + authorizationManager.removeAcls(organizationId, members, catalogAclParams); break; case RESET: catalogAclParams.setPermissions(null); - authorizationManager.removeAcls(members, catalogAclParams); + authorizationManager.removeAcls(organizationId, members, catalogAclParams); break; default: throw new CatalogException("Unexpected error occurred. No valid action found."); } - queryResultList = authorizationManager.getAcls(study.getUid(), cohortUids, members, Enums.Resource.COHORT, + queryResultList = authorizationManager.getAcls(organizationId, study.getUid(), cohortUids, members, Enums.Resource.COHORT, CohortPermissions.class); for (int i = 0; i < queryResultList.getResults().size(); i++) { queryResultList.getResults().get(i).setId(cohortList.get(i).getId()); } for (Cohort cohort : cohortList) { - auditManager.audit(operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.COHORT, cohort.getId(), + auditManager.audit(organizationId, operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.COHORT, cohort.getId(), cohort.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS), new ObjectMap()); } @@ -1358,55 +1430,15 @@ public OpenCGAResult> updateAcl(String studyId, } catch (CatalogException e) { if (cohortStrList != null) { for (String cohortId : cohortStrList) { - auditManager.audit(operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.COHORT, cohortId, "", + auditManager.audit(organizationId, operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.COHORT, cohortId, "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError()), new ObjectMap()); } } throw e; } finally { - auditManager.finishAuditBatch(operationId); + auditManager.finishAuditBatch(organizationId, operationId); } } - public DataResult facet(String studyId, Query query, QueryOptions options, boolean defaultStats, String token) - throws CatalogException, IOException { - ParamUtils.defaultObject(query, Query::new); - ParamUtils.defaultObject(options, QueryOptions::new); - - String userId = userManager.getUserId(token); - // We need to add variableSets and groups to avoid additional queries as it will be used in the catalogSolrManager - Study study = catalogManager.getStudyManager().resolveId(studyId, userId, new QueryOptions(QueryOptions.INCLUDE, - Arrays.asList(StudyDBAdaptor.QueryParams.VARIABLE_SET.key(), StudyDBAdaptor.QueryParams.GROUPS.key()))); - - ObjectMap auditParams = new ObjectMap() - .append("studyId", studyId) - .append("query", new Query(query)) - .append("options", options) - .append("defaultStats", defaultStats) - .append("token", token); - - try { - if (defaultStats || StringUtils.isEmpty(options.getString(QueryOptions.FACET))) { - String facet = options.getString(QueryOptions.FACET); - options.put(QueryOptions.FACET, StringUtils.isNotEmpty(facet) ? defaultFacet + ";" + facet : defaultFacet); - } - - AnnotationUtils.fixQueryAnnotationSearch(study, userId, query, authorizationManager); - - try (CatalogSolrManager catalogSolrManager = new CatalogSolrManager(catalogManager)) { - - DataResult result = catalogSolrManager.facetedQuery(study, CatalogSolrManager.COHORT_SOLR_COLLECTION, query, - options, userId); - auditManager.auditFacet(userId, Enums.Resource.COHORT, study.getId(), study.getUuid(), auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); - - return result; - } - } catch (CatalogException e) { - auditManager.auditFacet(userId, Enums.Resource.COHORT, study.getId(), study.getUuid(), auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(0, "", e.getMessage()))); - throw e; - } - } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FamilyManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FamilyManager.java index c135e390744..0cfa9018ef5 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FamilyManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FamilyManager.java @@ -30,7 +30,10 @@ import org.opencb.biodata.models.core.SexOntologyTermAnnotation; import org.opencb.biodata.models.pedigree.IndividualProperty; import org.opencb.biodata.tools.pedigree.ModeOfInheritance; -import org.opencb.commons.datastore.core.*; +import org.opencb.commons.datastore.core.Event; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.result.Error; import org.opencb.commons.utils.ListUtils; import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager; @@ -39,14 +42,11 @@ import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.models.InternalGetDataResult; -import org.opencb.opencga.catalog.stats.solr.CatalogSolrManager; -import org.opencb.opencga.catalog.utils.AnnotationUtils; -import org.opencb.opencga.catalog.utils.Constants; -import org.opencb.opencga.catalog.utils.ParamUtils; -import org.opencb.opencga.catalog.utils.UuidUtils; +import org.opencb.opencga.catalog.utils.*; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.models.AclEntryList; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.audit.AuditRecord; import org.opencb.opencga.core.models.clinical.ClinicalAnalysis; import org.opencb.opencga.core.models.common.AnnotationSet; @@ -132,8 +132,8 @@ Enums.Resource getEntity() { } @Override - InternalGetDataResult internalGet(long studyUid, List entryList, @Nullable Query query, QueryOptions options, - String user, boolean ignoreException) throws CatalogException { + InternalGetDataResult internalGet(String organizationId, long studyUid, List entryList, @Nullable Query query, + QueryOptions options, String user, boolean ignoreException) throws CatalogException { if (ListUtils.isEmpty(entryList)) { throw new CatalogException("Missing family entries."); } @@ -155,7 +155,7 @@ InternalGetDataResult internalGet(long studyUid, List entryList, // Ensure the field by which we are querying for will be kept in the results queryOptions = keepFieldInQueryOptions(queryOptions, idQueryParam.key()); - OpenCGAResult familyDataResult = familyDBAdaptor.get(studyUid, queryCopy, queryOptions, user); + OpenCGAResult familyDataResult = getFamilyDBAdaptor(organizationId).get(studyUid, queryCopy, queryOptions, user); Function familyStringFunction = Family::getId; if (idQueryParam.equals(FamilyDBAdaptor.QueryParams.UUID)) { @@ -166,7 +166,7 @@ InternalGetDataResult internalGet(long studyUid, List entryList, return keepOriginalOrder(uniqueList, familyStringFunction, familyDataResult, ignoreException, versioned); } // Query without adding the user check - OpenCGAResult resultsNoCheck = familyDBAdaptor.get(queryCopy, queryOptions); + OpenCGAResult resultsNoCheck = getFamilyDBAdaptor(organizationId).get(queryCopy, queryOptions); if (resultsNoCheck.getNumResults() == familyDataResult.getNumResults()) { throw CatalogException.notFound("families", @@ -193,30 +193,34 @@ FamilyDBAdaptor.QueryParams getFieldFilter(List idList) throws CatalogEx return idQueryParam; } - private OpenCGAResult getFamily(long studyUid, String familyUuid, QueryOptions options) throws CatalogException { + private OpenCGAResult getFamily(String organizationId, long studyUid, String familyUuid, QueryOptions options) + throws CatalogException { Query query = new Query() .append(FamilyDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(FamilyDBAdaptor.QueryParams.UUID.key(), familyUuid); - return familyDBAdaptor.get(query, options); + return getFamilyDBAdaptor(organizationId).get(query, options); } @Override public DBIterator iterator(String studyStr, Query query, QueryOptions options, String token) throws CatalogException { - ParamUtils.checkObj(token, "sessionId"); + ParamUtils.checkObj(token, "token"); query = ParamUtils.defaultObject(query, Query::new); options = ParamUtils.defaultObject(options, QueryOptions::new); - String userId = userManager.getUserId(token); - Study study = catalogManager.getStudyManager().resolveId(studyStr, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyStr, userId, organizationId); Query finalQuery = new Query(query); - fixQueryObject(study, finalQuery, token); + fixQueryObject(organizationId, study, finalQuery, tokenPayload); // Fix query if it contains any annotation - AnnotationUtils.fixQueryAnnotationSearch(study, finalQuery); + AnnotationUtils.fixQueryAnnotationSearch(organizationId, study, finalQuery); AnnotationUtils.fixQueryOptionAnnotation(options); finalQuery.append(FamilyDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - return familyDBAdaptor.iterator(study.getUid(), finalQuery, options, userId); + return getFamilyDBAdaptor(organizationId).iterator(study.getUid(), finalQuery, options, userId); } @Override @@ -227,8 +231,11 @@ public OpenCGAResult create(String studyStr, Family family, QueryOptions public OpenCGAResult create(String studyStr, Family family, List members, QueryOptions options, String token) throws CatalogException { options = ParamUtils.defaultObject(options, QueryOptions::new); - String userId = catalogManager.getUserManager().getUserId(token); - Study study = catalogManager.getStudyManager().resolveId(studyStr, userId, StudyManager.INCLUDE_VARIABLE_SET); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyStr, StudyManager.INCLUDE_VARIABLE_SET, userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("study", studyStr) @@ -237,7 +244,7 @@ public OpenCGAResult create(String studyStr, Family family, List .append("options", options) .append("token", token); try { - authorizationManager.checkStudyPermission(study.getUid(), userId, StudyPermissions.Permissions.WRITE_FAMILIES); + authorizationManager.checkStudyPermission(organizationId, study.getUid(), userId, StudyPermissions.Permissions.WRITE_FAMILIES); ParamUtils.checkObj(family, "family"); ParamUtils.checkIdentifier(family.getId(), "id"); @@ -262,13 +269,13 @@ public OpenCGAResult create(String studyStr, Family family, List Query query = new Query() .append(FamilyDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()) .append(FamilyDBAdaptor.QueryParams.ID.key(), family.getId()); - if (familyDBAdaptor.count(query).getNumMatches() > 0) { + if (getFamilyDBAdaptor(organizationId).count(query).getNumMatches() > 0) { throw new CatalogException("Family '" + family.getId() + "' already exists."); } validateNewAnnotationSets(study.getVariableSets(), family.getAnnotationSets()); - List existingMembers = autoCompleteFamilyMembers(study, family, members, userId); + List existingMembers = autoCompleteFamilyMembers(organizationId, study, family, members, userId); validateFamily(family, existingMembers); validatePhenotypes(family, existingMembers); validateDisorders(family, existingMembers); @@ -276,21 +283,21 @@ public OpenCGAResult create(String studyStr, Family family, List options = ParamUtils.defaultObject(options, QueryOptions::new); family.setUuid(UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.FAMILY)); - OpenCGAResult insert = familyDBAdaptor.insert(study.getUid(), family, existingMembers, study.getVariableSets(), - options); + OpenCGAResult insert = getFamilyDBAdaptor(organizationId).insert(study.getUid(), family, existingMembers, + study.getVariableSets(), options); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { // Fetch updated family - OpenCGAResult queryResult = getFamily(study.getUid(), family.getUuid(), options); + OpenCGAResult queryResult = getFamily(organizationId, study.getUid(), family.getUuid(), options); insert.setResults(queryResult.getResults()); } - auditManager.auditCreate(userId, Enums.Resource.FAMILY, family.getId(), family.getUuid(), study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.auditCreate(organizationId, userId, Enums.Resource.FAMILY, family.getId(), family.getUuid(), study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return insert; } catch (CatalogException e) { - auditManager.auditCreate(userId, Enums.Resource.FAMILY, family.getId(), "", study.getId(), study.getUuid(), auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditCreate(organizationId, userId, Enums.Resource.FAMILY, family.getId(), "", study.getId(), study.getUuid(), + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } @@ -300,9 +307,12 @@ public OpenCGAResult search(String studyId, Query query, QueryOptions op Query finalQuery = new Query(query); options = ParamUtils.defaultObject(options, QueryOptions::new); - String userId = catalogManager.getUserManager().getUserId(token); - Study study = studyManager.resolveId(studyId, userId, new QueryOptions(QueryOptions.INCLUDE, - StudyDBAdaptor.QueryParams.VARIABLE_SET.key())); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyId, new QueryOptions(QueryOptions.INCLUDE, StudyDBAdaptor.QueryParams.VARIABLE_SET.key()), + userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) @@ -310,22 +320,22 @@ public OpenCGAResult search(String studyId, Query query, QueryOptions op .append("options", options) .append("token", token); try { - fixQueryObject(study, finalQuery, token); + fixQueryObject(organizationId, study, finalQuery, tokenPayload); // Fix query if it contains any annotation - AnnotationUtils.fixQueryAnnotationSearch(study, finalQuery); + AnnotationUtils.fixQueryAnnotationSearch(organizationId, study, finalQuery); AnnotationUtils.fixQueryOptionAnnotation(options); finalQuery.append(FamilyDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - OpenCGAResult queryResult = familyDBAdaptor.get(study.getUid(), finalQuery, options, userId); + OpenCGAResult queryResult = getFamilyDBAdaptor(organizationId).get(study.getUid(), finalQuery, options, userId); - auditManager.auditSearch(userId, Enums.Resource.FAMILY, study.getId(), study.getUuid(), auditParams, + auditManager.auditSearch(organizationId, userId, Enums.Resource.FAMILY, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return queryResult; } catch (CatalogException e) { - auditManager.auditSearch(userId, Enums.Resource.FAMILY, study.getId(), study.getUuid(), auditParams, + auditManager.auditSearch(organizationId, userId, Enums.Resource.FAMILY, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -335,9 +345,12 @@ public OpenCGAResult search(String studyId, Query query, QueryOptions op public OpenCGAResult distinct(String studyId, List fields, Query query, String token) throws CatalogException { query = ParamUtils.defaultObject(query, Query::new); - String userId = userManager.getUserId(token); - Study study = catalogManager.getStudyManager().resolveId(studyId, userId, new QueryOptions(QueryOptions.INCLUDE, - StudyDBAdaptor.QueryParams.VARIABLE_SET.key())); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyId, new QueryOptions(QueryOptions.INCLUDE, + StudyDBAdaptor.QueryParams.VARIABLE_SET.key()), userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) @@ -345,25 +358,25 @@ public OpenCGAResult distinct(String studyId, List fields, Query quer .append("query", new Query(query)) .append("token", token); try { - fixQueryObject(study, query, userId); + fixQueryObject(organizationId, study, query, tokenPayload); // Fix query if it contains any annotation - AnnotationUtils.fixQueryAnnotationSearch(study, query); + AnnotationUtils.fixQueryAnnotationSearch(organizationId, study, query); query.append(FamilyDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - OpenCGAResult result = familyDBAdaptor.distinct(study.getUid(), fields, query, userId); + OpenCGAResult result = getFamilyDBAdaptor(organizationId).distinct(study.getUid(), fields, query, userId); - auditManager.auditDistinct(userId, Enums.Resource.FAMILY, study.getId(), study.getUuid(), auditParams, + auditManager.auditDistinct(organizationId, userId, Enums.Resource.FAMILY, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return result; } catch (CatalogException e) { - auditManager.auditDistinct(userId, Enums.Resource.FAMILY, study.getId(), study.getUuid(), auditParams, + auditManager.auditDistinct(organizationId, userId, Enums.Resource.FAMILY, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } - private void fixQueryObject(Study study, Query query, String sessionId) throws CatalogException { + private void fixQueryObject(String organizationId, Study study, Query query, JwtPayload payload) throws CatalogException { super.fixQueryObject(query); changeQueryId(query, ParamConstants.FAMILY_INTERNAL_STATUS_PARAM, FamilyDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key()); changeQueryId(query, ParamConstants.FAMILY_STATUS_PARAM, FamilyDBAdaptor.QueryParams.STATUS_ID.key()); @@ -376,9 +389,9 @@ private void fixQueryObject(Study study, Query query, String sessionId) throws C // The individuals introduced could be either ids or names. As so, we should use the smart resolutor to do this. // We change the MEMBERS parameters for MEMBER_UID which is what the DBAdaptor understands if (query.containsKey(FamilyDBAdaptor.QueryParams.MEMBERS.key())) { - String userId = userManager.getUserId(sessionId); + String userId = payload.getUserId(); - List memberList = catalogManager.getIndividualManager().internalGet(study.getUid(), + List memberList = catalogManager.getIndividualManager().internalGet(organizationId, study.getUid(), query.getAsStringList(FamilyDBAdaptor.QueryParams.MEMBERS.key()), IndividualManager.INCLUDE_INDIVIDUAL_IDS, userId, true).getResults(); if (ListUtils.isNotEmpty(memberList)) { @@ -400,7 +413,7 @@ private void fixQueryObject(Study study, Query query, String sessionId) throws C query.getString(IndividualDBAdaptor.QueryParams.SAMPLES.key())); QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, IndividualDBAdaptor.QueryParams.UID.key()); OpenCGAResult individualResult = catalogManager.getIndividualManager() - .search(study.getFqn(), newQuery, options, sessionId); + .search(study.getFqn(), newQuery, options, payload.getToken()); if (individualResult.getNumResults() == 0) { // Add -1 to query so no results are obtained @@ -420,9 +433,12 @@ public OpenCGAResult count(String studyId, Query query, String token) th query = ParamUtils.defaultObject(query, Query::new); Query finalQuery = new Query(query); - String userId = catalogManager.getUserManager().getUserId(token); - Study study = studyManager.resolveId(studyId, userId, new QueryOptions(QueryOptions.INCLUDE, - StudyDBAdaptor.QueryParams.VARIABLE_SET.key())); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyId, new QueryOptions(QueryOptions.INCLUDE, + StudyDBAdaptor.QueryParams.VARIABLE_SET.key()), userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) @@ -430,34 +446,38 @@ public OpenCGAResult count(String studyId, Query query, String token) th .append("token", token); try { // Fix query if it contains any annotation - AnnotationUtils.fixQueryAnnotationSearch(study, finalQuery); - fixQueryObject(study, finalQuery, token); + AnnotationUtils.fixQueryAnnotationSearch(organizationId, study, finalQuery); + fixQueryObject(organizationId, study, finalQuery, tokenPayload); finalQuery.append(FamilyDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - OpenCGAResult queryResultAux = familyDBAdaptor.count(finalQuery, userId); + OpenCGAResult queryResultAux = getFamilyDBAdaptor(organizationId).count(finalQuery, userId); - auditManager.auditCount(userId, Enums.Resource.FAMILY, study.getId(), study.getUuid(), auditParams, + auditManager.auditCount(organizationId, userId, Enums.Resource.FAMILY, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return new OpenCGAResult<>(queryResultAux.getTime(), queryResultAux.getEvents(), 0, Collections.emptyList(), queryResultAux.getNumMatches()); } catch (CatalogException e) { - auditManager.auditCount(userId, Enums.Resource.FAMILY, study.getId(), study.getUuid(), auditParams, + auditManager.auditCount(organizationId, userId, Enums.Resource.FAMILY, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } @Override - public OpenCGAResult delete(String studyStr, List familyIds, QueryOptions options, String token) throws CatalogException { + public OpenCGAResult delete(String studyStr, List familyIds, QueryOptions options, String token) + throws CatalogException { return delete(studyStr, familyIds, options, false, token); } public OpenCGAResult delete(String studyStr, List familyIds, ObjectMap params, boolean ignoreException, String token) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(token); - Study study = studyManager.resolveId(studyStr, userId, new QueryOptions(QueryOptions.INCLUDE, - StudyDBAdaptor.QueryParams.VARIABLE_SET.key())); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, new QueryOptions(QueryOptions.INCLUDE, + StudyDBAdaptor.QueryParams.VARIABLE_SET.key()), userId, organizationId); String operationUuid = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -471,9 +491,10 @@ public OpenCGAResult delete(String studyStr, List familyIds, ObjectMap p boolean checkPermissions; try { // If the user is the owner or the admin, we won't check if he has permissions for every single entry - checkPermissions = !authorizationManager.isOwnerOrAdmin(study.getUid(), userId); + long studyId = study.getUid(); + checkPermissions = !authorizationManager.isAtLeastStudyAdministrator(organizationId, studyId, userId); } catch (CatalogException e) { - auditManager.auditDelete(operationUuid, userId, Enums.Resource.FAMILY, "", "", study.getId(), study.getUuid(), + auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.FAMILY, "", "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -485,7 +506,7 @@ public OpenCGAResult delete(String studyStr, List familyIds, ObjectMap p String familyUuid = ""; try { - OpenCGAResult internalResult = internalGet(study.getUid(), id, INCLUDE_FAMILY_IDS, userId); + OpenCGAResult internalResult = internalGet(organizationId, study.getUid(), id, INCLUDE_FAMILY_IDS, userId); if (internalResult.getNumResults() == 0) { throw new CatalogException("Family '" + id + "' not found"); } @@ -496,17 +517,17 @@ public OpenCGAResult delete(String studyStr, List familyIds, ObjectMap p familyUuid = family.getUuid(); if (checkPermissions) { - authorizationManager.checkFamilyPermission(study.getUid(), family.getUid(), userId, + authorizationManager.checkFamilyPermission(organizationId, study.getUid(), family.getUid(), userId, FamilyPermissions.DELETE); } // Check family can be deleted - checkCanBeDeleted(study, family); + checkCanBeDeleted(organizationId, study, family); // Delete the family - result.append(familyDBAdaptor.delete(family)); + result.append(getFamilyDBAdaptor(organizationId).delete(family)); - auditManager.auditDelete(operationUuid, userId, Enums.Resource.FAMILY, family.getId(), family.getUuid(), + auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.FAMILY, family.getId(), family.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { Event event = new Event(Event.Type.ERROR, familyId, e.getMessage()); @@ -514,20 +535,21 @@ public OpenCGAResult delete(String studyStr, List familyIds, ObjectMap p result.setNumErrors(result.getNumErrors() + 1); logger.error("Cannot delete family {}: {}", familyId, e.getMessage(), e); - auditManager.auditDelete(operationUuid, userId, Enums.Resource.FAMILY, familyId, familyUuid, + auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.FAMILY, familyId, familyUuid, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); } } - auditManager.finishAuditBatch(operationUuid); + auditManager.finishAuditBatch(organizationId, operationUuid); return endResult(result, ignoreException); } - private void checkCanBeDeleted(Study study, Family family) throws CatalogException { + private void checkCanBeDeleted(String organizationId, Study study, Family family) throws CatalogException { Query query = new Query() .append(ClinicalAnalysisDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()) .append(ClinicalAnalysisDBAdaptor.QueryParams.FAMILY_UID.key(), family.getUid()); - OpenCGAResult result = clinicalDBAdaptor.get(query, ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS); + OpenCGAResult result = getClinicalAnalysisDBAdaptor(organizationId).get(query, + ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS); if (result.getNumResults() > 0) { String clinicalIds = result.getResults().stream().map(ClinicalAnalysis::getId).collect(Collectors.joining(", ")); throw new CatalogException("Could not delete family '" + family.getId() + "'. Family is in use in Clinical Analyses: '" @@ -545,9 +567,12 @@ public OpenCGAResult delete(String studyStr, Query query, ObjectMap params, bool Query finalQuery = new Query(ParamUtils.defaultObject(query, Query::new)); OpenCGAResult result = OpenCGAResult.empty(); - String userId = catalogManager.getUserManager().getUserId(token); - Study study = studyManager.resolveId(studyStr, userId, new QueryOptions(QueryOptions.INCLUDE, - StudyDBAdaptor.QueryParams.VARIABLE_SET.key())); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, new QueryOptions(QueryOptions.INCLUDE, + StudyDBAdaptor.QueryParams.VARIABLE_SET.key()), userId, organizationId); String operationUuid = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -565,16 +590,17 @@ public OpenCGAResult delete(String studyStr, Query query, ObjectMap params, bool DBIterator iterator; try { // Fix query if it contains any annotation - AnnotationUtils.fixQueryAnnotationSearch(study, finalQuery); - fixQueryObject(study, finalQuery, token); + AnnotationUtils.fixQueryAnnotationSearch(organizationId, study, finalQuery); + fixQueryObject(organizationId, study, finalQuery, tokenPayload); finalQuery.append(FamilyDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - iterator = familyDBAdaptor.iterator(study.getUid(), finalQuery, INCLUDE_FAMILY_IDS, userId); + iterator = getFamilyDBAdaptor(organizationId).iterator(study.getUid(), finalQuery, INCLUDE_FAMILY_IDS, userId); // If the user is the owner or the admin, we won't check if he has permissions for every single entry - checkPermissions = !authorizationManager.isOwnerOrAdmin(study.getUid(), userId); + long studyId = study.getUid(); + checkPermissions = !authorizationManager.isAtLeastStudyAdministrator(organizationId, studyId, userId); } catch (CatalogException e) { - auditManager.auditDelete(operationUuid, userId, Enums.Resource.FAMILY, "", "", study.getId(), study.getUuid(), + auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.FAMILY, "", "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -585,16 +611,16 @@ public OpenCGAResult delete(String studyStr, Query query, ObjectMap params, bool try { if (checkPermissions) { - authorizationManager.checkFamilyPermission(study.getUid(), family.getUid(), userId, + authorizationManager.checkFamilyPermission(organizationId, study.getUid(), family.getUid(), userId, FamilyPermissions.DELETE); } // TODO: Check if the family is used in a clinical analysis. At this point, it can be deleted no matter what. // Delete the family - result.append(familyDBAdaptor.delete(family)); + result.append(getFamilyDBAdaptor(organizationId).delete(family)); - auditManager.auditDelete(operationUuid, userId, Enums.Resource.FAMILY, family.getId(), family.getUuid(), + auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.FAMILY, family.getId(), family.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { String errorMsg = "Cannot delete family " + family.getId() + ": " + e.getMessage(); @@ -604,18 +630,18 @@ public OpenCGAResult delete(String studyStr, Query query, ObjectMap params, bool result.setNumErrors(result.getNumErrors() + 1); logger.error(errorMsg, e); - auditManager.auditDelete(operationUuid, userId, Enums.Resource.FAMILY, family.getId(), family.getUuid(), + auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.FAMILY, family.getId(), family.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); } } - auditManager.finishAuditBatch(operationUuid); + auditManager.finishAuditBatch(organizationId, operationUuid); return endResult(result, ignoreException); } @Override - public OpenCGAResult rank(String studyStr, Query query, String field, int numResults, boolean asc, String sessionId) throws - CatalogException { + public OpenCGAResult rank(String studyStr, Query query, String field, int numResults, boolean asc, String sessionId) + throws CatalogException { return null; } @@ -628,20 +654,23 @@ public OpenCGAResult groupBy(@Nullable String studyStr, Query query, List addAnnotationSet(String studyStr, String familyStr, public OpenCGAResult addAnnotationSets(String studyStr, String familyStr, List annotationSetList, QueryOptions options, String token) throws CatalogException { - return updateAnnotationSet(studyStr, familyStr, annotationSetList, ParamUtils.BasicUpdateAction.ADD, options, token); + return updateAnnotationSet(studyStr, familyStr, annotationSetList, ParamUtils.BasicUpdateAction.ADD, options, + token); } - public OpenCGAResult setAnnotationSet(String studyStr, String familyStr, AnnotationSet annotationSet, QueryOptions options, - String token) throws CatalogException { + public OpenCGAResult setAnnotationSet(String studyStr, String familyStr, AnnotationSet annotationSet, + QueryOptions options, String token) throws CatalogException { return setAnnotationSets(studyStr, familyStr, Collections.singletonList(annotationSet), options, token); } public OpenCGAResult setAnnotationSets(String studyStr, String familyStr, List annotationSetList, QueryOptions options, String token) throws CatalogException { - return updateAnnotationSet(studyStr, familyStr, annotationSetList, ParamUtils.BasicUpdateAction.SET, options, token); + return updateAnnotationSet(studyStr, familyStr, annotationSetList, ParamUtils.BasicUpdateAction.SET, options, + token); } - public OpenCGAResult removeAnnotationSet(String studyStr, String familyStr, String annotationSetId, QueryOptions options, - String token) throws CatalogException { + public OpenCGAResult removeAnnotationSet(String studyStr, String familyStr, String annotationSetId, + QueryOptions options, String token) throws CatalogException { return removeAnnotationSets(studyStr, familyStr, Collections.singletonList(annotationSetId), options, token); } - public OpenCGAResult removeAnnotationSets(String studyStr, String familyStr, List annotationSetIdList, - QueryOptions options, String token) throws CatalogException { + public OpenCGAResult removeAnnotationSets(String studyStr, String familyStr, + List annotationSetIdList, QueryOptions options, String token) + throws CatalogException { List annotationSetList = annotationSetIdList .stream() .map(id -> new AnnotationSet().setId(id)) .collect(Collectors.toList()); - return updateAnnotationSet(studyStr, familyStr, annotationSetList, ParamUtils.BasicUpdateAction.REMOVE, options, token); + return updateAnnotationSet(studyStr, familyStr, annotationSetList, ParamUtils.BasicUpdateAction.REMOVE, options, + token); } public OpenCGAResult updateAnnotations(String studyStr, String familyStr, String annotationSetId, @@ -704,27 +737,30 @@ public OpenCGAResult updateAnnotations(String studyStr, String familyStr return update(studyStr, familyStr, updateParams, options, token); } - public OpenCGAResult removeAnnotations(String studyStr, String familyStr, String annotationSetId, - List annotations, QueryOptions options, String token) throws CatalogException { - return updateAnnotations(studyStr, familyStr, annotationSetId, new ObjectMap("remove", StringUtils.join(annotations, ",")), - ParamUtils.CompleteUpdateAction.REMOVE, options, token); + public OpenCGAResult removeAnnotations(String studyStr, String familyStr, String annotationSetId, List annotations, + QueryOptions options, String token) throws CatalogException { + return updateAnnotations(studyStr, familyStr, annotationSetId, + new ObjectMap("remove", StringUtils.join(annotations, ",")), ParamUtils.CompleteUpdateAction.REMOVE, options, token); } public OpenCGAResult resetAnnotations(String studyStr, String familyStr, String annotationSetId, List annotations, QueryOptions options, String token) throws CatalogException { - return updateAnnotations(studyStr, familyStr, annotationSetId, new ObjectMap("reset", StringUtils.join(annotations, ",")), - ParamUtils.CompleteUpdateAction.RESET, options, token); + return updateAnnotations(studyStr, familyStr, annotationSetId, + new ObjectMap("reset", StringUtils.join(annotations, ",")), ParamUtils.CompleteUpdateAction.RESET, options, token); } - public OpenCGAResult update(String studyStr, Query query, FamilyUpdateParams updateParams, QueryOptions options, - String token) throws CatalogException { + public OpenCGAResult update(String studyStr, Query query, FamilyUpdateParams updateParams, QueryOptions options, String token) + throws CatalogException { return update(studyStr, query, updateParams, false, options, token); } public OpenCGAResult update(String studyStr, Query query, FamilyUpdateParams updateParams, boolean ignoreException, QueryOptions options, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId, StudyManager.INCLUDE_VARIABLE_SET); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, StudyManager.INCLUDE_VARIABLE_SET, userId, organizationId); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -747,16 +783,16 @@ public OpenCGAResult update(String studyStr, Query query, FamilyUpdatePa DBIterator iterator; try { - fixQueryObject(study, finalQuery, token); + fixQueryObject(organizationId, study, finalQuery, tokenPayload); // Fix query if it contains any annotation - AnnotationUtils.fixQueryAnnotationSearch(study, finalQuery); + AnnotationUtils.fixQueryAnnotationSearch(organizationId, study, finalQuery); finalQuery.append(FamilyDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - iterator = familyDBAdaptor.iterator(study.getUid(), finalQuery, QueryOptions.empty(), userId); + iterator = getFamilyDBAdaptor(organizationId).iterator(study.getUid(), finalQuery, QueryOptions.empty(), userId); } catch (CatalogException e) { - auditManager.auditUpdate(operationId, userId, Enums.Resource.FAMILY, "", "", study.getId(), study.getUuid(), + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.FAMILY, "", "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -766,30 +802,33 @@ public OpenCGAResult update(String studyStr, Query query, FamilyUpdatePa while (iterator.hasNext()) { Family family = iterator.next(); try { - OpenCGAResult queryResult = update(study, family, updateParams, options, userId); + OpenCGAResult queryResult = update(organizationId, study, family, updateParams, options, userId); result.append(queryResult); - auditManager.auditUpdate(operationId, userId, Enums.Resource.FAMILY, family.getId(), family.getUuid(), study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.FAMILY, family.getId(), family.getUuid(), + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { Event event = new Event(Event.Type.ERROR, family.getId(), e.getMessage()); result.getEvents().add(event); result.setNumErrors(result.getNumErrors() + 1); logger.error("Cannot update family {}: {}", family.getId(), e.getMessage(), e); - auditManager.auditUpdate(operationId, userId, Enums.Resource.FAMILY, family.getId(), family.getUuid(), study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.FAMILY, family.getId(), family.getUuid(), + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); } } - auditManager.finishAuditBatch(operationId); + auditManager.finishAuditBatch(organizationId, operationId); return endResult(result, ignoreException); } public OpenCGAResult update(String studyStr, String familyId, FamilyUpdateParams updateParams, QueryOptions options, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId, StudyManager.INCLUDE_VARIABLE_SET); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, StudyManager.INCLUDE_VARIABLE_SET, userId, organizationId); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -811,7 +850,7 @@ public OpenCGAResult update(String studyStr, String familyId, FamilyUpda String familyUuid = ""; try { - OpenCGAResult internalResult = internalGet(study.getUid(), familyId, QueryOptions.empty(), userId); + OpenCGAResult internalResult = internalGet(organizationId, study.getUid(), familyId, QueryOptions.empty(), userId); if (internalResult.getNumResults() == 0) { throw new CatalogException("Family '" + familyId + "' not found"); } @@ -821,18 +860,18 @@ public OpenCGAResult update(String studyStr, String familyId, FamilyUpda familyId = family.getId(); familyUuid = family.getUuid(); - OpenCGAResult updateResult = update(study, family, updateParams, options, userId); + OpenCGAResult updateResult = update(organizationId, study, family, updateParams, options, userId); result.append(updateResult); - auditManager.auditUpdate(operationId, userId, Enums.Resource.FAMILY, family.getId(), family.getUuid(), study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.FAMILY, family.getId(), family.getUuid(), + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { Event event = new Event(Event.Type.ERROR, familyId, e.getMessage()); result.getEvents().add(event); result.setNumErrors(result.getNumErrors() + 1); logger.error("Cannot update family {}: {}", familyId, e.getMessage()); - auditManager.auditUpdate(operationId, userId, Enums.Resource.FAMILY, familyId, familyUuid, study.getId(), + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.FAMILY, familyId, familyUuid, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -843,7 +882,8 @@ public OpenCGAResult update(String studyStr, String familyId, FamilyUpda /** * Update families from catalog. * - * @param studyStr Study id in string format. Could be one of [id|user@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy]. + * @param studyStr Study id in string format. Could be one of + * [id|organization@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy] * @param familyIds List of family ids. Could be either the id or uuid. * @param updateParams Data model filled only with the parameters to be updated. * @param options QueryOptions object. @@ -859,8 +899,11 @@ public OpenCGAResult update(String studyStr, List familyIds, Fam public OpenCGAResult update(String studyStr, List familyIds, FamilyUpdateParams updateParams, boolean ignoreException, QueryOptions options, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId, StudyManager.INCLUDE_VARIABLE_SET); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, StudyManager.INCLUDE_VARIABLE_SET, userId, organizationId); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -886,7 +929,7 @@ public OpenCGAResult update(String studyStr, List familyIds, Fam String familyUuid = ""; try { - OpenCGAResult internalResult = internalGet(study.getUid(), familyId, QueryOptions.empty(), userId); + OpenCGAResult internalResult = internalGet(organizationId, study.getUid(), familyId, QueryOptions.empty(), userId); if (internalResult.getNumResults() == 0) { throw new CatalogException("Family '" + id + "' not found"); } @@ -896,28 +939,28 @@ public OpenCGAResult update(String studyStr, List familyIds, Fam familyId = family.getId(); familyUuid = family.getUuid(); - OpenCGAResult updateResult = update(study, family, updateParams, options, userId); + OpenCGAResult updateResult = update(organizationId, study, family, updateParams, options, userId); result.append(updateResult); - auditManager.auditUpdate(operationId, userId, Enums.Resource.FAMILY, family.getId(), family.getUuid(), study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.FAMILY, family.getId(), family.getUuid(), + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { Event event = new Event(Event.Type.ERROR, id, e.getMessage()); result.getEvents().add(event); result.setNumErrors(result.getNumErrors() + 1); logger.error("Cannot update family {}: {}", familyId, e.getMessage()); - auditManager.auditUpdate(operationId, userId, Enums.Resource.FAMILY, familyId, familyUuid, study.getId(), + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.FAMILY, familyId, familyUuid, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); } } - auditManager.finishAuditBatch(operationId); + auditManager.finishAuditBatch(organizationId, operationId); return endResult(result, ignoreException); } - private OpenCGAResult update(Study study, Family family, FamilyUpdateParams updateParams, QueryOptions options, String userId) - throws CatalogException { + private OpenCGAResult update(String organizationId, Study study, Family family, FamilyUpdateParams updateParams, + QueryOptions options, String userId) throws CatalogException { options = ParamUtils.defaultObject(options, QueryOptions::new); ObjectMap parameters = new ObjectMap(); @@ -957,13 +1000,13 @@ private OpenCGAResult update(Study study, Family family, FamilyUpdatePar // Check permissions... // Only check write annotation permissions if the user wants to update the annotation sets if (updateParams != null && updateParams.getAnnotationSets() != null) { - authorizationManager.checkFamilyPermission(study.getUid(), family.getUid(), userId, + authorizationManager.checkFamilyPermission(organizationId, study.getUid(), family.getUid(), userId, FamilyPermissions.WRITE_ANNOTATIONS); } // Only check update permissions if the user wants to update anything apart from the annotation sets if ((parameters.size() == 1 && !parameters.containsKey(FamilyDBAdaptor.QueryParams.ANNOTATION_SETS.key())) || parameters.size() > 1) { - authorizationManager.checkFamilyPermission(study.getUid(), family.getUid(), userId, + authorizationManager.checkFamilyPermission(organizationId, study.getUid(), family.getUid(), userId, FamilyPermissions.WRITE); } @@ -986,7 +1029,7 @@ private OpenCGAResult update(Study study, Family family, FamilyUpdatePar throw new CatalogException("Found members without any id."); } } - List updatedMembers = autoCompleteFamilyMembers(study, tmpFamily, memberIds, userId); + List updatedMembers = autoCompleteFamilyMembers(organizationId, study, tmpFamily, memberIds, userId); tmpFamily.setMembers(updatedMembers); } else { // We use the list of members from the stored family @@ -1018,13 +1061,14 @@ private OpenCGAResult update(Study study, Family family, FamilyUpdatePar } } - checkUpdateAnnotations(study, family, parameters, options, VariableSet.AnnotableDataModels.FAMILY, familyDBAdaptor, - userId); + checkUpdateAnnotations(organizationId, study, family, parameters, options, VariableSet.AnnotableDataModels.FAMILY, + getFamilyDBAdaptor(organizationId), userId); - OpenCGAResult update = familyDBAdaptor.update(family.getUid(), parameters, study.getVariableSets(), options); + OpenCGAResult update = getFamilyDBAdaptor(organizationId).update(family.getUid(), parameters, study.getVariableSets(), + options); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { // Fetch updated family - OpenCGAResult result = familyDBAdaptor.get(study.getUid(), + OpenCGAResult result = getFamilyDBAdaptor(organizationId).get(study.getUid(), new Query(FamilyDBAdaptor.QueryParams.UID.key(), family.getUid()), options, userId); update.setResults(result.getResults()); } @@ -1038,8 +1082,8 @@ public Map> calculateFamilyGenotypes(String studyStr, Strin Disorder disorder = null; if (StringUtils.isNotEmpty(clinicalAnalysisId)) { - OpenCGAResult clinicalAnalysisDataResult = catalogManager.getClinicalAnalysisManager().get(studyStr, - clinicalAnalysisId, new QueryOptions(QueryOptions.INCLUDE, Arrays.asList( + OpenCGAResult clinicalAnalysisDataResult = catalogManager.getClinicalAnalysisManager() + .get(studyStr, clinicalAnalysisId, new QueryOptions(QueryOptions.INCLUDE, Arrays.asList( ClinicalAnalysisDBAdaptor.QueryParams.PROBAND.key(), ClinicalAnalysisDBAdaptor.QueryParams.FAMILY.key(), ClinicalAnalysisDBAdaptor.QueryParams.DISORDER.key())), token); if (clinicalAnalysisDataResult.getNumResults() == 0) { @@ -1097,14 +1141,18 @@ clinicalAnalysisId, new QueryOptions(QueryOptions.INCLUDE, Arrays.asList( // ************************** ACLs ******************************** // public OpenCGAResult> getAcls(String studyId, List familyList, String member, boolean ignoreException, String token) throws CatalogException { - return getAcls(studyId, familyList, StringUtils.isNotEmpty(member) ? Collections.singletonList(member) : Collections.emptyList(), + return getAcls(studyId, familyList, + StringUtils.isNotEmpty(member) ? Collections.singletonList(member) : Collections.emptyList(), ignoreException, token); } public OpenCGAResult> getAcls(String studyId, List familyList, List members, boolean ignoreException, String token) throws CatalogException { - String user = userManager.getUserId(token); - Study study = studyManager.resolveId(studyId, user); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyId, userId, organizationId); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); ObjectMap auditParams = new ObjectMap() @@ -1118,7 +1166,8 @@ public OpenCGAResult> getAcls(String studyId, Li Map missingMap = new HashMap<>(); try { auditManager.initAuditBatch(operationId); - InternalGetDataResult queryResult = internalGet(study.getUid(), familyList, INCLUDE_FAMILY_IDS, user, ignoreException); + InternalGetDataResult queryResult = internalGet(organizationId, study.getUid(), familyList, INCLUDE_FAMILY_IDS, userId, + ignoreException); if (queryResult.getMissing() != null) { missingMap = queryResult.getMissing().stream() @@ -1127,11 +1176,11 @@ public OpenCGAResult> getAcls(String studyId, Li List familyUids = queryResult.getResults().stream().map(Family::getUid).collect(Collectors.toList()); if (CollectionUtils.isNotEmpty(members)) { - familyAcls = authorizationManager.getAcl(user, study.getUid(), familyUids, members, Enums.Resource.FAMILY, - FamilyPermissions.class); + familyAcls = authorizationManager.getAcl(organizationId, study.getUid(), familyUids, members, Enums.Resource.FAMILY, + FamilyPermissions.class, userId); } else { - familyAcls = authorizationManager.getAcl(user, study.getUid(), familyUids, Enums.Resource.FAMILY, - FamilyPermissions.class); + familyAcls = authorizationManager.getAcl(organizationId, study.getUid(), familyUids, Enums.Resource.FAMILY, + FamilyPermissions.class, userId); } // Include non-existing samples to the result list @@ -1142,14 +1191,14 @@ public OpenCGAResult> getAcls(String studyId, Li if (!missingMap.containsKey(familyId)) { Family family = queryResult.getResults().get(counter); resultList.add(familyAcls.getResults().get(counter)); - auditManager.audit(operationId, user, Enums.Action.FETCH_ACLS, Enums.Resource.FAMILY, family.getId(), + auditManager.audit(organizationId, operationId, userId, Enums.Action.FETCH_ACLS, Enums.Resource.FAMILY, family.getId(), family.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS), new ObjectMap()); counter++; } else { resultList.add(new AclEntryList<>()); eventList.add(new Event(Event.Type.ERROR, familyId, missingMap.get(familyId).getErrorMsg())); - auditManager.audit(operationId, user, Enums.Action.FETCH_ACLS, Enums.Resource.FAMILY, familyId, "", + auditManager.audit(organizationId, operationId, userId, Enums.Action.FETCH_ACLS, Enums.Resource.FAMILY, familyId, "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(0, "", missingMap.get(familyId).getErrorMsg())), new ObjectMap()); } @@ -1161,7 +1210,7 @@ public OpenCGAResult> getAcls(String studyId, Li familyAcls.setEvents(eventList); } catch (CatalogException e) { for (String familyId : familyList) { - auditManager.audit(operationId, user, Enums.Action.FETCH_ACLS, Enums.Resource.FAMILY, familyId, "", + auditManager.audit(organizationId, operationId, userId, Enums.Action.FETCH_ACLS, Enums.Resource.FAMILY, familyId, "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError()), new ObjectMap()); } @@ -1174,7 +1223,7 @@ public OpenCGAResult> getAcls(String studyId, Li } } } finally { - auditManager.finishAuditBatch(operationId); + auditManager.finishAuditBatch(organizationId, operationId); } return familyAcls; @@ -1182,8 +1231,11 @@ public OpenCGAResult> getAcls(String studyId, Li public OpenCGAResult> updateAcl(String studyId, FamilyAclParams aclParams, String memberList, ParamUtils.AclAction action, String token) throws CatalogException { - String user = userManager.getUserId(token); - Study study = studyManager.resolveId(studyId, user); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyId, userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) @@ -1196,7 +1248,7 @@ public OpenCGAResult> updateAcl(String studyId, List familyList = null; try { auditManager.initAuditBatch(operationUUID); - authorizationManager.checkCanAssignOrSeePermissions(study.getUid(), user); + authorizationManager.checkCanAssignOrSeePermissions(organizationId, study.getUid(), userId); int count = 0; count += aclParams.getFamily() != null && !aclParams.getFamily().isEmpty() ? 1 : 0; @@ -1248,7 +1300,7 @@ public OpenCGAResult> updateAcl(String studyId, members = Collections.emptyList(); } authorizationManager.checkNotAssigningPermissionsToAdminsGroup(members); - checkMembers(study.getUid(), members); + checkMembers(organizationId, study.getUid(), members); List aclParamsList = new LinkedList<>(); List familyUids = familyList.stream().map(Family::getUid).collect(Collectors.toList()); @@ -1290,31 +1342,31 @@ public OpenCGAResult> updateAcl(String studyId, switch (action) { case SET: - authorizationManager.setAcls(study.getUid(), members, aclParamsList); + authorizationManager.setAcls(organizationId, study.getUid(), members, aclParamsList); break; case ADD: - authorizationManager.addAcls(study.getUid(), members, aclParamsList); + authorizationManager.addAcls(organizationId, study.getUid(), members, aclParamsList); break; case REMOVE: - authorizationManager.removeAcls(members, aclParamsList); + authorizationManager.removeAcls(organizationId, members, aclParamsList); break; case RESET: for (AuthorizationManager.CatalogAclParams auxAclParams : aclParamsList) { auxAclParams.setPermissions(null); } - authorizationManager.removeAcls(members, aclParamsList); + authorizationManager.removeAcls(organizationId, members, aclParamsList); break; default: throw new CatalogException("Unexpected error occurred. No valid action found."); } - OpenCGAResult> remainingAcls = authorizationManager.getAcls(study.getUid(), + OpenCGAResult> remainingAcls = authorizationManager.getAcls(organizationId, study.getUid(), familyUids, members, Enums.Resource.FAMILY, FamilyPermissions.class); for (int i = 0; i < remainingAcls.getResults().size(); i++) { remainingAcls.getResults().get(i).setId(familyList.get(i).getId()); } for (Family family : familyList) { - auditManager.audit(operationUUID, user, Enums.Action.UPDATE_ACLS, Enums.Resource.FAMILY, family.getId(), + auditManager.audit(organizationId, operationUUID, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.FAMILY, family.getId(), family.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS), new ObjectMap()); } @@ -1323,54 +1375,14 @@ public OpenCGAResult> updateAcl(String studyId, } catch (CatalogException e) { if (familyList != null) { for (Family family : familyList) { - auditManager.audit(operationUUID, user, Enums.Action.UPDATE_ACLS, Enums.Resource.FAMILY, family.getId(), "", - study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, - e.getError()), new ObjectMap()); + auditManager.audit(organizationId, operationUUID, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.FAMILY, + family.getId(), "", study.getId(), study.getUuid(), auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError()), new ObjectMap()); } } throw e; } finally { - auditManager.finishAuditBatch(operationUUID); - } - } - - public DataResult facet(String studyId, Query query, QueryOptions options, boolean defaultStats, String token) - throws CatalogException { - ParamUtils.defaultObject(query, Query::new); - ParamUtils.defaultObject(options, QueryOptions::new); - - String userId = userManager.getUserId(token); - // We need to add variableSets and groups to avoid additional queries as it will be used in the catalogSolrManager - Study study = catalogManager.getStudyManager().resolveId(studyId, userId, new QueryOptions(QueryOptions.INCLUDE, - Arrays.asList(StudyDBAdaptor.QueryParams.VARIABLE_SET.key(), StudyDBAdaptor.QueryParams.GROUPS.key()))); - - ObjectMap auditParams = new ObjectMap() - .append("studyId", studyId) - .append("query", new Query(query)) - .append("options", options) - .append("defaultStats", defaultStats) - .append("token", token); - - try { - if (defaultStats || StringUtils.isEmpty(options.getString(QueryOptions.FACET))) { - String facet = options.getString(QueryOptions.FACET); - options.put(QueryOptions.FACET, StringUtils.isNotEmpty(facet) ? defaultFacet + ";" + facet : defaultFacet); - } - - AnnotationUtils.fixQueryAnnotationSearch(study, userId, query, authorizationManager); - - try (CatalogSolrManager catalogSolrManager = new CatalogSolrManager(catalogManager)) { - DataResult result = catalogSolrManager.facetedQuery(study, CatalogSolrManager.FAMILY_SOLR_COLLECTION, query, - options, userId); - auditManager.auditFacet(userId, Enums.Resource.FAMILY, study.getId(), study.getUuid(), auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); - - return result; - } - } catch (CatalogException e) { - auditManager.auditFacet(userId, Enums.Resource.FAMILY, study.getId(), study.getUuid(), auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(0, "", e.getMessage()))); - throw e; + auditManager.finishAuditBatch(organizationId, operationUUID); } } @@ -1379,35 +1391,43 @@ public DataResult facet(String studyId, Query query, QueryOptions op * It also makes sure the members provided inside the family object are valid and can be successfully created. * Returns the list of already existing members that will be associated to the family. * - * @param study study. - * @param family family object. - * @param members Already existing members. - * @param userId user id. + * @param organizationId Organization id. + * @param study study. + * @param family family object. + * @param members Already existing members. + * @param userId user id. * @return list of already existing members that will be associated to the family. * @throws CatalogException if there is any kind of error. */ - private List autoCompleteFamilyMembers(Study study, Family family, List members, String userId) - throws CatalogException { - if (family.getMembers() != null && !family.getMembers().isEmpty()) { + private List autoCompleteFamilyMembers(String organizationId, Study study, Family family, List members, + String userId) throws CatalogException { + if (CollectionUtils.isNotEmpty(family.getMembers())) { List memberList = new ArrayList<>(); // Check the user can create new individuals - authorizationManager.checkStudyPermission(study.getUid(), userId, StudyPermissions.Permissions.WRITE_INDIVIDUALS); + authorizationManager.checkStudyPermission(organizationId, study.getUid(), userId, + StudyPermissions.Permissions.WRITE_INDIVIDUALS); // Validate the individuals can be created and are valid for (Individual individual : family.getMembers()) { - catalogManager.getIndividualManager().validateNewIndividual(study, individual, null, userId, false); + catalogManager.getIndividualManager().validateNewIndividual(organizationId, study, individual, null, userId, false); memberList.add(individual); } family.setMembers(memberList); } - if (members != null && !members.isEmpty()) { - // We remove any possible duplicate - ArrayList deduplicatedMemberIds = new ArrayList<>(new HashSet<>(members)); + if (CollectionUtils.isNotEmpty(members)) { + // We check for possible duplicates + Set memberSet = new HashSet<>(); + for (String member : members) { + boolean unique = memberSet.add(member); + if (!unique) { + throw new CatalogException("Duplicated member '" + member + "' passed."); + } + } - InternalGetDataResult individualDataResult = catalogManager.getIndividualManager().internalGet(study.getUid(), - deduplicatedMemberIds, IndividualManager.INCLUDE_INDIVIDUAL_DISORDERS_PHENOTYPES, userId, false); + InternalGetDataResult individualDataResult = catalogManager.getIndividualManager().internalGet(organizationId, + study.getUid(), members, IndividualManager.INCLUDE_INDIVIDUAL_DISORDERS_PHENOTYPES, userId, false); return individualDataResult.getResults(); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java index 812bdfae0ca..386eb98cafa 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java @@ -24,19 +24,21 @@ import org.opencb.biodata.models.common.Status; import org.opencb.biodata.models.variant.VariantFileMetadata; import org.opencb.biodata.models.variant.metadata.VariantSetStats; -import org.opencb.commons.datastore.core.*; +import org.opencb.commons.datastore.core.Event; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.result.Error; import org.opencb.commons.utils.FileUtils; import org.opencb.commons.utils.ListUtils; import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager; import org.opencb.opencga.catalog.db.DBAdaptorFactory; import org.opencb.opencga.catalog.db.api.*; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; import org.opencb.opencga.catalog.exceptions.*; import org.opencb.opencga.catalog.io.IOManager; import org.opencb.opencga.catalog.io.IOManagerFactory; import org.opencb.opencga.catalog.models.InternalGetDataResult; -import org.opencb.opencga.catalog.stats.solr.CatalogSolrManager; import org.opencb.opencga.catalog.utils.*; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.IOUtils; @@ -45,6 +47,7 @@ import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.config.HookConfiguration; import org.opencb.opencga.core.models.AclEntryList; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.audit.AuditRecord; import org.opencb.opencga.core.models.common.AnnotationSet; import org.opencb.opencga.core.models.common.Enums; @@ -141,18 +144,18 @@ Enums.Resource getEntity() { } @Override - OpenCGAResult internalGet(long studyUid, String entry, @Nullable Query query, QueryOptions options, String user) - throws CatalogException { + OpenCGAResult internalGet(String organizationId, long studyUid, String entry, @Nullable Query query, QueryOptions options, + String user) throws CatalogException { // We make this comparison because in File, the absence of a fileName means the user is actually looking for the / directory if (StringUtils.isNotEmpty(entry) || entry == null) { ParamUtils.checkIsSingleID(entry); } - return internalGet(studyUid, Collections.singletonList(entry), query, options, user, false); + return internalGet(organizationId, studyUid, Collections.singletonList(entry), query, options, user, false); } @Override - InternalGetDataResult internalGet(long studyUid, List entryList, @Nullable Query query, QueryOptions options, - String user, boolean ignoreException) throws CatalogException { + InternalGetDataResult internalGet(String organizationId, long studyUid, List entryList, @Nullable Query query, + QueryOptions options, String user, boolean ignoreException) throws CatalogException { if (org.apache.commons.collections4.CollectionUtils.isEmpty(entryList)) { throw new CatalogException("Missing file entries."); } @@ -197,7 +200,7 @@ InternalGetDataResult internalGet(long studyUid, List entryList, @ // Ensure the field by which we are querying for will be kept in the results queryOptions = keepFieldInQueryOptions(queryOptions, idQueryParam.key()); - OpenCGAResult fileDataResult = fileDBAdaptor.get(studyUid, queryCopy, queryOptions, user); + OpenCGAResult fileDataResult = getFileDBAdaptor(organizationId).get(studyUid, queryCopy, queryOptions, user); if (fileDataResult.getNumResults() != correctedFileList.size() && idQueryParam == FileDBAdaptor.QueryParams.PATH && canBeSearchedAsName) { // We also search by name @@ -208,7 +211,7 @@ InternalGetDataResult internalGet(long studyUid, List entryList, @ // Ensure the field by which we are querying for will be kept in the results queryOptions = keepFieldInQueryOptions(queryOptions, FileDBAdaptor.QueryParams.NAME.key()); - OpenCGAResult nameDataResult = fileDBAdaptor.get(studyUid, queryCopy, queryOptions, user); + OpenCGAResult nameDataResult = getFileDBAdaptor(organizationId).get(studyUid, queryCopy, queryOptions, user); if (nameDataResult.getNumResults() > fileDataResult.getNumResults()) { fileDataResult = nameDataResult; fileStringFunction = File::getName; @@ -222,7 +225,7 @@ InternalGetDataResult internalGet(long studyUid, List entryList, @ } else { // The file could not be found or the user does not have permissions to see it // Check if the file can be found without adding the user restriction - OpenCGAResult resultsNoCheck = fileDBAdaptor.get(queryCopy, queryOptions); + OpenCGAResult resultsNoCheck = getFileDBAdaptor(organizationId).get(queryCopy, queryOptions); if (resultsNoCheck.getNumResults() == correctedFileList.size()) { throw new CatalogAuthorizationException("Permission denied. " + user + " is not allowed to see some or none of the files."); } @@ -231,14 +234,15 @@ InternalGetDataResult internalGet(long studyUid, List entryList, @ queryCopy = query == null ? new Query() : new Query(query); queryCopy.append(FileDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(FileDBAdaptor.QueryParams.PATH.key(), correctedFileList); - resultsNoCheck = fileDBAdaptor.get(queryCopy, queryOptions); + resultsNoCheck = getFileDBAdaptor(organizationId).get(queryCopy, queryOptions); if (resultsNoCheck.getNumResults() == correctedFileList.size()) { throw new CatalogAuthorizationException("Permission denied. " + user + " is not allowed to see some or none of the " + "files."); } } - throw CatalogException.notFound("files", getMissingFields(uniqueList, fileDataResult.getResults(), fileStringFunction)); + throw CatalogException.notFound("files at " + studyUid, getMissingFields(uniqueList, fileDataResult.getResults(), + fileStringFunction)); } } @@ -265,19 +269,20 @@ FileDBAdaptor.QueryParams getFieldFilter(List idList) throws CatalogExce return idQueryParam; } - private OpenCGAResult getFile(long studyUid, String fileUuid, QueryOptions options) throws CatalogException { + private OpenCGAResult getFile(String organizationId, long studyUid, String fileUuid, QueryOptions options) + throws CatalogException { Query query = new Query() .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(FileDBAdaptor.QueryParams.UUID.key(), fileUuid); - return fileDBAdaptor.get(query, options); + return getFileDBAdaptor(organizationId).get(query, options); } - public URI getUri(File file) throws CatalogException { + public URI getUri(String organizationId, File file) throws CatalogException { ParamUtils.checkObj(file, "File"); if (file.getUri() != null) { return file.getUri(); } else { - OpenCGAResult fileDataResult = fileDBAdaptor.get(file.getUid(), INCLUDE_STUDY_URI); + OpenCGAResult fileDataResult = getFileDBAdaptor(organizationId).get(file.getUid(), INCLUDE_STUDY_URI); if (fileDataResult.getNumResults() == 0) { throw new CatalogException("File " + file.getUid() + " not found"); } @@ -285,32 +290,37 @@ public URI getUri(File file) throws CatalogException { } } - public Study getStudy(File file, String sessionId) throws CatalogException { + public Study getStudy(String organizationId, File file, String token) throws CatalogException { ParamUtils.checkObj(file, "file"); - ParamUtils.checkObj(sessionId, "session id"); + ParamUtils.checkObj(token, "token"); if (file.getStudyUid() <= 0) { throw new CatalogException("Missing study uid field in file"); } - String user = userManager.getUserId(sessionId); + JwtPayload payload = userManager.validateToken(token); + String user = payload.getUserId(organizationId); Query query = new Query(StudyDBAdaptor.QueryParams.UID.key(), file.getStudyUid()); - OpenCGAResult studyDataResult = studyDBAdaptor.get(query, QueryOptions.empty(), user); + OpenCGAResult studyDataResult = getStudyDBAdaptor(organizationId).get(query, QueryOptions.empty(), user); if (studyDataResult.getNumResults() == 1) { return studyDataResult.first(); } else { - authorizationManager.checkCanViewStudy(file.getStudyUid(), user); + authorizationManager.checkCanViewStudy(organizationId, file.getStudyUid(), user); throw new CatalogException("Incorrect study uid"); } } - public void matchUpVariantFiles(String studyStr, List transformedFiles, String sessionId) throws CatalogException { - String userId = userManager.getUserId(sessionId); - Study study = studyManager.resolveId(studyStr, userId); + public void matchUpVariantFiles(String studyStr, List transformedFiles, String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, userId, organizationId); for (File transformedFile : transformedFiles) { - authorizationManager.checkFilePermission(study.getUid(), transformedFile.getUid(), userId, FilePermissions.WRITE); + authorizationManager.checkFilePermission(organizationId, study.getUid(), transformedFile.getUid(), userId, + FilePermissions.WRITE); String variantPathName = getMainVariantFile(transformedFile.getPath()); if (variantPathName == null) { // Skip the file. @@ -325,7 +335,7 @@ public void matchUpVariantFiles(String studyStr, List transformedFiles, St .append(FileDBAdaptor.QueryParams.PATH.key(), variantPathName) .append(FileDBAdaptor.QueryParams.BIOFORMAT.key(), File.Bioformat.VARIANT); - List fileList = fileDBAdaptor.get(query, new QueryOptions()).getResults(); + List fileList = getFileDBAdaptor(organizationId).get(query, new QueryOptions()).getResults(); if (fileList.isEmpty()) { // Search by name in the whole study @@ -335,7 +345,7 @@ public void matchUpVariantFiles(String studyStr, List transformedFiles, St .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()) .append(FileDBAdaptor.QueryParams.NAME.key(), variantFileName) .append(FileDBAdaptor.QueryParams.BIOFORMAT.key(), File.Bioformat.VARIANT); - fileList = new ArrayList<>(fileDBAdaptor.get(query, new QueryOptions()).getResults()); + fileList = new ArrayList<>(getFileDBAdaptor(organizationId).get(query, new QueryOptions()).getResults()); // In case of finding more than one file, try to find the proper one. if (fileList.size() > 1) { @@ -363,7 +373,7 @@ public void matchUpVariantFiles(String studyStr, List transformedFiles, St .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()) .append(FileDBAdaptor.QueryParams.PATH.key(), jsonPathName) .append(FileDBAdaptor.QueryParams.FORMAT.key(), File.Format.JSON); - fileList = fileDBAdaptor.get(query, new QueryOptions()).getResults(); + fileList = getFileDBAdaptor(organizationId).get(query, new QueryOptions()).getResults(); if (fileList.size() != 1) { // Skip. This should not ever happen logger.warn("The json file corresponding to the file " + transformedFile.getName() + " could not be found"); @@ -381,7 +391,7 @@ public void matchUpVariantFiles(String studyStr, List transformedFiles, St if (!relatedFiles.contains(producedFromRelation)) { relatedFiles.add(producedFromRelation); ObjectMap params = new ObjectMap(FileDBAdaptor.QueryParams.RELATED_FILES.key(), relatedFiles); - fileDBAdaptor.update(json.getUid(), params, QueryOptions.empty()); + getFileDBAdaptor(organizationId).update(json.getUid(), params, QueryOptions.empty()); } // Update transformed file @@ -392,7 +402,7 @@ public void matchUpVariantFiles(String studyStr, List transformedFiles, St relatedFiles.add(producedFromRelation); transformedFile.setRelatedFiles(relatedFiles); ObjectMap params = new ObjectMap(FileDBAdaptor.QueryParams.RELATED_FILES.key(), relatedFiles); - fileDBAdaptor.update(transformedFile.getUid(), params, QueryOptions.empty()); + getFileDBAdaptor(organizationId).update(transformedFile.getUid(), params, QueryOptions.empty()); } // Update vcf file @@ -412,7 +422,7 @@ public void matchUpVariantFiles(String studyStr, List transformedFiles, St index.setStatus(new VariantIndexStatus(VariantIndexStatus.TRANSFORMED, "Found transformed file")); } ObjectMap params = new ObjectMap(FileDBAdaptor.QueryParams.INTERNAL_VARIANT_INDEX.key(), index); - fileDBAdaptor.update(vcf.getUid(), params, QueryOptions.empty()); + getFileDBAdaptor(organizationId).update(vcf.getUid(), params, QueryOptions.empty()); // Update variant stats Path statsFile = Paths.get(json.getUri().getRawPath()); @@ -426,54 +436,61 @@ public void matchUpVariantFiles(String studyStr, List transformedFiles, St new FileQualityControl().setVariant( new VariantFileQualityControl(stats, null))), new QueryOptions(), - sessionId); + token); } catch (IOException e) { throw new CatalogException("Error reading file \"" + statsFile + "\"", e); } } } - public OpenCGAResult updateFileInternalVariantIndex(File file, FileInternalVariantIndex index, String token) + public OpenCGAResult updateFileInternalVariantIndex(String studyFqn, File file, FileInternalVariantIndex index, String token) throws CatalogException { - return updateFileInternalField(file, index, FileDBAdaptor.QueryParams.INTERNAL_VARIANT_INDEX.key(), token); + return updateFileInternalField(studyFqn, file, index, FileDBAdaptor.QueryParams.INTERNAL_VARIANT_INDEX.key(), token); } - public OpenCGAResult updateFileInternalVariantAnnotationIndex(File file, FileInternalVariantAnnotationIndex index, String token) + public OpenCGAResult updateFileInternalVariantAnnotationIndex(String studyFqn, File file, + FileInternalVariantAnnotationIndex index, String token) throws CatalogException { - return updateFileInternalField(file, index, FileDBAdaptor.QueryParams.INTERNAL_VARIANT_ANNOTATION_INDEX.key(), token); + return updateFileInternalField(studyFqn, file, index, FileDBAdaptor.QueryParams.INTERNAL_VARIANT_ANNOTATION_INDEX.key(), + token); } - public OpenCGAResult updateFileInternalVariantSecondaryAnnotationIndex(File file, FileInternalVariantSecondaryAnnotationIndex index, - String token) - throws CatalogException { - return updateFileInternalField(file, index, Arrays.asList( + public OpenCGAResult updateFileInternalVariantSecondaryAnnotationIndex(String studyFqn, File file, + FileInternalVariantSecondaryAnnotationIndex index, + String token) throws CatalogException { + return updateFileInternalField(studyFqn, file, index, Arrays.asList( FileDBAdaptor.QueryParams.INTERNAL_VARIANT_SECONDARY_ANNOTATION_INDEX.key(), FileDBAdaptor.QueryParams.INTERNAL_VARIANT_SECONDARY_INDEX.key() ), token); } - public OpenCGAResult updateFileInternalAlignmentIndex(File file, FileInternalAlignmentIndex index, String token) - throws CatalogException { - return updateFileInternalField(file, index, FileDBAdaptor.QueryParams.INTERNAL_ALIGNMENT_INDEX.key(), token); + public OpenCGAResult updateFileInternalAlignmentIndex(String studyFqn, File file, FileInternalAlignmentIndex index, + String token) throws CatalogException { + return updateFileInternalField(studyFqn, file, index, FileDBAdaptor.QueryParams.INTERNAL_ALIGNMENT_INDEX.key(), token); } - private OpenCGAResult updateFileInternalField(File file, Object value, String fieldKey, String token) throws CatalogException { - return updateFileInternalField(file, value, Collections.singletonList(fieldKey), token); + private OpenCGAResult updateFileInternalField(String studyFqn, File file, Object value, String fieldKey, String token) + throws CatalogException { + return updateFileInternalField(studyFqn, file, value, Collections.singletonList(fieldKey), token); } - private OpenCGAResult updateFileInternalField(File file, Object value, List fieldKeys, String token) + private OpenCGAResult updateFileInternalField(String studyFqn, File file, Object value, List fieldKeys, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyDBAdaptor.get(file.getStudyUid(), StudyManager.INCLUDE_STUDY_IDS).first(); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn catalogFqn = CatalogFqn.extractFqnFromStudy(studyFqn, tokenPayload); + String organizationId = catalogFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = getStudyDBAdaptor(organizationId).get(file.getStudyUid(), StudyManager.INCLUDE_STUDY_IDS).first(); ObjectMap auditParams = new ObjectMap() + .append("studyFqn", studyFqn) .append("file", file) .append("token", token); for (String fieldKey : fieldKeys) { auditParams.append(fieldKey, value); } - authorizationManager.checkFilePermission(study.getUid(), file.getUid(), userId, FilePermissions.WRITE); + authorizationManager.checkFilePermission(organizationId, study.getUid(), file.getUid(), userId, FilePermissions.WRITE); ObjectMap params; try { @@ -485,28 +502,13 @@ private OpenCGAResult updateFileInternalField(File file, Object value, List update = fileDBAdaptor.update(file.getUid(), params, QueryOptions.empty()); - auditManager.auditUpdate(userId, Enums.Resource.FILE, file.getId(), file.getUuid(), study.getId(), study.getUuid(), + OpenCGAResult update = getFileDBAdaptor(organizationId).update(file.getUid(), params, QueryOptions.empty()); + auditManager.auditUpdate(organizationId, userId, Enums.Resource.FILE, file.getId(), file.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return new OpenCGAResult<>(update.getTime(), update.getEvents(), 1, Collections.emptyList(), 1); } - @Deprecated - public OpenCGAResult getParents(long fileId, QueryOptions options, String sessionId) throws CatalogException { - OpenCGAResult fileDataResult = fileDBAdaptor.get(fileId, new QueryOptions(QueryOptions.INCLUDE, - Arrays.asList(FileDBAdaptor.QueryParams.PATH.key(), FileDBAdaptor.QueryParams.STUDY_UID.key()))); - - if (fileDataResult.getNumResults() == 0) { - return fileDataResult; - } - - String userId = userManager.getUserId(sessionId); - authorizationManager.checkFilePermission(fileDataResult.first().getStudyUid(), fileId, userId, FilePermissions.VIEW); - - return getParents(fileDataResult.first().getStudyUid(), fileDataResult.first().getPath(), true, options); - } - public OpenCGAResult createFolder(String studyStr, String path, boolean parents, String description, QueryOptions options, String token) throws CatalogException { return createFolder(studyStr, path, parents, description, "", options, token); @@ -517,8 +519,11 @@ public OpenCGAResult createFolder(String studyStr, String path, boolean pa ParamUtils.checkPath(path, "folderPath"); options = ParamUtils.defaultObject(options, QueryOptions::new); - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, userId, organizationId); if (path.startsWith("/")) { path = path.substring(1); @@ -528,7 +533,7 @@ public OpenCGAResult createFolder(String studyStr, String path, boolean pa } OpenCGAResult fileDataResult; - switch (checkPathExists(path, study.getUid())) { + switch (checkPathExists(organizationId, study.getUid(), path)) { case FREE_PATH: FileCreateParams file = new FileCreateParams() .setPath(path) @@ -541,7 +546,7 @@ public OpenCGAResult createFolder(String studyStr, String path, boolean pa Query query = new Query() .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()) .append(FileDBAdaptor.QueryParams.PATH.key(), path); - fileDataResult = fileDBAdaptor.get(study.getUid(), query, options, userId); + fileDataResult = getFileDBAdaptor(organizationId).get(study.getUid(), query, options, userId); fileDataResult.getEvents().add(new Event(Event.Type.WARNING, path, "Folder already existed")); break; case FILE_EXISTS: @@ -553,14 +558,18 @@ public OpenCGAResult createFolder(String studyStr, String path, boolean pa } @Override - public OpenCGAResult create(String studyStr, File entry, QueryOptions options, String token) throws CatalogException { + public OpenCGAResult create(String studyStr, File entry, QueryOptions options, String token) + throws CatalogException { throw new NotImplementedException("Call to create passing parents and content variables"); } public OpenCGAResult create(String studyStr, FileCreateParams createParams, boolean parents, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("study", studyStr) @@ -603,16 +612,16 @@ public OpenCGAResult create(String studyStr, FileCreateParams createParams } } - OpenCGAResult parentResult = getParents(study.getUid(), path, false, FileManager.INCLUDE_FILE_URI_PATH); + OpenCGAResult parentResult = getParents(organizationId, study.getUid(), path, false, FileManager.INCLUDE_FILE_URI_PATH); // Check user can write in path - authorizationManager.checkFilePermission(study.getUid(), parentResult.first().getUid(), userId, + authorizationManager.checkFilePermission(organizationId, study.getUid(), parentResult.first().getUid(), userId, FilePermissions.WRITE); // Check available path Query pathQuery = new Query() .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()) .append(FileDBAdaptor.QueryParams.PATH.key(), path); - boolean fileExists = fileDBAdaptor.count(pathQuery).getNumMatches() > 0; + boolean fileExists = getFileDBAdaptor(organizationId).count(pathQuery).getNumMatches() > 0; if (fileExists) { throw new CatalogException("There already exists a file '" + path + "'"); } @@ -631,8 +640,8 @@ public OpenCGAResult create(String studyStr, FileCreateParams createParams if (createParams.getType().equals(File.Type.FILE)) { if (org.apache.commons.collections4.CollectionUtils.isNotEmpty(createParams.getSampleIds())) { // Check samples - InternalGetDataResult sampleResult = catalogManager.getSampleManager().internalGet(study.getUid(), - createParams.getSampleIds(), SampleManager.INCLUDE_SAMPLE_IDS, userId, true); + InternalGetDataResult sampleResult = catalogManager.getSampleManager().internalGet(organizationId, + study.getUid(), createParams.getSampleIds(), SampleManager.INCLUDE_SAMPLE_IDS, userId, true); if (!sampleResult.getMissing().isEmpty()) { throw new CatalogException("Could not find samples: '" + sampleResult.getMissing() .stream().map(InternalGetDataResult.Missing::getId).collect(Collectors.joining("', '")) + "'"); @@ -645,7 +654,7 @@ public OpenCGAResult create(String studyStr, FileCreateParams createParams createParams.getCreationDate(), createParams.getModificationDate(), createParams.getDescription(), false, 0, createParams.getSoftware(), null, createParams.getSampleIds(), null, createParams.getJobId(), 1, createParams.getTags(), null, null, null, createParams.getStatus() != null ? createParams.getStatus().toStatus() : null, null, null); - List eventList = validateNewFile(study, file, false); + List eventList = validateNewFile(organizationId, study, file, false); fileId = file.getId(); if (file.getType().equals(File.Type.FILE)) { @@ -673,29 +682,30 @@ public OpenCGAResult create(String studyStr, FileCreateParams createParams // No samples added on creation, so we will check if it is a VCF or any other file from which a list of samples could // be extracted. This will also calculate the size new FileMetadataReader(catalogManager).addMetadataInformation(study.getFqn(), file); - validateNewSamples(study, file, existingSamples, nonExistingSamples, token); + validateNewSamples(organizationId, study, file, existingSamples, nonExistingSamples, tokenPayload); } else { long fileSize = ioManager.getFileSize(file.getUri()); file.setSize(fileSize); } } // Register file in Catalog - OpenCGAResult result = register(study, file, existingSamples, nonExistingSamples, parents, QueryOptions.empty(), token); + OpenCGAResult result = register(organizationId, study, file, existingSamples, nonExistingSamples, parents, + QueryOptions.empty(), tokenPayload); result.setEvents(eventList); - auditManager.auditCreate(userId, Enums.Resource.FILE, file.getId(), file.getUuid(), study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.auditCreate(organizationId, userId, Enums.Resource.FILE, file.getId(), file.getUuid(), study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return result; } catch (CatalogException e) { - auditManager.auditCreate(userId, Enums.Resource.FILE, fileId, "", study.getId(), study.getUuid(), auditParams, + auditManager.auditCreate(organizationId, userId, Enums.Resource.FILE, fileId, "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } - List validateNewFile(Study study, File file, boolean overwrite) throws CatalogException { + List validateNewFile(String organizationId, Study study, File file, boolean overwrite) throws CatalogException { /** Check and set all the params and create a File object **/ ParamUtils.checkObj(file, "File"); ParamUtils.checkPath(file.getPath(), "path"); @@ -765,9 +775,9 @@ List validateNewFile(Study study, File file, boolean overwrite) throws Ca URI uri; try { if (file.getType() == File.Type.DIRECTORY) { - uri = getFileUri(study.getUid(), file.getPath(), true); + uri = getFileUri(organizationId, study.getUid(), file.getPath(), true); } else { - uri = getFileUri(study.getUid(), file.getPath(), false); + uri = getFileUri(organizationId, study.getUid(), file.getPath(), false); } } catch (URISyntaxException e) { throw new CatalogException(e); @@ -779,14 +789,14 @@ List validateNewFile(Study study, File file, boolean overwrite) throws Ca Query query = new Query() .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()) .append(FileDBAdaptor.QueryParams.PATH.key(), file.getPath()); - if (fileDBAdaptor.count(query).getNumMatches() > 0) { + if (getFileDBAdaptor(organizationId).count(query).getNumMatches() > 0) { logger.warn("The file '{}' already exists in catalog", file.getPath()); throw new CatalogException("The file '" + file.getPath() + "' already exists in catalog"); } query = new Query() .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()) .append(FileDBAdaptor.QueryParams.URI.key(), uri); - OpenCGAResult fileResult = fileDBAdaptor.get(query, + OpenCGAResult fileResult = getFileDBAdaptor(organizationId).get(query, new QueryOptions(QueryOptions.INCLUDE, FileDBAdaptor.QueryParams.PATH.key())); if (fileResult.getNumResults() > 0) { logger.warn("The uri '{}' of the file is already in catalog but in path '{}'.", uri, fileResult.first().getPath()); @@ -807,24 +817,27 @@ List validateNewFile(Study study, File file, boolean overwrite) throws Ca return eventList; } - OpenCGAResult register(Study study, File file, List existingSamples, List nonExistingSamples, - boolean parents, QueryOptions options, String sessionId) throws CatalogException { - String userId = userManager.getUserId(sessionId); + OpenCGAResult register(String organizationId, Study study, File file, List existingSamples, + List nonExistingSamples, boolean parents, QueryOptions options, JwtPayload tokenPayload) + throws CatalogException { + String token = tokenPayload.getToken(); + String userId = tokenPayload.getUserId(organizationId); + long studyId = study.getUid(); //Find parent. If parents == true, create folders. String parentPath = getParentPath(file.getPath()); - long parentFileId = fileDBAdaptor.getId(studyId, parentPath); + long parentFileId = getFileDBAdaptor(organizationId).getId(studyId, parentPath); boolean newParent = false; if (parentFileId < 0 && StringUtils.isNotEmpty(parentPath)) { if (parents) { newParent = true; File parentFile = new File(File.Type.DIRECTORY, File.Format.NONE, File.Bioformat.NONE, parentPath, "", FileInternal.init(), 0, Collections.emptyList(), null, "", new FileQualityControl(), Collections.emptyMap(), Collections.emptyMap()); - validateNewFile(study, parentFile, false); - parentFileId = register(study, parentFile, Collections.emptyList(), Collections.emptyList(), parents, options, sessionId) - .first().getUid(); + validateNewFile(organizationId, study, parentFile, false); + parentFileId = register(organizationId, study, parentFile, Collections.emptyList(), Collections.emptyList(), parents, + options, tokenPayload).first().getUid(); } else { throw new CatalogDBException("Directory not found " + parentPath); } @@ -836,41 +849,42 @@ OpenCGAResult register(Study study, File file, List existingSample } else { if (!newParent) { //If parent has been created, for sure we have permissions to create the new file. - authorizationManager.checkFilePermission(studyId, parentFileId, userId, FilePermissions.WRITE); + authorizationManager.checkFilePermission(organizationId, studyId, parentFileId, userId, FilePermissions.WRITE); } } - fileDBAdaptor.insert(studyId, file, existingSamples, nonExistingSamples, study.getVariableSets(), options); - OpenCGAResult queryResult = getFile(studyId, file.getUuid(), options); + getFileDBAdaptor(organizationId).insert(studyId, file, existingSamples, nonExistingSamples, study.getVariableSets(), options); + OpenCGAResult queryResult = getFile(organizationId, studyId, file.getUuid(), options); // We obtain the permissions set in the parent folder and set them to the file or folder being created - OpenCGAResult> allFileAcls = authorizationManager.getAcls(studyId, parentFileId, + OpenCGAResult> allFileAcls = authorizationManager.getAcls(organizationId, studyId, parentFileId, Enums.Resource.FILE, FilePermissions.class); // Propagate ACLs if (allFileAcls.getNumResults() > 0) { - authorizationManager.replicateAcls(Collections.singletonList(queryResult.first().getUid()), allFileAcls.getResults().get(0), - Enums.Resource.FILE); + authorizationManager.replicateAcls(organizationId, Collections.singletonList(queryResult.first().getUid()), + allFileAcls.getResults().get(0), Enums.Resource.FILE); } - matchUpVariantFiles(study.getFqn(), queryResult.getResults(), sessionId); + matchUpVariantFiles(study.getFqn(), queryResult.getResults(), token); return queryResult; } - private void validateNewSamples(Study study, File file, List existingSamples, List nonExistingSamples, String sessionId) - throws CatalogException { + private void validateNewSamples(String organizationId, Study study, File file, List existingSamples, + List nonExistingSamples, JwtPayload jwtPayload) throws CatalogException { if (file.getSampleIds() == null || file.getSampleIds().isEmpty()) { return; } - String userId = catalogManager.getUserManager().getUserId(sessionId); + String userId = jwtPayload.getUserId(organizationId); + String token = jwtPayload.getToken(); - InternalGetDataResult sampleResult = catalogManager.getSampleManager().internalGet(study.getUid(), file.getSampleIds(), - SampleManager.INCLUDE_SAMPLE_IDS, userId, true); + InternalGetDataResult sampleResult = catalogManager.getSampleManager().internalGet(organizationId, study.getUid(), + file.getSampleIds(), SampleManager.INCLUDE_SAMPLE_IDS, userId, true); existingSamples.addAll(sampleResult.getResults()); for (InternalGetDataResult.Missing missing : sampleResult.getMissing()) { Sample sample = new Sample().setId(missing.getId()); - catalogManager.getSampleManager().validateNewSample(study, sample, userId); + catalogManager.getSampleManager().validateNewSample(organizationId, study, sample, userId); nonExistingSamples.add(sample); } } @@ -916,8 +930,11 @@ public OpenCGAResult upload(String studyStr, InputStream fileInputStream, if (StringUtils.isNotEmpty(expectedChecksum)) { calculateChecksum = true; } - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId, StudyManager.INCLUDE_VARIABLE_SET); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, StudyManager.INCLUDE_VARIABLE_SET, userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("studyStr", studyStr) @@ -929,13 +946,13 @@ public OpenCGAResult upload(String studyStr, InputStream fileInputStream, .append("expectedSize", expectedSize) .append("token", token); try { - validateNewFile(study, file, overwrite); + validateNewFile(organizationId, study, file, overwrite); File overwrittenFile = null; Query query = new Query() .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()) .append(FileDBAdaptor.QueryParams.PATH.key(), file.getPath()); - OpenCGAResult fileDataResult = fileDBAdaptor.get(query, QueryOptions.empty()); + OpenCGAResult fileDataResult = getFileDBAdaptor(organizationId).get(query, QueryOptions.empty()); if (fileDataResult.getNumResults() > 0) { if (overwrite) { overwrittenFile = fileDataResult.first(); @@ -944,16 +961,16 @@ public OpenCGAResult upload(String studyStr, InputStream fileInputStream, } } - OpenCGAResult parentFolders = getParents(study.getUid(), file.getPath(), false, QueryOptions.empty()); + OpenCGAResult parentFolders = getParents(organizationId, study.getUid(), file.getPath(), false, QueryOptions.empty()); if (parentFolders.getNumResults() == 0) { // There always must be at least the root folder throw new CatalogException("Unexpected error happened."); } // Check permissions over the most internal path - authorizationManager.checkFilePermission(study.getUid(), parentFolders.first().getUid(), userId, + authorizationManager.checkFilePermission(organizationId, study.getUid(), parentFolders.first().getUid(), userId, FilePermissions.UPLOAD); - authorizationManager.checkFilePermission(study.getUid(), parentFolders.first().getUid(), userId, + authorizationManager.checkFilePermission(organizationId, study.getUid(), parentFolders.first().getUid(), userId, FilePermissions.WRITE); // We obtain the basic studyPath where we will upload the file temporarily @@ -1050,7 +1067,7 @@ public OpenCGAResult upload(String studyStr, InputStream fileInputStream, // Improve metadata information and extract samples if any new FileMetadataReader(catalogManager).addMetadataInformation(study.getFqn(), file); - validateNewSamples(study, file, existingSamples, nonExistingSamples, token); + validateNewSamples(organizationId, study, file, existingSamples, nonExistingSamples, tokenPayload); } catch (CatalogException e) { try { logger.error("Error uploading file. Deleting temp directory", e); @@ -1093,10 +1110,10 @@ public OpenCGAResult upload(String studyStr, InputStream fileInputStream, params.put(FileDBAdaptor.QueryParams.STATS.key(), stats); } - fileDBAdaptor.update(overwrittenFile.getUid(), params, null, queryOptions); + getFileDBAdaptor(organizationId).update(overwrittenFile.getUid(), params, null, queryOptions); } else { // We need to register a new file - register(study, file, existingSamples, nonExistingSamples, parents, QueryOptions.empty(), token); + register(organizationId, study, file, existingSamples, nonExistingSamples, parents, QueryOptions.empty(), tokenPayload); } } catch (CatalogException e) { try { @@ -1108,12 +1125,12 @@ public OpenCGAResult upload(String studyStr, InputStream fileInputStream, throw new CatalogException("Upload file failed. Could not register the file in the DB", e); } - auditManager.auditCreate(userId, Enums.Action.UPLOAD, Enums.Resource.FILE, file.getId(), file.getUuid(), + auditManager.auditCreate(organizationId, userId, Enums.Action.UPLOAD, Enums.Resource.FILE, file.getId(), file.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); - return fileDBAdaptor.get(query, QueryOptions.empty()); + return getFileDBAdaptor(organizationId).get(query, QueryOptions.empty()); } catch (CatalogException e) { - auditManager.auditCreate(userId, Enums.Action.UPLOAD, Enums.Resource.FILE, file.getId(), "", study.getId(), + auditManager.auditCreate(organizationId, userId, Enums.Action.UPLOAD, Enums.Resource.FILE, file.getId(), "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -1132,8 +1149,11 @@ public OpenCGAResult upload(String studyStr, InputStream fileInputStream, */ public OpenCGAResult moveAndRegister(String studyStr, Path fileSource, @Nullable Path folderDestiny, @Nullable String path, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("studyStr", studyStr) @@ -1159,7 +1179,7 @@ public OpenCGAResult moveAndRegister(String studyStr, Path fileSource, @Nu if (path.startsWith("/")) { path = path.substring(1); } - File parentFolder = getParents(study.getUid(), path, false, INCLUDE_FILE_URI_PATH).first(); + File parentFolder = getParents(organizationId, study.getUid(), path, false, INCLUDE_FILE_URI_PATH).first(); // We get the relative path String relativePath = Paths.get(parentFolder.getPath()).relativize(Paths.get(path)).toString(); @@ -1183,11 +1203,13 @@ public OpenCGAResult moveAndRegister(String studyStr, Path fileSource, @Nu path = Paths.get(study.getUri().getPath()).relativize(folderDestiny).toString(); } - File parentFolder = getParents(study.getUid(), path, false, INCLUDE_FILE_URI_PATH).first(); - authorizationManager.checkFilePermission(study.getUid(), parentFolder.getUid(), userId, FilePermissions.WRITE); + File parentFolder = getParents(organizationId, study.getUid(), path, false, INCLUDE_FILE_URI_PATH).first(); + authorizationManager.checkFilePermission(organizationId, study.getUid(), parentFolder.getUid(), userId, + FilePermissions.WRITE); } else { // It will be moved to an external folder. Only admins can move to that directory - if (!authorizationManager.isOwnerOrAdmin(study.getUid(), userId)) { + long studyId = study.getUid(); + if (!authorizationManager.isAtLeastStudyAdministrator(organizationId, studyId, userId)) { throw new CatalogAuthorizationException("Only owners or administrative users are allowed to move to folders different " + "than the main OpenCGA workspace"); } @@ -1203,7 +1225,7 @@ public OpenCGAResult moveAndRegister(String studyStr, Path fileSource, @Nu Query query = new Query() .append(FileDBAdaptor.QueryParams.PATH.key(), filePath) .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - if (fileDBAdaptor.count(query).getNumMatches() > 0) { + if (getFileDBAdaptor(organizationId).count(query).getNumMatches() > 0) { throw new CatalogException("Path '" + filePath + "' already in use in OpenCGA"); } @@ -1227,9 +1249,10 @@ public OpenCGAResult moveAndRegister(String studyStr, Path fileSource, @Nu OpenCGAResult result; if (external) { - result = link(study.getFqn(), folderDestiny.resolve(fileName).toUri(), path, new ObjectMap("parents", true), token); + result = link(study.getFqn(), folderDestiny.resolve(fileName).toUri(), path, new ObjectMap("parents", true), + token); } else { - CheckPath checkPath = checkPathExists(filePath, study.getUid()); + CheckPath checkPath = checkPathExists(organizationId, study.getUid(), filePath); if (checkPath != CheckPath.FREE_PATH) { throw new CatalogException("A file or folder with the same name already exists in the path of Catalog"); } @@ -1237,32 +1260,33 @@ public OpenCGAResult moveAndRegister(String studyStr, Path fileSource, @Nu File file = new File() .setPath(filePath) .setType(File.Type.FILE); - validateNewFile(study, file, false); + validateNewFile(organizationId, study, file, false); List nonExistingSamples = new LinkedList<>(); List existingSamples = new LinkedList<>(); if (file.getType() == File.Type.FILE && ioManager.exists(file.getUri())) { new FileMetadataReader(catalogManager).addMetadataInformation(study.getFqn(), file); - validateNewSamples(study, file, existingSamples, nonExistingSamples, token); + validateNewSamples(organizationId, study, file, existingSamples, nonExistingSamples, tokenPayload); } - result = register(study, file, existingSamples, nonExistingSamples, true, QueryOptions.empty(), token); + result = register(organizationId, study, file, existingSamples, nonExistingSamples, true, QueryOptions.empty(), + tokenPayload); } - auditManager.audit(userId, Enums.Action.MOVE_AND_REGISTER, Enums.Resource.FILE, result.first().getId(), + auditManager.audit(organizationId, userId, Enums.Action.MOVE_AND_REGISTER, Enums.Resource.FILE, result.first().getId(), result.first().getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return result; } catch (CatalogException e) { - auditManager.audit(userId, Enums.Action.MOVE_AND_REGISTER, Enums.Resource.FILE, "", "", study.getId(), + auditManager.audit(organizationId, userId, Enums.Action.MOVE_AND_REGISTER, Enums.Resource.FILE, "", "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } @Deprecated - public OpenCGAResult get(Long fileId, QueryOptions options, String sessionId) throws CatalogException { - return get(null, String.valueOf(fileId), options, sessionId); + public OpenCGAResult get(String organizationId, Long fileUid, QueryOptions options, String token) throws CatalogException { + return get(null, String.valueOf(fileUid), options, token); } public OpenCGAResult getTree(@Nullable String studyId, String fileId, int maxDepth, QueryOptions options, String token) @@ -1271,8 +1295,11 @@ public OpenCGAResult getTree(@Nullable String studyId, String fileId, options = ParamUtils.defaultObject(options, QueryOptions::new); - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyId, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyId, userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) @@ -1306,7 +1333,7 @@ public OpenCGAResult getTree(@Nullable String studyId, String fileId, } } - File file = internalGet(study.getUid(), fileId, options, userId).first(); + File file = internalGet(organizationId, study.getUid(), fileId, options, userId).first(); // Check if the id does not correspond to a directory if (!file.getType().equals(File.Type.DIRECTORY)) { @@ -1327,7 +1354,7 @@ public OpenCGAResult getTree(@Nullable String studyId, String fileId, FileTreeBuilder treeBuilder = new FileTreeBuilder(file); int numResults; - try (DBIterator iterator = fileDBAdaptor.iterator(study.getUid(), query, options, userId)) { + try (DBIterator iterator = getFileDBAdaptor(organizationId).iterator(study.getUid(), query, options, userId)) { if (iterator.getNumMatches() > MAX_LIMIT) { throw new CatalogException("Please, decrease the maximum depth. More than " + MAX_LIMIT + " files found"); } @@ -1339,24 +1366,27 @@ public OpenCGAResult getTree(@Nullable String studyId, String fileId, FileTree fileTree = treeBuilder.toFileTree(); int dbTime = (int) (System.currentTimeMillis() - startTime); - auditManager.audit(userId, Enums.Action.TREE, Enums.Resource.FILE, file.getId(), file.getUuid(), study.getId(), + auditManager.audit(organizationId, userId, Enums.Action.TREE, Enums.Resource.FILE, file.getId(), file.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return new OpenCGAResult<>(dbTime, Collections.emptyList(), numResults, Collections.singletonList(fileTree), numResults); } catch (CatalogException e) { - auditManager.audit(userId, Enums.Action.TREE, Enums.Resource.FILE, fileId, "", study.getId(), study.getUuid(), + auditManager.audit(organizationId, userId, Enums.Action.TREE, Enums.Resource.FILE, fileId, "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } - public OpenCGAResult getFilesFromFolder(String folderStr, String studyStr, QueryOptions options, String sessionId) + public OpenCGAResult getFilesFromFolder(String studyStr, String folderStr, QueryOptions options, String token) throws CatalogException { ParamUtils.checkObj(folderStr, "folder"); - String userId = userManager.getUserId(sessionId); - Study study = studyManager.resolveId(studyStr, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, userId, organizationId); - File file = internalGet(study.getUid(), folderStr, new QueryOptions(QueryOptions.INCLUDE, + File file = internalGet(organizationId, study.getUid(), folderStr, new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(FileDBAdaptor.QueryParams.PATH.key(), FileDBAdaptor.QueryParams.TYPE.key())), userId).first(); options = ParamUtils.defaultObject(options, QueryOptions::new); @@ -1365,24 +1395,27 @@ public OpenCGAResult getFilesFromFolder(String folderStr, String studyStr, throw new CatalogDBException("File {path:'" + file.getPath() + "'} is not a folder."); } Query query = new Query(FileDBAdaptor.QueryParams.DIRECTORY.key(), file.getPath()); - return search(studyStr, query, options, sessionId); + return search(studyStr, query, options, token); } @Override public DBIterator iterator(String studyStr, Query query, QueryOptions options, String sessionId) throws CatalogException { query = ParamUtils.defaultObject(query, Query::new); options = ParamUtils.defaultObject(options, QueryOptions::new); - String userId = userManager.getUserId(sessionId); - Study study = studyManager.resolveId(studyStr, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(sessionId); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, userId, organizationId); Query finalQuery = new Query(query); // Fix query if it contains any annotation - AnnotationUtils.fixQueryAnnotationSearch(study, finalQuery); + AnnotationUtils.fixQueryAnnotationSearch(organizationId, study, finalQuery); AnnotationUtils.fixQueryOptionAnnotation(options); fixQueryObject(study, finalQuery, userId); finalQuery.append(FileDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - return fileDBAdaptor.iterator(study.getUid(), query, options, userId); + return getFileDBAdaptor(organizationId).iterator(study.getUid(), query, options, userId); } @Override @@ -1391,9 +1424,12 @@ public OpenCGAResult search(String studyId, Query query, QueryOptions opti Query finalQuery = new Query(query); options = ParamUtils.defaultObject(options, QueryOptions::new); - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyId, userId, new QueryOptions(QueryOptions.INCLUDE, - StudyDBAdaptor.QueryParams.VARIABLE_SET.key())); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyId, new QueryOptions(QueryOptions.INCLUDE, + StudyDBAdaptor.QueryParams.VARIABLE_SET.key()), userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) @@ -1402,18 +1438,18 @@ public OpenCGAResult search(String studyId, Query query, QueryOptions opti .append("token", token); try { // Fix query if it contains any annotation - AnnotationUtils.fixQueryAnnotationSearch(study, finalQuery); + AnnotationUtils.fixQueryAnnotationSearch(organizationId, study, finalQuery); AnnotationUtils.fixQueryOptionAnnotation(options); fixQueryObject(study, finalQuery, userId); finalQuery.append(FileDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - OpenCGAResult queryResult = fileDBAdaptor.get(study.getUid(), finalQuery, options, userId); - auditManager.auditSearch(userId, Enums.Resource.FILE, study.getId(), study.getUuid(), auditParams, + OpenCGAResult queryResult = getFileDBAdaptor(organizationId).get(study.getUid(), finalQuery, options, userId); + auditManager.auditSearch(organizationId, userId, Enums.Resource.FILE, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return queryResult; } catch (CatalogException e) { - auditManager.auditSearch(userId, Enums.Resource.FILE, study.getId(), study.getUuid(), auditParams, + auditManager.auditSearch(organizationId, userId, Enums.Resource.FILE, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -1423,9 +1459,12 @@ public OpenCGAResult search(String studyId, Query query, QueryOptions opti public OpenCGAResult distinct(String studyId, List fields, Query query, String token) throws CatalogException { query = ParamUtils.defaultObject(query, Query::new); - String userId = userManager.getUserId(token); - Study study = catalogManager.getStudyManager().resolveId(studyId, userId, new QueryOptions(QueryOptions.INCLUDE, - StudyDBAdaptor.QueryParams.VARIABLE_SET.key())); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyId, new QueryOptions(QueryOptions.INCLUDE, + StudyDBAdaptor.QueryParams.VARIABLE_SET.key()), userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) @@ -1435,17 +1474,17 @@ public OpenCGAResult distinct(String studyId, List fields, Query quer try { fixQueryObject(study, query, userId); // Fix query if it contains any annotation - AnnotationUtils.fixQueryAnnotationSearch(study, query); + AnnotationUtils.fixQueryAnnotationSearch(organizationId, study, query); query.append(FileDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - OpenCGAResult result = fileDBAdaptor.distinct(study.getUid(), fields, query, userId); + OpenCGAResult result = getFileDBAdaptor(organizationId).distinct(study.getUid(), fields, query, userId); - auditManager.auditDistinct(userId, Enums.Resource.FILE, study.getId(), study.getUuid(), auditParams, + auditManager.auditDistinct(organizationId, userId, Enums.Resource.FILE, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return result; } catch (CatalogException e) { - auditManager.auditDistinct(userId, Enums.Resource.FILE, study.getId(), study.getUuid(), auditParams, + auditManager.auditDistinct(organizationId, userId, Enums.Resource.FILE, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -1501,9 +1540,12 @@ private void validateQueryPath(Query query, String key) { public OpenCGAResult count(String studyId, Query query, String token) throws CatalogException { query = ParamUtils.defaultObject(query, Query::new); - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyId, userId, new QueryOptions(QueryOptions.INCLUDE, - StudyDBAdaptor.QueryParams.VARIABLE_SET.key())); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyId, new QueryOptions(QueryOptions.INCLUDE, + StudyDBAdaptor.QueryParams.VARIABLE_SET.key()), userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) @@ -1511,20 +1553,20 @@ public OpenCGAResult count(String studyId, Query query, String token) thro .append("token", token); try { // Fix query if it contains any annotation - AnnotationUtils.fixQueryAnnotationSearch(study, query); + AnnotationUtils.fixQueryAnnotationSearch(organizationId, study, query); // The samples introduced could be either ids or names. As so, we should use the smart resolutor to do this. fixQueryObject(study, query, userId); query.append(FileDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - OpenCGAResult queryResultAux = fileDBAdaptor.count(query, userId); + OpenCGAResult queryResultAux = getFileDBAdaptor(organizationId).count(query, userId); - auditManager.auditCount(userId, Enums.Resource.FILE, study.getId(), study.getUuid(), auditParams, + auditManager.auditCount(organizationId, userId, Enums.Resource.FILE, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return new OpenCGAResult<>(queryResultAux.getTime(), queryResultAux.getEvents(), 0, Collections.emptyList(), queryResultAux.getNumMatches()); } catch (CatalogException e) { - auditManager.auditCount(userId, Enums.Resource.FILE, study.getId(), study.getUuid(), auditParams, + auditManager.auditCount(organizationId, userId, Enums.Resource.FILE, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -1537,9 +1579,12 @@ public OpenCGAResult delete(String studyStr, List fileIds, QueryOptions public OpenCGAResult delete(String studyStr, List fileIds, QueryOptions options, boolean ignoreException, String token) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(token); - Study study = studyManager.resolveId(studyStr, userId, new QueryOptions(QueryOptions.INCLUDE, - StudyDBAdaptor.QueryParams.VARIABLE_SET.key())); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, new QueryOptions(QueryOptions.INCLUDE, + StudyDBAdaptor.QueryParams.VARIABLE_SET.key()), userId, organizationId); String operationUuid = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -1563,7 +1608,7 @@ public OpenCGAResult delete(String studyStr, List fileIds, QueryOptions String fileUuid = ""; try { - OpenCGAResult internalResult = internalGet(study.getUid(), id, INCLUDE_FILE_URI_PATH, userId); + OpenCGAResult internalResult = internalGet(organizationId, study.getUid(), id, INCLUDE_FILE_URI_PATH, userId); if (internalResult.getNumResults() == 0) { throw new CatalogException("File '" + id + "' not found"); } @@ -1577,7 +1622,7 @@ public OpenCGAResult delete(String studyStr, List fileIds, QueryOptions continue; } - OpenCGAResult updateResult = delete(study, file, physicalDelete, userId); + OpenCGAResult updateResult = delete(organizationId, study, file, physicalDelete, userId); result.append(updateResult); // We store the processed path as is @@ -1585,19 +1630,19 @@ public OpenCGAResult delete(String studyStr, List fileIds, QueryOptions processedPaths.add(file.getPath()); } - auditManager.auditDelete(operationUuid, userId, Enums.Resource.FILE, file.getId(), file.getUuid(), study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.FILE, file.getId(), file.getUuid(), + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { Event event = new Event(Event.Type.ERROR, fileId, e.getMessage()); result.getEvents().add(event); result.setNumErrors(result.getNumErrors() + 1); logger.error("Could not delete file {}: {}", fileId, e.getMessage(), e); - auditManager.auditDelete(operationUuid, userId, Enums.Resource.FILE, fileId, fileUuid, study.getId(), + auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.FILE, fileId, fileUuid, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); } } - auditManager.finishAuditBatch(operationUuid); + auditManager.finishAuditBatch(organizationId, operationUuid); return endResult(result, ignoreException); } @@ -1614,9 +1659,12 @@ public OpenCGAResult delete(String studyStr, Query query, ObjectMap params, bool OpenCGAResult dataResult = OpenCGAResult.empty(); - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId, new QueryOptions(QueryOptions.INCLUDE, - StudyDBAdaptor.QueryParams.VARIABLE_SET.key())); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, new QueryOptions(QueryOptions.INCLUDE, + StudyDBAdaptor.QueryParams.VARIABLE_SET.key()), userId, organizationId); StopWatch watch = StopWatch.createStarted(); String operationUuid = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -1632,14 +1680,14 @@ public OpenCGAResult delete(String studyStr, Query query, ObjectMap params, bool DBIterator fileIterator; try { // Fix query if it contains any annotation - AnnotationUtils.fixQueryAnnotationSearch(study, finalQuery); + AnnotationUtils.fixQueryAnnotationSearch(organizationId, study, finalQuery); fixQueryObject(study, finalQuery, userId); finalQuery.append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - fileIterator = fileDBAdaptor.iterator(study.getUid(), finalQuery, INCLUDE_FILE_URI_PATH, userId); + fileIterator = getFileDBAdaptor(organizationId).iterator(study.getUid(), finalQuery, INCLUDE_FILE_URI_PATH, userId); } catch (CatalogException e) { - auditManager.auditDelete(operationUuid, userId, Enums.Resource.FILE, "", "", study.getId(), study.getUuid(), auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.FILE, "", "", study.getId(), study.getUuid(), + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); throw e; } @@ -1659,7 +1707,7 @@ public OpenCGAResult delete(String studyStr, Query query, ObjectMap params, bool } try { - OpenCGAResult result = delete(study, file, physicalDelete, userId); + OpenCGAResult result = delete(organizationId, study, file, physicalDelete, userId); dataResult.append(result); // We store the processed path as is @@ -1667,8 +1715,8 @@ public OpenCGAResult delete(String studyStr, Query query, ObjectMap params, bool processedPaths.add(file.getPath()); } - auditManager.auditDelete(operationUuid, userId, Enums.Resource.FILE, file.getId(), file.getUuid(), study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.FILE, file.getId(), file.getUuid(), + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { String errorMsg; @@ -1681,11 +1729,11 @@ public OpenCGAResult delete(String studyStr, Query query, ObjectMap params, bool dataResult.setNumErrors(dataResult.getNumErrors() + 1); logger.error(errorMsg, e); - auditManager.auditDelete(operationUuid, userId, Enums.Resource.FILE, file.getId(), file.getUuid(), study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.FILE, file.getId(), file.getUuid(), + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); } } - auditManager.finishAuditBatch(operationUuid); + auditManager.finishAuditBatch(organizationId, operationUuid); dataResult.setTime((int) watch.getTime(TimeUnit.MILLISECONDS)); dataResult.setNumMatches(dataResult.getNumMatches() + numMatches); @@ -1693,10 +1741,10 @@ public OpenCGAResult delete(String studyStr, Query query, ObjectMap params, bool return endResult(dataResult, ignoreException); } - private OpenCGAResult delete(Study study, File file, boolean physicalDelete, String userId) + private OpenCGAResult delete(String organizationId, Study study, File file, boolean physicalDelete, String userId) throws CatalogException { // Check if the file or the folder plus any nested files/folders can be deleted - checkCanDeleteFile(study, file.getPath(), false, Collections.singletonList(FileStatus.PENDING_DELETE), userId); + checkCanDeleteFile(organizationId, study, file.getPath(), false, Collections.singletonList(FileStatus.PENDING_DELETE), userId); String currentStatus = file.getInternal().getStatus().getId(); if (FileStatus.DELETED.equals(currentStatus)) { @@ -1710,9 +1758,9 @@ private OpenCGAResult delete(Study study, File file, boolean physicalDelete, Str } if (physicalDelete) { - return physicalDelete(study, file); + return physicalDelete(organizationId, study, file); } else { - return sendToTrash(file); + return sendToTrash(organizationId, file); } } @@ -1737,16 +1785,19 @@ public OpenCGAResult syncUntrackedFiles(String studyId, String folderId, P public OpenCGAResult syncUntrackedFiles(String studyId, String folderId, Predicate filter, String jobId, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyId, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyId, userId, organizationId); - File folder = internalGet(study.getUid(), folderId, INCLUDE_FILE_URI_PATH, userId).first(); + File folder = internalGet(organizationId, study.getUid(), folderId, INCLUDE_FILE_URI_PATH, userId).first(); if (folder.getType() == File.Type.FILE) { throw new CatalogException("Provided folder '" + folderId + "' is actually a file"); } - authorizationManager.checkFilePermission(study.getUid(), folder.getUid(), userId, FilePermissions.WRITE); + authorizationManager.checkFilePermission(organizationId, study.getUid(), folder.getUid(), userId, FilePermissions.WRITE); IOManager ioManager; try { @@ -1783,7 +1834,8 @@ public OpenCGAResult syncUntrackedFiles(String studyId, String folderId, P } try { - File registeredFile = internalGet(study.getUid(), finalCatalogPath, INCLUDE_FILE_URI_PATH, userId).first(); + File registeredFile = internalGet(organizationId, study.getUid(), finalCatalogPath, INCLUDE_FILE_URI_PATH, userId) + .first(); if (!registeredFile.getUri().equals(fileUri)) { eventList.add(new Event(Event.Type.WARNING, registeredFile.getPath(), "The uri registered in Catalog '" + registeredFile.getUri().getPath() + "' for the path does not match the uri that would have been synced '" @@ -1791,7 +1843,7 @@ public OpenCGAResult syncUntrackedFiles(String studyId, String folderId, P } fileList.add(registeredFile); } catch (CatalogException e) { - File file = registerFile(study, finalCatalogPath, fileUri, jobId, token).first(); + File file = registerFile(organizationId, study, finalCatalogPath, fileUri, jobId, tokenPayload).first(); result.setNumInserted(result.getNumInserted() + 1); fileList.add(file); @@ -1807,8 +1859,11 @@ public OpenCGAResult syncUntrackedFiles(String studyId, String folderId, P } public OpenCGAResult unlink(@Nullable String studyId, String fileId, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyId, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyId, userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("study", studyId) @@ -1818,22 +1873,22 @@ public OpenCGAResult unlink(@Nullable String studyId, String fileId, Strin try { ParamUtils.checkParameter(fileId, "File"); - File file = internalGet(study.getUid(), fileId, QueryOptions.empty(), userId).first(); + File file = internalGet(organizationId, study.getUid(), fileId, QueryOptions.empty(), userId).first(); if (!file.isExternal()) { throw new CatalogException("Only previously linked files can be unlinked. Please, use delete instead."); } // Check if the file or the folder plus any nested files/folders can be deleted - checkCanDeleteFile(study, file.getPath(), true, Collections.singletonList(FileStatus.PENDING_DELETE), userId); + checkCanDeleteFile(organizationId, study, file.getPath(), true, Collections.singletonList(FileStatus.PENDING_DELETE), userId); - OpenCGAResult result = unlink(file); - auditManager.audit(userId, Enums.Action.UNLINK, Enums.Resource.FILE, file.getId(), file.getUuid(), study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + OpenCGAResult result = unlink(organizationId, file); + auditManager.audit(organizationId, userId, Enums.Action.UNLINK, Enums.Resource.FILE, file.getId(), file.getUuid(), + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return result; } catch (Exception e) { - auditManager.audit(userId, Enums.Action.UNLINK, Enums.Resource.FILE, fileId, "", study.getId(), study.getUuid(), + auditManager.audit(organizationId, userId, Enums.Action.UNLINK, Enums.Resource.FILE, fileId, "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(0, "", e.getMessage()))); throw new CatalogException("Could not unlink file '" + fileId + "'", e); } @@ -1842,12 +1897,13 @@ public OpenCGAResult unlink(@Nullable String studyId, String fileId, Strin /** * Delete the file from the file system and from OpenCGA. * - * @param study Study object. - * @param file File or folder. + * @param organizationId + * @param study Study object. + * @param file File or folder. * @return a OpenCGAResult object. */ - private OpenCGAResult physicalDelete(Study study, File file) throws CatalogException { - URI fileUri = getUri(file); + private OpenCGAResult physicalDelete(String organizationId, Study study, File file) throws CatalogException { + URI fileUri = getUri(organizationId, file); IOManager ioManager = null; try { ioManager = ioManagerFactory.get(fileUri); @@ -1855,11 +1911,11 @@ private OpenCGAResult physicalDelete(Study study, File file) throws CatalogExcep throw CatalogIOException.ioManagerException(fileUri, e); } - OpenCGAResult result = OpenCGAResult.empty(); + OpenCGAResult result; if (file.getType() == File.Type.FILE) { // 1. Set the file status to deleting ObjectMap update = new ObjectMap(FileDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), FileStatus.DELETING); - fileDBAdaptor.update(file.getUid(), update, QueryOptions.empty()); + getFileDBAdaptor(organizationId).update(file.getUid(), update, QueryOptions.empty()); // 2. Delete file from the file system logger.debug("Deleting file '{} ({})' with uri '{}' from the file system", file.getPath(), file.getUid(), fileUri); @@ -1877,20 +1933,20 @@ private OpenCGAResult physicalDelete(Study study, File file) throws CatalogExcep } // 3. Delete file from the database - result = fileDBAdaptor.delete(file, FileStatus.DELETED); + result = getFileDBAdaptor(organizationId).delete(file, FileStatus.DELETED); } else { // 1. Set the folder and all nested files/folders to DELETING Query query = new Query() .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()) .append(FileDBAdaptor.QueryParams.PATH.key(), "~^" + file.getPath() + "*"); ObjectMap update = new ObjectMap(FileDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), FileStatus.DELETING); - fileDBAdaptor.update(query, update, QueryOptions.empty()); + getFileDBAdaptor(organizationId).update(query, update, QueryOptions.empty()); // 2. Delete files to be deleted from the file system QueryOptions options = new QueryOptions(INCLUDE_FILE_URI_PATH) .append(QueryOptions.SORT, FileDBAdaptor.QueryParams.PATH.key()) .append(QueryOptions.ORDER, QueryOptions.DESCENDING); - DBIterator iterator = fileDBAdaptor.iterator(query, options); + DBIterator iterator = getFileDBAdaptor(organizationId).iterator(query, options); while (iterator.hasNext()) { File tmpFile = iterator.next(); if (ioManager.isDirectory(tmpFile.getUri())) { @@ -1905,18 +1961,18 @@ private OpenCGAResult physicalDelete(Study study, File file) throws CatalogExcep } // 3. Delete the folder and all nested files/folders to DELETED - result = fileDBAdaptor.delete(file, FileStatus.DELETED); + result = getFileDBAdaptor(organizationId).delete(file, FileStatus.DELETED); } return result; } - private OpenCGAResult sendToTrash(File file) throws CatalogException { - return fileDBAdaptor.delete(file, FileStatus.TRASHED); + private OpenCGAResult sendToTrash(String organizationId, File file) throws CatalogException { + return getFileDBAdaptor(organizationId).delete(file, FileStatus.TRASHED); } - private OpenCGAResult unlink(File file) throws CatalogException { - return fileDBAdaptor.delete(file, FileStatus.REMOVED); + private OpenCGAResult unlink(String organizationId, File file) throws CatalogException { + return getFileDBAdaptor(organizationId).delete(file, FileStatus.REMOVED); } private boolean subpathInPath(String subpath, Set pathSet) { @@ -1931,9 +1987,9 @@ private boolean subpathInPath(String subpath, Set pathSet) { return false; } - public OpenCGAResult updateAnnotations(String studyStr, String fileStr, String annotationSetId, - Map annotations, ParamUtils.CompleteUpdateAction action, - QueryOptions options, String token) throws CatalogException { + public OpenCGAResult updateAnnotations(String studyStr, String fileStr, String annotationSetId, Map annotations, + ParamUtils.CompleteUpdateAction action, QueryOptions options, String token) + throws CatalogException { if (annotations == null || annotations.isEmpty()) { throw new CatalogException("Missing array of annotations."); } @@ -1945,16 +2001,16 @@ public OpenCGAResult updateAnnotations(String studyStr, String fileStr, St return update(studyStr, fileStr, updateParams, options, token); } - public OpenCGAResult removeAnnotations(String studyStr, String fileStr, String annotationSetId, - List annotations, QueryOptions options, String token) throws CatalogException { - return updateAnnotations(studyStr, fileStr, annotationSetId, new ObjectMap("remove", StringUtils.join(annotations, ",")), - ParamUtils.CompleteUpdateAction.REMOVE, options, token); + public OpenCGAResult removeAnnotations(String studyStr, String fileStr, String annotationSetId, List annotations, + QueryOptions options, String token) throws CatalogException { + return updateAnnotations(studyStr, fileStr, annotationSetId, + new ObjectMap("remove", StringUtils.join(annotations, ",")), ParamUtils.CompleteUpdateAction.REMOVE, options, token); } public OpenCGAResult resetAnnotations(String studyStr, String fileStr, String annotationSetId, List annotations, QueryOptions options, String token) throws CatalogException { - return updateAnnotations(studyStr, fileStr, annotationSetId, new ObjectMap("reset", StringUtils.join(annotations, ",")), - ParamUtils.CompleteUpdateAction.RESET, options, token); + return updateAnnotations(studyStr, fileStr, annotationSetId, + new ObjectMap("reset", StringUtils.join(annotations, ",")), ParamUtils.CompleteUpdateAction.RESET, options, token); } public OpenCGAResult update(String studyStr, Query query, FileUpdateParams updateParams, QueryOptions options, String token) @@ -1964,8 +2020,11 @@ public OpenCGAResult update(String studyStr, Query query, FileUpdateParams public OpenCGAResult update(String studyStr, Query query, FileUpdateParams updateParams, boolean ignoreException, QueryOptions options, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId, StudyManager.INCLUDE_VARIABLE_SET); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, StudyManager.INCLUDE_VARIABLE_SET, userId, organizationId); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -1989,13 +2048,13 @@ public OpenCGAResult update(String studyStr, Query query, FileUpdateParams DBIterator iterator; try { // Fix query if it contains any annotation - AnnotationUtils.fixQueryAnnotationSearch(study, finalQuery); + AnnotationUtils.fixQueryAnnotationSearch(organizationId, study, finalQuery); fixQueryObject(study, finalQuery, userId); finalQuery.append(FileDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - iterator = fileDBAdaptor.iterator(study.getUid(), finalQuery, EXCLUDE_FILE_ATTRIBUTES, userId); + iterator = getFileDBAdaptor(organizationId).iterator(study.getUid(), finalQuery, EXCLUDE_FILE_ATTRIBUTES, userId); } catch (CatalogException e) { - auditManager.auditUpdate(operationId, userId, Enums.Resource.FILE, "", "", study.getId(), study.getUuid(), + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.FILE, "", "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -2005,29 +2064,32 @@ public OpenCGAResult update(String studyStr, Query query, FileUpdateParams while (iterator.hasNext()) { File file = iterator.next(); try { - OpenCGAResult updateResult = update(study, file, updateParams, options, userId, token); + OpenCGAResult updateResult = update(organizationId, study, file, updateParams, options, userId, token); result.append(updateResult); - auditManager.auditUpdate(operationId, userId, Enums.Resource.FILE, file.getId(), file.getUuid(), study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.FILE, file.getId(), file.getUuid(), + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { Event event = new Event(Event.Type.ERROR, file.getId(), e.getMessage()); result.getEvents().add(event); result.setNumErrors(result.getNumErrors() + 1); logger.error("Cannot update file {}: {}", file.getId(), e.getMessage()); - auditManager.auditUpdate(operationId, userId, Enums.Resource.FILE, file.getId(), file.getUuid(), study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.FILE, file.getId(), file.getUuid(), + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); } } - auditManager.finishAuditBatch(operationId); + auditManager.finishAuditBatch(organizationId, operationId); return endResult(result, ignoreException); } public OpenCGAResult update(String studyStr, String fileId, FileUpdateParams updateParams, QueryOptions options, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId, StudyManager.INCLUDE_VARIABLE_SET); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, StudyManager.INCLUDE_VARIABLE_SET, userId, organizationId); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -2048,7 +2110,7 @@ public OpenCGAResult update(String studyStr, String fileId, FileUpdatePara OpenCGAResult result = OpenCGAResult.empty(); String fileUuid = ""; try { - OpenCGAResult internalResult = internalGet(study.getUid(), fileId, EXCLUDE_FILE_ATTRIBUTES, userId); + OpenCGAResult internalResult = internalGet(organizationId, study.getUid(), fileId, EXCLUDE_FILE_ATTRIBUTES, userId); if (internalResult.getNumResults() == 0) { throw new CatalogException("File '" + fileId + "' not found"); } @@ -2058,18 +2120,18 @@ public OpenCGAResult update(String studyStr, String fileId, FileUpdatePara fileId = file.getId(); fileUuid = file.getUuid(); - OpenCGAResult updateResult = update(study, file, updateParams, options, userId, token); + OpenCGAResult updateResult = update(organizationId, study, file, updateParams, options, userId, token); result.append(updateResult); - auditManager.auditUpdate(userId, Enums.Resource.FILE, file.getId(), file.getUuid(), study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.auditUpdate(organizationId, userId, Enums.Resource.FILE, file.getId(), file.getUuid(), study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { Event event = new Event(Event.Type.ERROR, fileId, e.getMessage()); result.getEvents().add(event); result.setNumErrors(result.getNumErrors() + 1); logger.error("Cannot update file {}: {}", fileId, e.getMessage()); - auditManager.auditUpdate(operationId, userId, Enums.Resource.FILE, fileId, fileUuid, study.getId(), + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.FILE, fileId, fileUuid, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -2079,7 +2141,8 @@ public OpenCGAResult update(String studyStr, String fileId, FileUpdatePara /** * Update a File from catalog. * - * @param studyStr Study id in string format. Could be one of [id|user@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy]. + * @param studyStr Study id in string format. Could be one of + * [id|organization@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy] * @param fileIds List of file ids. Could be either the id, path or uuid. * @param updateParams Data model filled only with the parameters to be updated. * @param options QueryOptions object. @@ -2095,8 +2158,11 @@ public OpenCGAResult update(String studyStr, List fileIds, FileUpd public OpenCGAResult update(String studyStr, List fileIds, FileUpdateParams updateParams, boolean ignoreException, QueryOptions options, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId, StudyManager.INCLUDE_VARIABLE_SET); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, StudyManager.INCLUDE_VARIABLE_SET, userId, organizationId); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -2122,7 +2188,7 @@ public OpenCGAResult update(String studyStr, List fileIds, FileUpd String fileUuid = ""; try { - OpenCGAResult internalResult = internalGet(study.getUid(), fileId, EXCLUDE_FILE_ATTRIBUTES, userId); + OpenCGAResult internalResult = internalGet(organizationId, study.getUid(), fileId, EXCLUDE_FILE_ATTRIBUTES, userId); if (internalResult.getNumResults() == 0) { throw new CatalogException("File '" + id + "' not found"); } @@ -2132,27 +2198,27 @@ public OpenCGAResult update(String studyStr, List fileIds, FileUpd fileId = file.getId(); fileUuid = file.getUuid(); - OpenCGAResult updateResult = update(study, file, updateParams, options, userId, token); + OpenCGAResult updateResult = update(organizationId, study, file, updateParams, options, userId, token); result.append(updateResult); - auditManager.auditUpdate(userId, Enums.Resource.FILE, file.getId(), file.getUuid(), study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.auditUpdate(organizationId, userId, Enums.Resource.FILE, file.getId(), file.getUuid(), study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { Event event = new Event(Event.Type.ERROR, id, e.getMessage()); result.getEvents().add(event); result.setNumErrors(result.getNumErrors() + 1); logger.error("Cannot update file {}: {}", fileId, e.getMessage()); - auditManager.auditUpdate(operationId, userId, Enums.Resource.FILE, fileId, fileUuid, study.getId(), + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.FILE, fileId, fileUuid, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); } } - auditManager.finishAuditBatch(operationId); + auditManager.finishAuditBatch(organizationId, operationId); return endResult(result, ignoreException); } - private OpenCGAResult update(Study study, File file, FileUpdateParams updateParams, QueryOptions options, String userId, - String token) throws CatalogException { + private OpenCGAResult update(String organizationId, Study study, File file, FileUpdateParams updateParams, QueryOptions options, + String userId, String token) throws CatalogException { ObjectMap parameters = new ObjectMap(); if (updateParams != null) { try { @@ -2190,7 +2256,8 @@ private OpenCGAResult update(Study study, File file, FileUpdateParams upda if (StringUtils.isEmpty(relatedFile.getFile()) || relatedFile.getRelation() == null) { throw new CatalogException("Missing file or relation in relatedFiles list"); } - File relatedFileFile = internalGet(study.getUid(), relatedFile.getFile(), null, INCLUDE_FILE_URI_PATH, userId).first(); + File relatedFileFile = internalGet(organizationId, study.getUid(), relatedFile.getFile(), null, INCLUDE_FILE_URI_PATH, + userId).first(); relatedFileList.add(new FileRelatedFile(relatedFileFile, relatedFile.getRelation())); } parameters.put(FileDBAdaptor.QueryParams.RELATED_FILES.key(), relatedFileList); @@ -2206,13 +2273,13 @@ private OpenCGAResult update(Study study, File file, FileUpdateParams upda // Check permissions... // Only check write annotation permissions if the user wants to update the annotation sets if (updateParams != null && updateParams.getAnnotationSets() != null) { - authorizationManager.checkFilePermission(study.getUid(), file.getUid(), userId, + authorizationManager.checkFilePermission(organizationId, study.getUid(), file.getUid(), userId, FilePermissions.WRITE_ANNOTATIONS); } // Only check update permissions if the user wants to update anything apart from the annotation sets if ((parameters.size() == 1 && !parameters.containsKey(FileDBAdaptor.QueryParams.ANNOTATION_SETS.key())) || parameters.size() > 1) { - authorizationManager.checkFilePermission(study.getUid(), file.getUid(), userId, FilePermissions.WRITE); + authorizationManager.checkFilePermission(organizationId, study.getUid(), file.getUid(), userId, FilePermissions.WRITE); } if (isRootFolder(file)) { @@ -2221,17 +2288,18 @@ private OpenCGAResult update(Study study, File file, FileUpdateParams upda // We make a query to check both if the samples exists and if the user has permissions to see them if (updateParams != null && ListUtils.isNotEmpty(updateParams.getSampleIds())) { - catalogManager.getSampleManager().internalGet(study.getUid(), updateParams.getSampleIds(), SampleManager.INCLUDE_SAMPLE_IDS, - userId, false); + catalogManager.getSampleManager().internalGet(organizationId, study.getUid(), updateParams.getSampleIds(), + SampleManager.INCLUDE_SAMPLE_IDS, userId, false); } - checkUpdateAnnotations(study, file, parameters, options, VariableSet.AnnotableDataModels.FILE, fileDBAdaptor, userId); + checkUpdateAnnotations(organizationId, study, file, parameters, options, VariableSet.AnnotableDataModels.FILE, + getFileDBAdaptor(organizationId), userId); - OpenCGAResult update = fileDBAdaptor.update(file.getUid(), parameters, study.getVariableSets(), options); + OpenCGAResult update = getFileDBAdaptor(organizationId).update(file.getUid(), parameters, study.getVariableSets(), options); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { // Fetch updated file - OpenCGAResult result = fileDBAdaptor.get(study.getUid(), new Query(FileDBAdaptor.QueryParams.UID.key(), file.getUid()), - options, userId); + OpenCGAResult result = getFileDBAdaptor(organizationId).get(study.getUid(), new Query(FileDBAdaptor.QueryParams.UID.key(), + file.getUid()), options, userId); update.setResults(result.getResults()); } return update; @@ -2243,10 +2311,13 @@ public OpenCGAResult update(String studyStr, String entryStr, ObjectMap pa ParamUtils.checkObj(parameters, "Parameters"); options = ParamUtils.defaultObject(options, QueryOptions::new); - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId, StudyManager.INCLUDE_VARIABLE_SET); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, StudyManager.INCLUDE_VARIABLE_SET, userId, organizationId); - File file = internalGet(study.getUid(), entryStr, QueryOptions.empty(), userId).first(); + File file = internalGet(organizationId, study.getUid(), entryStr, QueryOptions.empty(), userId).first(); ObjectMap auditParams = new ObjectMap() .append("study", studyStr) @@ -2258,13 +2329,13 @@ public OpenCGAResult update(String studyStr, String entryStr, ObjectMap pa // Check permissions... // Only check write annotation permissions if the user wants to update the annotation sets if (parameters.containsKey(FileDBAdaptor.QueryParams.ANNOTATION_SETS.key())) { - authorizationManager.checkFilePermission(study.getUid(), file.getUid(), userId, + authorizationManager.checkFilePermission(organizationId, study.getUid(), file.getUid(), userId, FilePermissions.WRITE_ANNOTATIONS); } // Only check update permissions if the user wants to update anything apart from the annotation sets if ((parameters.size() == 1 && !parameters.containsKey(FileDBAdaptor.QueryParams.ANNOTATION_SETS.key())) || parameters.size() > 1) { - authorizationManager.checkFilePermission(study.getUid(), file.getUid(), userId, FilePermissions.WRITE); + authorizationManager.checkFilePermission(organizationId, study.getUid(), file.getUid(), userId, FilePermissions.WRITE); } try { @@ -2277,22 +2348,23 @@ public OpenCGAResult update(String studyStr, String entryStr, ObjectMap pa if (parameters.get(FileDBAdaptor.QueryParams.SAMPLE_IDS.key()) != null && ListUtils.isNotEmpty(parameters.getAsStringList(FileDBAdaptor.QueryParams.SAMPLE_IDS.key()))) { List sampleIds = parameters.getAsStringList(FileDBAdaptor.QueryParams.SAMPLE_IDS.key()); - catalogManager.getSampleManager().internalGet(study.getUid(), sampleIds, SampleManager.INCLUDE_SAMPLE_IDS, userId, false); + catalogManager.getSampleManager().internalGet(organizationId, study.getUid(), sampleIds, SampleManager.INCLUDE_SAMPLE_IDS, + userId, false); } - OpenCGAResult queryResult = unsafeUpdate(study, file, parameters, options, userId); - auditManager.auditUpdate(userId, Enums.Resource.FILE, file.getId(), file.getUuid(), study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + OpenCGAResult queryResult = unsafeUpdate(organizationId, study, file, parameters, options, userId); + auditManager.auditUpdate(organizationId, userId, Enums.Resource.FILE, file.getId(), file.getUuid(), study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return queryResult; } catch (CatalogException e) { - auditManager.auditUpdate(userId, Enums.Resource.FILE, file.getId(), file.getUuid(), study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditUpdate(organizationId, userId, Enums.Resource.FILE, file.getId(), file.getUuid(), study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } - OpenCGAResult unsafeUpdate(Study study, File file, ObjectMap parameters, QueryOptions options, String userId) - throws CatalogException { + OpenCGAResult unsafeUpdate(String organizationId, Study study, File file, ObjectMap parameters, QueryOptions options, + String userId) throws CatalogException { if (isRootFolder(file)) { throw new CatalogException("Cannot modify root folder"); } @@ -2303,16 +2375,17 @@ OpenCGAResult unsafeUpdate(Study study, File file, ObjectMap parameters, Q throw new CatalogException("Could not update: " + e.getMessage(), e); } - checkUpdateAnnotations(study, file, parameters, options, VariableSet.AnnotableDataModels.FILE, fileDBAdaptor, userId); + checkUpdateAnnotations(organizationId, study, file, parameters, options, VariableSet.AnnotableDataModels.FILE, + getFileDBAdaptor(organizationId), userId); - fileDBAdaptor.update(file.getUid(), parameters, study.getVariableSets(), options); - return fileDBAdaptor.get(file.getUid(), options); + getFileDBAdaptor(organizationId).update(file.getUid(), parameters, study.getVariableSets(), options); + return getFileDBAdaptor(organizationId).get(file.getUid(), options); } public OpenCGAResult move(String studyStr, String entryStr, String targetPathStr, QueryOptions options, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); ObjectMap auditParams = new ObjectMap() .append("study", studyStr) @@ -2321,35 +2394,44 @@ public OpenCGAResult move(String studyStr, String entryStr, String targetP .append("options", options) .append("token", token); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + String studyId = studyFqn.getStudyId(); + String studyUuid = studyFqn.getStudyUuid(); String fileId = entryStr; String fileUuid = ""; try { - File file = internalGet(study.getUid(), entryStr, QueryOptions.empty(), userId).first(); + Study study = catalogManager.getStudyManager().resolveId(studyFqn, QueryOptions.empty(), tokenPayload); + studyId = study.getId(); + studyUuid = study.getUuid(); + + File file = internalGet(organizationId, study.getUid(), entryStr, QueryOptions.empty(), userId).first(); fileId = file.getId(); fileUuid = file.getUuid(); // Check user has write permissions on file/folder - authorizationManager.checkFilePermission(study.getUid(), file.getUid(), userId, FilePermissions.WRITE); + authorizationManager.checkFilePermission(organizationId, study.getUid(), file.getUid(), userId, FilePermissions.WRITE); - OpenCGAResult parents = getParents(study.getUid(), targetPathStr, false, INCLUDE_FILE_IDS); + OpenCGAResult parents = getParents(organizationId, study.getUid(), targetPathStr, false, INCLUDE_FILE_IDS); // Check user can write in target path File parentFolder = parents.first(); - authorizationManager.checkFilePermission(study.getUid(), parentFolder.getUid(), userId, FilePermissions.WRITE); + authorizationManager.checkFilePermission(organizationId, study.getUid(), parentFolder.getUid(), userId, FilePermissions.WRITE); ObjectMap parameters = new ObjectMap(FileDBAdaptor.QueryParams.PATH.key(), targetPathStr); - OpenCGAResult update = fileDBAdaptor.update(file.getUid(), parameters, Collections.emptyList(), QueryOptions.empty()); + OpenCGAResult update = getFileDBAdaptor(organizationId) + .update(file.getUid(), parameters, Collections.emptyList(), QueryOptions.empty()); - auditManager.audit(userId, Enums.Action.MOVE, Enums.Resource.FILE, file.getId(), file.getUuid(), study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.audit(organizationId, userId, Enums.Action.MOVE, Enums.Resource.FILE, file.getId(), file.getUuid(), studyId, + studyUuid, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { // Fetch updated file - OpenCGAResult result = fileDBAdaptor.get(study.getUid(), + OpenCGAResult result = getFileDBAdaptor(organizationId).get(study.getUid(), new Query(FileDBAdaptor.QueryParams.UID.key(), file.getUid()), options, userId); update.setResults(result.getResults()); } return update; } catch (Exception e) { - auditManager.audit(userId, Enums.Action.MOVE, Enums.Resource.FILE, fileId, fileUuid, study.getId(), study.getUuid(), + auditManager.audit(organizationId, userId, Enums.Action.MOVE, Enums.Resource.FILE, fileId, fileUuid, studyId, studyUuid, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(0, fileId, e.getMessage()))); throw e; } @@ -2358,8 +2440,11 @@ public OpenCGAResult move(String studyStr, String entryStr, String targetP public OpenCGAResult link(String studyStr, FileLinkParams params, boolean parents, String token) throws CatalogException { // We make two attempts to link to ensure the behaviour remains even if it is being called at the same time link from different // threads - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("study", studyStr) @@ -2367,20 +2452,20 @@ public OpenCGAResult link(String studyStr, FileLinkParams params, boolean .append("parents", parents) .append("token", token); try { - OpenCGAResult result = privateLink(study, params, parents, token); - auditManager.auditCreate(userId, Enums.Action.LINK, Enums.Resource.FILE, result.first().getId(), + OpenCGAResult result = privateLink(organizationId, study, params, parents, tokenPayload); + auditManager.auditCreate(organizationId, userId, Enums.Action.LINK, Enums.Resource.FILE, result.first().getId(), result.first().getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return result; } catch (Exception e) { try { - OpenCGAResult result = privateLink(study, params, parents, token); - auditManager.auditCreate(userId, Enums.Action.LINK, Enums.Resource.FILE, result.first().getId(), + OpenCGAResult result = privateLink(organizationId, study, params, parents, tokenPayload); + auditManager.auditCreate(organizationId, userId, Enums.Action.LINK, Enums.Resource.FILE, result.first().getId(), result.first().getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return result; } catch (Exception e2) { - auditManager.auditCreate(userId, Enums.Action.LINK, Enums.Resource.FILE, params.getUri(), "", + auditManager.auditCreate(organizationId, userId, Enums.Action.LINK, Enums.Resource.FILE, params.getUri(), "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(0, "", e2.getMessage()))); e.addSuppressed(e2); @@ -2408,10 +2493,13 @@ public OpenCGAResult rank(String studyStr, Query query, String field, int numRes ParamUtils.checkObj(field, "field"); ParamUtils.checkObj(sessionId, "sessionId"); - String userId = userManager.getUserId(sessionId); - Study study = studyManager.resolveId(studyStr, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(sessionId); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, userId, organizationId); - authorizationManager.checkStudyPermission(study.getUid(), userId, StudyPermissions.Permissions.VIEW_FILES); + authorizationManager.checkStudyPermission(organizationId, study.getUid(), userId, StudyPermissions.Permissions.VIEW_FILES); // TODO: In next release, we will have to check the count parameter from the queryOptions object. boolean count = true; @@ -2419,7 +2507,7 @@ public OpenCGAResult rank(String studyStr, Query query, String field, int numRes OpenCGAResult queryResult = null; if (count) { // We do not need to check for permissions when we show the count of files - queryResult = fileDBAdaptor.rank(query, field, numResults, asc); + queryResult = getFileDBAdaptor(organizationId).rank(query, field, numResults, asc); } return ParamUtils.defaultObject(queryResult, OpenCGAResult::new); @@ -2434,8 +2522,11 @@ public OpenCGAResult groupBy(@Nullable String studyStr, Query query, List grep(String studyId, String fileId, String pat throws CatalogException { long startTime = System.currentTimeMillis(); - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyId, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyId, userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) @@ -2463,10 +2557,10 @@ public OpenCGAResult grep(String studyId, String fileId, String pat .append("numLines", numLines) .append("token", token); try { - File file = internalGet(study.getUid(), fileId, INCLUDE_FILE_URI, userId).first(); - authorizationManager.checkFilePermission(study.getUid(), file.getUid(), userId, FilePermissions.VIEW_CONTENT); + File file = internalGet(organizationId, study.getUid(), fileId, INCLUDE_FILE_URI, userId).first(); + authorizationManager.checkFilePermission(organizationId, study.getUid(), file.getUid(), userId, FilePermissions.VIEW_CONTENT); - URI fileUri = getUri(file); + URI fileUri = getUri(organizationId, file); FileContent fileContent; try { fileContent = ioManagerFactory.get(fileUri).grep(Paths.get(fileUri), pattern, numLines, ignoreCase); @@ -2474,21 +2568,24 @@ public OpenCGAResult grep(String studyId, String fileId, String pat throw CatalogIOException.ioManagerException(fileUri, e); } - auditManager.audit(userId, Enums.Action.GREP, Enums.Resource.FILE, file.getId(), file.getUuid(), study.getId(), + auditManager.audit(organizationId, userId, Enums.Action.GREP, Enums.Resource.FILE, file.getId(), file.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return new OpenCGAResult<>((int) (System.currentTimeMillis() - startTime), Collections.emptyList(), 1, Collections.singletonList(fileContent), 1); } catch (CatalogException e) { - auditManager.audit(userId, Enums.Action.GREP, Enums.Resource.FILE, fileId, "", study.getId(), study.getUuid(), + auditManager.audit(organizationId, userId, Enums.Action.GREP, Enums.Resource.FILE, fileId, "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } public OpenCGAResult image(String studyStr, String fileId, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, userId, organizationId); long startTime = System.currentTimeMillis(); @@ -2498,14 +2595,14 @@ public OpenCGAResult image(String studyStr, String fileId, String t .append("token", token); File file; try { - file = internalGet(study.getUid(), fileId, INCLUDE_FILE_URI_PATH, userId).first(); - authorizationManager.checkFilePermission(study.getUid(), file.getUid(), userId, FilePermissions.VIEW_CONTENT); + file = internalGet(organizationId, study.getUid(), fileId, INCLUDE_FILE_URI_PATH, userId).first(); + authorizationManager.checkFilePermission(organizationId, study.getUid(), file.getUid(), userId, FilePermissions.VIEW_CONTENT); if (file.getFormat() != File.Format.IMAGE) { throw new CatalogException("File '" + fileId + "' is not an image. Format of file is '" + file.getFormat() + "'."); } - URI fileUri = getUri(file); + URI fileUri = getUri(organizationId, file); FileContent fileContent; try { @@ -2513,21 +2610,25 @@ public OpenCGAResult image(String studyStr, String fileId, String t } catch (IOException e) { throw CatalogIOException.ioManagerException(fileUri, e); } - auditManager.audit(userId, Enums.Action.IMAGE_CONTENT, Enums.Resource.FILE, file.getId(), file.getUuid(), study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.audit(organizationId, userId, Enums.Action.IMAGE_CONTENT, Enums.Resource.FILE, file.getId(), file.getUuid(), + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return new OpenCGAResult<>((int) (System.currentTimeMillis() - startTime), Collections.emptyList(), 1, Collections.singletonList(fileContent), 1); } catch (CatalogException e) { - auditManager.audit(userId, Enums.Action.IMAGE_CONTENT, Enums.Resource.FILE, fileId, "", study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.audit(organizationId, userId, Enums.Action.IMAGE_CONTENT, Enums.Resource.FILE, fileId, "", study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } - public OpenCGAResult head(String studyStr, String fileId, long offset, int lines, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId); + public OpenCGAResult head(String studyStr, String fileId, long offset, int lines, String token) + throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, userId, organizationId); long startTime = System.currentTimeMillis(); @@ -2539,30 +2640,34 @@ public OpenCGAResult head(String studyStr, String fileId, long offs .append("token", token); File file; try { - file = internalGet(study.getUid(), fileId, INCLUDE_FILE_URI, userId).first(); - authorizationManager.checkFilePermission(study.getUid(), file.getUid(), userId, FilePermissions.VIEW_CONTENT); - URI fileUri = getUri(file); + file = internalGet(organizationId, study.getUid(), fileId, INCLUDE_FILE_URI, userId).first(); + authorizationManager.checkFilePermission(organizationId, study.getUid(), file.getUid(), userId, FilePermissions.VIEW_CONTENT); + URI fileUri = getUri(organizationId, file); FileContent fileContent; try { fileContent = ioManagerFactory.get(fileUri).head(Paths.get(fileUri), offset, lines); } catch (IOException e) { throw CatalogIOException.ioManagerException(fileUri, e); } - auditManager.audit(userId, Enums.Action.HEAD_CONTENT, Enums.Resource.FILE, file.getId(), file.getUuid(), study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.audit(organizationId, userId, Enums.Action.HEAD_CONTENT, Enums.Resource.FILE, file.getId(), file.getUuid(), + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return new OpenCGAResult<>((int) (System.currentTimeMillis() - startTime), Collections.emptyList(), 1, Collections.singletonList(fileContent), 1); } catch (CatalogException e) { - auditManager.audit(userId, Enums.Action.HEAD_CONTENT, Enums.Resource.FILE, fileId, "", study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.audit(organizationId, userId, Enums.Action.HEAD_CONTENT, Enums.Resource.FILE, fileId, "", study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } - public OpenCGAResult tail(String studyStr, String fileId, int lines, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId); + public OpenCGAResult tail(String studyStr, String fileId, int lines, String token) + throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, userId, organizationId); long startTime = System.currentTimeMillis(); @@ -2573,23 +2678,23 @@ public OpenCGAResult tail(String studyStr, String fileId, int lines .append("token", token); File file; try { - file = internalGet(study.getUid(), fileId, INCLUDE_FILE_URI, userId).first(); - authorizationManager.checkFilePermission(study.getUid(), file.getUid(), userId, FilePermissions.VIEW_CONTENT); - URI fileUri = getUri(file); + file = internalGet(organizationId, study.getUid(), fileId, INCLUDE_FILE_URI, userId).first(); + authorizationManager.checkFilePermission(organizationId, study.getUid(), file.getUid(), userId, FilePermissions.VIEW_CONTENT); + URI fileUri = getUri(organizationId, file); FileContent fileContent; try { fileContent = ioManagerFactory.get(fileUri).tail(Paths.get(fileUri), lines); } catch (IOException e) { throw CatalogIOException.ioManagerException(fileUri, e); } - auditManager.audit(userId, Enums.Action.TAIL_CONTENT, Enums.Resource.FILE, file.getId(), file.getUuid(), study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.audit(organizationId, userId, Enums.Action.TAIL_CONTENT, Enums.Resource.FILE, file.getId(), file.getUuid(), + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return new OpenCGAResult<>((int) (System.currentTimeMillis() - startTime), Collections.emptyList(), 1, Collections.singletonList(fileContent), 1); } catch (CatalogException e) { - auditManager.audit(userId, Enums.Action.TAIL_CONTENT, Enums.Resource.FILE, fileId, "", study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.audit(organizationId, userId, Enums.Action.TAIL_CONTENT, Enums.Resource.FILE, fileId, "", study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } @@ -2599,8 +2704,11 @@ public DataInputStream download(String studyStr, String fileId, String token) th } public DataInputStream download(String studyStr, String fileId, int start, int limit, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("study", studyStr) @@ -2610,9 +2718,9 @@ public DataInputStream download(String studyStr, String fileId, int start, int l .append("token", token); File file; try { - file = internalGet(study.getUid(), fileId, INCLUDE_FILE_URI, userId).first(); - authorizationManager.checkFilePermission(study.getUid(), file.getUid(), userId, FilePermissions.DOWNLOAD); - URI fileUri = getUri(file); + file = internalGet(organizationId, study.getUid(), fileId, INCLUDE_FILE_URI, userId).first(); + authorizationManager.checkFilePermission(organizationId, study.getUid(), file.getUid(), userId, FilePermissions.DOWNLOAD); + URI fileUri = getUri(organizationId, file); DataInputStream dataInputStream; try { dataInputStream = ioManagerFactory.get(fileUri).getFileObject(fileUri, start, limit); @@ -2620,12 +2728,12 @@ public DataInputStream download(String studyStr, String fileId, int start, int l throw CatalogIOException.ioManagerException(fileUri, e); } - auditManager.audit(userId, Enums.Action.DOWNLOAD, Enums.Resource.FILE, file.getId(), file.getUuid(), study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.audit(organizationId, userId, Enums.Action.DOWNLOAD, Enums.Resource.FILE, file.getId(), file.getUuid(), + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return dataInputStream; } catch (CatalogException e) { - auditManager.audit(userId, Enums.Action.DOWNLOAD, Enums.Resource.FILE, fileId, "", study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.audit(organizationId, userId, Enums.Action.DOWNLOAD, Enums.Resource.FILE, fileId, "", study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } @@ -2633,14 +2741,17 @@ public DataInputStream download(String studyStr, String fileId, int start, int l // ************************** ACLs ******************************** // public OpenCGAResult> getAcls(String studyId, List fileList, String member, boolean ignoreException, String token) throws CatalogException { - return getAcls(studyId, fileList, StringUtils.isNotEmpty(member) ? Collections.singletonList(member) : Collections.emptyList(), - ignoreException, token); + return getAcls(studyId, fileList, + StringUtils.isNotEmpty(member) ? Collections.singletonList(member) : Collections.emptyList(), ignoreException, token); } public OpenCGAResult> getAcls(String studyId, List fileList, List members, boolean ignoreException, String token) throws CatalogException { - String user = userManager.getUserId(token); - Study study = studyManager.resolveId(studyId, user); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyId, userId, organizationId); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); ObjectMap auditParams = new ObjectMap() @@ -2654,7 +2765,8 @@ public OpenCGAResult> getAcls(String studyId, List Map missingMap = new HashMap<>(); try { auditManager.initAuditBatch(operationId); - InternalGetDataResult queryResult = internalGet(study.getUid(), fileList, INCLUDE_FILE_IDS, user, ignoreException); + InternalGetDataResult queryResult = internalGet(organizationId, study.getUid(), fileList, INCLUDE_FILE_IDS, userId, + ignoreException); if (queryResult.getMissing() != null) { missingMap = queryResult.getMissing().stream() @@ -2662,10 +2774,12 @@ public OpenCGAResult> getAcls(String studyId, List } List fileUids = queryResult.getResults().stream().map(File::getUid).collect(Collectors.toList()); - if (org.apache.commons.collections4.CollectionUtils.isNotEmpty(members)) { - fileAcls = authorizationManager.getAcl(user, study.getUid(), fileUids, members, Enums.Resource.FILE, FilePermissions.class); + if (CollectionUtils.isNotEmpty(members)) { + fileAcls = authorizationManager.getAcl(organizationId, study.getUid(), fileUids, members, Enums.Resource.FILE, + FilePermissions.class, userId); } else { - fileAcls = authorizationManager.getAcl(user, study.getUid(), fileUids, Enums.Resource.FILE, FilePermissions.class); + fileAcls = authorizationManager.getAcl(organizationId, study.getUid(), fileUids, Enums.Resource.FILE, FilePermissions.class, + userId); } // Include non-existing samples to the result list @@ -2676,14 +2790,14 @@ public OpenCGAResult> getAcls(String studyId, List if (!missingMap.containsKey(fileId)) { File file = queryResult.getResults().get(counter); resultList.add(fileAcls.getResults().get(counter)); - auditManager.audit(operationId, user, Enums.Action.FETCH_ACLS, Enums.Resource.FILE, file.getId(), + auditManager.audit(organizationId, operationId, userId, Enums.Action.FETCH_ACLS, Enums.Resource.FILE, file.getId(), file.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS), new ObjectMap()); counter++; } else { resultList.add(new AclEntryList<>()); eventList.add(new Event(Event.Type.ERROR, fileId, missingMap.get(fileId).getErrorMsg())); - auditManager.audit(operationId, user, Enums.Action.FETCH_ACLS, Enums.Resource.FILE, fileId, "", + auditManager.audit(organizationId, operationId, userId, Enums.Action.FETCH_ACLS, Enums.Resource.FILE, fileId, "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(0, "", missingMap.get(fileId).getErrorMsg())), new ObjectMap()); } @@ -2695,7 +2809,7 @@ public OpenCGAResult> getAcls(String studyId, List fileAcls.setEvents(eventList); } catch (CatalogException e) { for (String fileId : fileList) { - auditManager.audit(operationId, user, Enums.Action.FETCH_ACLS, Enums.Resource.FILE, fileId, "", + auditManager.audit(organizationId, operationId, userId, Enums.Action.FETCH_ACLS, Enums.Resource.FILE, fileId, "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError()), new ObjectMap()); } @@ -2708,18 +2822,20 @@ public OpenCGAResult> getAcls(String studyId, List } } } finally { - auditManager.finishAuditBatch(operationId); + auditManager.finishAuditBatch(organizationId, operationId); } return fileAcls; } public OpenCGAResult> updateAcl(String studyId, List fileStrList, String memberList, - FileAclParams aclParams, ParamUtils.AclAction action, - String token) + FileAclParams aclParams, ParamUtils.AclAction action, String token) throws CatalogException { - String user = userManager.getUserId(token); - Study study = studyManager.resolveId(studyId, user); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyId, userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) @@ -2756,20 +2872,22 @@ public OpenCGAResult> updateAcl(String studyId, Li List extendedFileList; if (StringUtils.isNotEmpty(aclParams.getSample())) { // Obtain the sample ids - OpenCGAResult sampleDataResult = catalogManager.getSampleManager().internalGet(study.getUid(), - Arrays.asList(StringUtils.split(aclParams.getSample(), ",")), SampleManager.INCLUDE_SAMPLE_IDS, user, false); + OpenCGAResult sampleDataResult = catalogManager.getSampleManager().internalGet(organizationId, study.getUid(), + Arrays.asList(StringUtils.split(aclParams.getSample(), ",")), SampleManager.INCLUDE_SAMPLE_IDS, userId, false); Query query = new Query(FileDBAdaptor.QueryParams.SAMPLE_IDS.key(), sampleDataResult.getResults().stream().map(Sample::getId).collect(Collectors.toList())); - extendedFileList = catalogManager.getFileManager().search(studyId, query, EXCLUDE_FILE_ATTRIBUTES, token).getResults(); + extendedFileList = catalogManager.getFileManager().search(studyId, query, EXCLUDE_FILE_ATTRIBUTES, token) + .getResults(); } else { - extendedFileList = internalGet(study.getUid(), fileStrList, EXCLUDE_FILE_ATTRIBUTES, user, false).getResults(); + extendedFileList = internalGet(organizationId, study.getUid(), fileStrList, EXCLUDE_FILE_ATTRIBUTES, userId, false) + .getResults(); } - authorizationManager.checkCanAssignOrSeePermissions(study.getUid(), user); + authorizationManager.checkCanAssignOrSeePermissions(organizationId, study.getUid(), userId); // Increase the list with the files/folders within the list of ids that correspond with folders - extendedFileList = getRecursiveFilesAndFolders(study.getUid(), extendedFileList); + extendedFileList = getRecursiveFilesAndFolders(organizationId, study.getUid(), extendedFileList); // Validate that the members are actually valid members List members; @@ -2779,7 +2897,7 @@ public OpenCGAResult> updateAcl(String studyId, Li members = Collections.emptyList(); } authorizationManager.checkNotAssigningPermissionsToAdminsGroup(members); - checkMembers(study.getUid(), members); + checkMembers(organizationId, study.getUid(), members); List fileUids = extendedFileList.stream().map(File::getUid).collect(Collectors.toList()); AuthorizationManager.CatalogAclParams catalogAclParams = new AuthorizationManager.CatalogAclParams(fileUids, permissions, @@ -2789,29 +2907,29 @@ public OpenCGAResult> updateAcl(String studyId, Li OpenCGAResult> queryResultList; switch (action) { case SET: - authorizationManager.setAcls(study.getUid(), members, catalogAclParams); + authorizationManager.setAcls(organizationId, study.getUid(), members, catalogAclParams); break; case ADD: - authorizationManager.addAcls(study.getUid(), members, catalogAclParams); + authorizationManager.addAcls(organizationId, study.getUid(), members, catalogAclParams); break; case REMOVE: - authorizationManager.removeAcls(members, catalogAclParams); + authorizationManager.removeAcls(organizationId, members, catalogAclParams); break; case RESET: catalogAclParams.setPermissions(null); - authorizationManager.removeAcls(members, catalogAclParams); + authorizationManager.removeAcls(organizationId, members, catalogAclParams); break; default: throw new CatalogException("Unexpected error occurred. No valid action found."); } - queryResultList = authorizationManager.getAcls(study.getUid(), fileUids, members, Enums.Resource.FILE, + queryResultList = authorizationManager.getAcls(organizationId, study.getUid(), fileUids, members, Enums.Resource.FILE, FilePermissions.class); for (int i = 0; i < queryResultList.getResults().size(); i++) { queryResultList.getResults().get(i).setId(extendedFileList.get(i).getId()); } for (File file : extendedFileList) { - auditManager.audit(operationId, user, Enums.Action.UPDATE_ACLS, Enums.Resource.FILE, file.getId(), + auditManager.audit(organizationId, operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.FILE, file.getId(), file.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS), new ObjectMap()); } @@ -2819,27 +2937,30 @@ public OpenCGAResult> updateAcl(String studyId, Li } catch (CatalogException e) { if (fileStrList != null) { for (String fileId : fileStrList) { - auditManager.audit(operationId, user, Enums.Action.UPDATE_ACLS, Enums.Resource.FILE, fileId, "", + auditManager.audit(organizationId, operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.FILE, fileId, "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError()), new ObjectMap()); } } throw e; } finally { - auditManager.finishAuditBatch(operationId); + auditManager.finishAuditBatch(organizationId, operationId); } } public OpenCGAResult getParents(String studyStr, String path, boolean rootFirst, QueryOptions options, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, userId, organizationId); List paths = org.opencb.opencga.catalog.managers.FileUtils.calculateAllPossiblePaths(path); Query query = new Query(FileDBAdaptor.QueryParams.PATH.key(), paths); query.put(FileDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - OpenCGAResult result = fileDBAdaptor.get(study.getUid(), query, options, userId); + OpenCGAResult result = getFileDBAdaptor(organizationId).get(study.getUid(), query, options, userId); result.getResults().sort(rootFirst ? ROOT_FIRST_COMPARATOR : ROOT_LAST_COMPARATOR); return result; } @@ -2853,11 +2974,12 @@ private boolean isRootFolder(File file) throws CatalogException { /** * Fetch all the recursive files and folders within the list of file ids given. * - * @param studyUid Study uid. - * @param fileList List of files + * @param organizationId Organization id. + * @param studyUid Study uid. + * @param fileList List of files * @return a more complete file list containing all the nested files */ - private List getRecursiveFilesAndFolders(long studyUid, List fileList) throws CatalogException { + private List getRecursiveFilesAndFolders(String organizationId, long studyUid, List fileList) throws CatalogException { List fileListCopy = new LinkedList<>(); fileListCopy.addAll(fileList); @@ -2876,7 +2998,7 @@ private List getRecursiveFilesAndFolders(long studyUid, List fileLis Query query = new Query() .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(FileDBAdaptor.QueryParams.PATH.key(), pathList); - OpenCGAResult fileDataResult1 = fileDBAdaptor.get(query, INCLUDE_FILE_URI_PATH); + OpenCGAResult fileDataResult1 = getFileDBAdaptor(organizationId).get(query, INCLUDE_FILE_URI_PATH); for (File file1 : fileDataResult1.getResults()) { if (!uidFileSet.contains(file1.getUid())) { uidFileSet.add(file1.getUid()); @@ -2911,13 +3033,13 @@ private String getVariantMetadataFile(String path) { } } - private OpenCGAResult getParents(long studyUid, String filePath, boolean rootFirst, QueryOptions options) + private OpenCGAResult getParents(String organizationId, long studyUid, String filePath, boolean rootFirst, QueryOptions options) throws CatalogException { List paths = org.opencb.opencga.catalog.managers.FileUtils.calculateAllPossiblePaths(filePath); Query query = new Query(FileDBAdaptor.QueryParams.PATH.key(), paths); query.put(FileDBAdaptor.QueryParams.STUDY_UID.key(), studyUid); - OpenCGAResult result = fileDBAdaptor.get(query, options); + OpenCGAResult result = getFileDBAdaptor(organizationId).get(query, options); result.getResults().sort(rootFirst ? ROOT_FIRST_COMPARATOR : ROOT_LAST_COMPARATOR); return result; } @@ -2936,15 +3058,17 @@ private String getParentPath(String path) { /** * Get the URI where a file should be in Catalog, given a study and a path. * - * @param studyId Study identifier - * @param path Path to locate - * @param directory Boolean indicating if the file is a directory + * @param organizationId Organization id. + * @param studyId Study identifier + * @param path Path to locate + * @param directory Boolean indicating if the file is a directory * @return URI where the file should be placed * @throws CatalogException CatalogException */ - private URI getFileUri(long studyId, String path, boolean directory) throws CatalogException, URISyntaxException { + private URI getFileUri(String organizationId, long studyId, String path, boolean directory) + throws CatalogException, URISyntaxException { // Get the closest existing parent. If parents == true, may happen that the parent is not registered in catalog yet. - File existingParent = getParents(studyId, path, false, null).first(); + File existingParent = getParents(organizationId, studyId, path, false, null).first(); //Relative path to the existing parent String relativePath = Paths.get(existingParent.getPath()).relativize(Paths.get(path)).toString(); @@ -2977,20 +3101,24 @@ private boolean isExternal(Study study, String catalogFilePath, URI fileUri) thr * * @param studyStr Study. * @param fileId File or folder id. - * @param unlink Boolean indicating whether the operation only expects to remove the entry from the database or also remove the file - * from disk. + * @param unlink Boolean indicating whether the operation only expects to remove the entry from the database or also remove the + * file from disk. * @param token Token of the user for which DELETE permissions will be checked. * @throws CatalogException if any of the files cannot be deleted. */ public void checkCanDeleteFile(String studyStr, String fileId, boolean unlink, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId); - checkCanDeleteFile(study, fileId, unlink, Arrays.asList(FileStatus.READY, FileStatus.TRASHED), userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, userId, organizationId); + checkCanDeleteFile(organizationId, study, fileId, unlink, Arrays.asList(FileStatus.READY, FileStatus.TRASHED), userId); } /** * Method to check if a file or folder can be deleted. It will check for indexation, status, permissions and file system availability. * + * @param organizationId Organization id. * @param study Study. * @param fileId File or folder id. * @param unlink Boolean indicating whether the operation only expects to remove the entry from the database or also remove the @@ -3001,22 +3129,23 @@ public void checkCanDeleteFile(String studyStr, String fileId, boolean unlink, S * @param userId user for which DELETE permissions will be checked. * @throws CatalogException if any of the files cannot be deleted. */ - private void checkCanDeleteFile(Study study, String fileId, boolean unlink, List acceptedStatus, String userId) - throws CatalogException { + private void checkCanDeleteFile(String organizationId, Study study, String fileId, boolean unlink, List acceptedStatus, + String userId) throws CatalogException { QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(FileDBAdaptor.QueryParams.UID.key(), FileDBAdaptor.QueryParams.NAME.key(), FileDBAdaptor.QueryParams.TYPE.key(), FileDBAdaptor.QueryParams.RELATED_FILES.key(), FileDBAdaptor.QueryParams.SIZE.key(), FileDBAdaptor.QueryParams.URI.key(), FileDBAdaptor.QueryParams.PATH.key(), FileDBAdaptor.QueryParams.INTERNAL.key(), FileDBAdaptor.QueryParams.EXTERNAL.key())); - OpenCGAResult fileOpenCGAResult = internalGet(study.getUid(), fileId, options, userId); + OpenCGAResult fileOpenCGAResult = internalGet(organizationId, study.getUid(), fileId, options, userId); if (fileOpenCGAResult.getNumResults() == 0) { throw new CatalogException("File " + fileId + " not found"); } File file = fileOpenCGAResult.first(); // If the user is the owner or the admin, we won't check if he has permissions for every single file - boolean checkPermissions = !authorizationManager.isOwnerOrAdmin(study.getUid(), userId); + long studyId = study.getUid(); + boolean checkPermissions = !authorizationManager.isAtLeastStudyAdministrator(organizationId, studyId, userId); Set indexFiles = new HashSet<>(); @@ -3034,8 +3163,8 @@ private void checkCanDeleteFile(Study study, String fileId, boolean unlink, List } if (file.getType() == File.Type.FILE) { if (checkPermissions) { - authorizationManager.checkFilePermission(study.getUid(), file.getUid(), userId, FilePermissions.WRITE); - authorizationManager.checkFilePermission(study.getUid(), file.getUid(), userId, FilePermissions.DELETE); + authorizationManager.checkFilePermission(organizationId, study.getUid(), file.getUid(), userId, FilePermissions.WRITE); + authorizationManager.checkFilePermission(organizationId, study.getUid(), file.getUid(), userId, FilePermissions.DELETE); } // File must exist in the file system @@ -3060,7 +3189,7 @@ private void checkCanDeleteFile(Study study, String fileId, boolean unlink, List // Only external files/folders are allowed within the folder Query tmpQuery = new Query(query) .append(FileDBAdaptor.QueryParams.EXTERNAL.key(), false); - long numMatches = fileDBAdaptor.count(tmpQuery).getNumMatches(); + long numMatches = getFileDBAdaptor(organizationId).count(tmpQuery).getNumMatches(); if (numMatches > 0) { throw new CatalogException(numMatches + " local files detected within the external " @@ -3070,7 +3199,7 @@ private void checkCanDeleteFile(Study study, String fileId, boolean unlink, List // Only non-external files/folders are allowed within the folder Query tmpQuery = new Query(query) .append(FileDBAdaptor.QueryParams.EXTERNAL.key(), true); - long numMatches = fileDBAdaptor.count(tmpQuery).getNumMatches(); + long numMatches = getFileDBAdaptor(organizationId).count(tmpQuery).getNumMatches(); if (numMatches > 0) { throw new CatalogException(numMatches + " external files detected within the local " @@ -3078,13 +3207,15 @@ private void checkCanDeleteFile(Study study, String fileId, boolean unlink, List } } - DBIterator iterator = fileDBAdaptor.iterator(query, options); + DBIterator iterator = getFileDBAdaptor(organizationId).iterator(query, options); while (iterator.hasNext()) { File tmpFile = iterator.next(); if (checkPermissions) { - authorizationManager.checkFilePermission(study.getUid(), tmpFile.getUid(), userId, FilePermissions.DELETE); - authorizationManager.checkFilePermission(study.getUid(), tmpFile.getUid(), userId, FilePermissions.WRITE); + authorizationManager.checkFilePermission(organizationId, study.getUid(), tmpFile.getUid(), userId, + FilePermissions.DELETE); + authorizationManager.checkFilePermission(organizationId, study.getUid(), tmpFile.getUid(), userId, + FilePermissions.WRITE); } // File must exist in the file system @@ -3103,7 +3234,7 @@ private void checkCanDeleteFile(Study study, String fileId, boolean unlink, List .append(ClinicalAnalysisDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()) .append(ClinicalAnalysisDBAdaptor.QueryParams.FILES_UID.key(), file.getUid()) .append(ClinicalAnalysisDBAdaptor.QueryParams.LOCKED.key(), true); - OpenCGAResult count = clinicalDBAdaptor.count(clinicalQuery); + OpenCGAResult count = getClinicalAnalysisDBAdaptor(organizationId).count(clinicalQuery); if (count.getNumMatches() > 0) { throw new CatalogException("The file " + file.getName() + " is part of " + count.getNumMatches() + " clinical analyses"); } @@ -3112,8 +3243,8 @@ private void checkCanDeleteFile(Study study, String fileId, boolean unlink, List // Check the original files are not being indexed at the moment if (!indexFiles.isEmpty()) { Query query = new Query(FileDBAdaptor.QueryParams.UID.key(), new ArrayList<>(indexFiles)); - try (DBIterator iterator = fileDBAdaptor.iterator(query, new QueryOptions(QueryOptions.INCLUDE, Arrays.asList( - FileDBAdaptor.QueryParams.INTERNAL.key(), FileDBAdaptor.QueryParams.UID.key())))) { + try (DBIterator iterator = getFileDBAdaptor(organizationId).iterator(query, new QueryOptions(QueryOptions.INCLUDE, + Arrays.asList(FileDBAdaptor.QueryParams.INTERNAL.key(), FileDBAdaptor.QueryParams.UID.key())))) { while (iterator.hasNext()) { File next = iterator.next(); String status = FileInternal.getVariantIndexStatusId(next.getInternal()); @@ -3127,7 +3258,6 @@ private void checkCanDeleteFile(Study study, String fileId, boolean unlink, List // We need to remove the reference to the transformed files and change their status from TRANSFORMED to NONE next.getInternal().getVariant().getIndex().setTransform(null); next.getInternal().getVariant().getIndex().getStatus().setId(VariantIndexStatus.NONE); - next.getInternal().getVariant().getIndex().getStatus().setName(VariantIndexStatus.NONE); break; case VariantIndexStatus.NONE: case VariantIndexStatus.DELETED: @@ -3165,48 +3295,10 @@ void checkValidStatusForDeletion(File file, List expectedStatus) throws throw new CatalogException("Cannot delete file: " + file.getName() + ". The status is " + file.getInternal().getStatus().getId()); } - public DataResult facet(String studyId, Query query, QueryOptions options, boolean defaultStats, String token) - throws CatalogException, IOException { - ParamUtils.defaultObject(query, Query::new); - ParamUtils.defaultObject(options, QueryOptions::new); - - String userId = userManager.getUserId(token); - // We need to add variableSets and groups to avoid additional queries as it will be used in the catalogSolrManager - Study study = studyManager.resolveId(studyId, userId, new QueryOptions(QueryOptions.INCLUDE, - Arrays.asList(StudyDBAdaptor.QueryParams.VARIABLE_SET.key(), StudyDBAdaptor.QueryParams.GROUPS.key()))); - - ObjectMap auditParams = new ObjectMap() - .append("studyId", studyId) - .append("query", new Query(query)) - .append("options", options) - .append("defaultStats", defaultStats) - .append("token", token); - try { - if (defaultStats || StringUtils.isEmpty(options.getString(QueryOptions.FACET))) { - String facet = options.getString(QueryOptions.FACET); - options.put(QueryOptions.FACET, StringUtils.isNotEmpty(facet) ? defaultFacet + ";" + facet : defaultFacet); - } - - AnnotationUtils.fixQueryAnnotationSearch(study, userId, query, authorizationManager); - - try (CatalogSolrManager catalogSolrManager = new CatalogSolrManager(catalogManager)) { - DataResult result = catalogSolrManager.facetedQuery(study, CatalogSolrManager.FILE_SOLR_COLLECTION, query, - options, userId); - auditManager.auditFacet(userId, Enums.Resource.FILE, study.getId(), study.getUuid(), auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); - - return result; - } - } catch (CatalogException e) { - auditManager.auditFacet(userId, Enums.Resource.FILE, study.getId(), study.getUuid(), auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(0, "", e.getMessage()))); - throw e; - } - } - /** * Create the parent directories that are needed. * + * @param organizationId Organization id. * @param study study where they will be created. * @param userId user that is creating the parents. * @param studyURI Base URI where the created folders will be pointing to. (base physical location) @@ -3216,10 +3308,11 @@ public DataResult facet(String studyId, Query query, QueryOptions op * that is available in catalog. * @throws CatalogDBException */ - private void createParents(Study study, String userId, URI studyURI, Path path, boolean checkPermissions) throws CatalogException { + private void createParents(String organizationId, Study study, String userId, URI studyURI, Path path, boolean checkPermissions) + throws CatalogException { if (path == null) { if (checkPermissions) { - authorizationManager.checkStudyPermission(study.getUid(), userId, StudyPermissions.Permissions.WRITE_FILES); + authorizationManager.checkStudyPermission(organizationId, study.getUid(), userId, StudyPermissions.Permissions.WRITE_FILES); } return; } @@ -3244,42 +3337,43 @@ private void createParents(Study study, String userId, URI studyURI, Path path, .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()) .append(FileDBAdaptor.QueryParams.PATH.key(), stringPath); - if (fileDBAdaptor.count(query).getNumMatches() == 0) { - createParents(study, userId, studyURI, path.getParent(), checkPermissions); + if (getFileDBAdaptor(organizationId).count(query).getNumMatches() == 0) { + createParents(organizationId, study, userId, studyURI, path.getParent(), checkPermissions); } else { if (checkPermissions) { - long fileId = fileDBAdaptor.getId(study.getUid(), stringPath); - authorizationManager.checkFilePermission(study.getUid(), fileId, userId, FilePermissions.WRITE); + long fileId = getFileDBAdaptor(organizationId).getId(study.getUid(), stringPath); + authorizationManager.checkFilePermission(organizationId, study.getUid(), fileId, userId, FilePermissions.WRITE); } return; } String parentPath = getParentPath(stringPath); - long parentFileId = fileDBAdaptor.getId(study.getUid(), parentPath); + long parentFileId = getFileDBAdaptor(organizationId).getId(study.getUid(), parentPath); // We obtain the permissions set in the parent folder and set them to the file or folder being created OpenCGAResult> allFileAcls = - authorizationManager.getAcls(study.getUid(), parentFileId, Enums.Resource.FILE, FilePermissions.class); + authorizationManager.getAcls(organizationId, study.getUid(), parentFileId, Enums.Resource.FILE, FilePermissions.class); URI completeURI = Paths.get(studyURI).resolve(path).toUri(); // Create the folder in catalog File folder = new File(path.getFileName().toString(), File.Type.DIRECTORY, File.Format.PLAIN, File.Bioformat.NONE, completeURI, stringPath, null, TimeUtils.getTime(), TimeUtils.getTime(), "", false, 0, null, new FileExperiment(), - Collections.emptyList(), Collections.emptyList(), "", studyManager.getCurrentRelease(study), Collections.emptyList(), - Collections.emptyList(), new FileQualityControl(), null, new Status(), FileInternal.init(), null); + Collections.emptyList(), Collections.emptyList(), "", studyManager.getCurrentRelease(study), + Collections.emptyList(), Collections.emptyList(), new FileQualityControl(), null, new Status(), FileInternal.init(), null); folder.setUuid(UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.FILE)); checkHooks(folder, study.getFqn(), HookConfiguration.Stage.CREATE); - fileDBAdaptor.insert(study.getUid(), folder, Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), - new QueryOptions()); - OpenCGAResult queryResult = getFile(study.getUid(), folder.getUuid(), QueryOptions.empty()); + getFileDBAdaptor(organizationId).insert(study.getUid(), folder, Collections.emptyList(), Collections.emptyList(), + Collections.emptyList(), new QueryOptions()); + OpenCGAResult queryResult = getFile(organizationId, study.getUid(), folder.getUuid(), QueryOptions.empty()); // Propagate ACLs if (allFileAcls != null && allFileAcls.getNumResults() > 0) { - authorizationManager.replicateAcls(Collections.singletonList(queryResult.first().getUid()), allFileAcls.getResults().get(0), - Enums.Resource.FILE); + authorizationManager.replicateAcls(organizationId, Collections.singletonList(queryResult.first().getUid()), + allFileAcls.getResults().get(0), Enums.Resource.FILE); } } - private OpenCGAResult privateLink(Study study, FileLinkParams params, boolean parents, String token) throws CatalogException { + private OpenCGAResult privateLink(String organizationId, Study study, FileLinkParams params, boolean parents, + JwtPayload jwtPayload) throws CatalogException { ParamUtils.checkObj(params, "FileLinkParams"); ParamUtils.checkParameter(params.getUri(), "uri"); URI uriOrigin; @@ -3306,8 +3400,8 @@ private OpenCGAResult privateLink(Study study, FileLinkParams params, bool throw new CatalogException(e); } - String userId = userManager.getUserId(token); - authorizationManager.checkStudyPermission(study.getUid(), userId, StudyPermissions.Permissions.WRITE_FILES); + String userId = jwtPayload.getUserId(organizationId); + authorizationManager.checkStudyPermission(organizationId, study.getUid(), userId, StudyPermissions.Permissions.WRITE_FILES); params.setPath(ParamUtils.defaultString(params.getPath(), "")); if (params.getPath().length() == 1 && (params.getPath().equals(".") || params.getPath().equals("/"))) { @@ -3341,7 +3435,7 @@ private OpenCGAResult privateLink(Study study, FileLinkParams params, bool .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()) .append(FileDBAdaptor.QueryParams.PATH.key(), externalPathDestinyStr) .append(FileDBAdaptor.QueryParams.EXTERNAL.key(), false); - if (fileDBAdaptor.count(query).getNumMatches() > 0) { + if (getFileDBAdaptor(organizationId).count(query).getNumMatches() > 0) { throw new CatalogException("Cannot link to " + externalPathDestinyStr + ". The path already existed and is not external."); } @@ -3352,7 +3446,7 @@ private OpenCGAResult privateLink(Study study, FileLinkParams params, bool .append(FileDBAdaptor.QueryParams.PATH.key(), externalPathDestinyStr) .append(FileDBAdaptor.QueryParams.EXTERNAL.key(), true); - if (fileDBAdaptor.count(query).getNumMatches() > 0) { + if (getFileDBAdaptor(organizationId).count(query).getNumMatches() > 0) { // Create a regular expression on URI to return everything linked from that URI query.put(FileDBAdaptor.QueryParams.URI.key(), "~^" + normalizedUri); query.remove(FileDBAdaptor.QueryParams.PATH.key()); @@ -3361,7 +3455,7 @@ private OpenCGAResult privateLink(Study study, FileLinkParams params, bool QueryOptions queryOptions = new QueryOptions() .append(QueryOptions.LIMIT, 100); - return fileDBAdaptor.get(query, queryOptions) + return getFileDBAdaptor(organizationId).get(query, queryOptions) .addEvent(new Event(Event.Type.INFO, ParamConstants.FILE_ALREADY_LINKED)); } @@ -3370,9 +3464,9 @@ private OpenCGAResult privateLink(Study study, FileLinkParams params, bool .append(FileDBAdaptor.QueryParams.URI.key(), normalizedUri) .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()) .append(FileDBAdaptor.QueryParams.EXTERNAL.key(), true); - if (fileDBAdaptor.count(query).getNumMatches() > 0) { + if (getFileDBAdaptor(organizationId).count(query).getNumMatches() > 0) { QueryOptions queryOptions = new QueryOptions(QueryOptions.INCLUDE, FileDBAdaptor.QueryParams.PATH.key()); - String path = fileDBAdaptor.get(query, queryOptions).first().getPath(); + String path = getFileDBAdaptor(organizationId).get(query, queryOptions).first().getPath(); throw new CatalogException(normalizedUri + " was already linked to other path: " + path); } @@ -3385,7 +3479,8 @@ private OpenCGAResult privateLink(Study study, FileLinkParams params, bool if (relatedFilesParams != null) { relatedFiles = new ArrayList<>(relatedFilesParams.size()); for (SmallRelatedFileParams relatedFileParams : relatedFilesParams) { - File tmpFile = internalGet(study.getUid(), relatedFileParams.getFile(), INCLUDE_FILE_URI_PATH, userId).first(); + File tmpFile = internalGet(organizationId, study.getUid(), relatedFileParams.getFile(), INCLUDE_FILE_URI_PATH, userId) + .first(); relatedFiles.add(new FileRelatedFile(tmpFile, relatedFileParams.getRelation())); } } else { @@ -3397,17 +3492,17 @@ private OpenCGAResult privateLink(Study study, FileLinkParams params, bool if (params.getPath().isEmpty()) { // If no destiny is given, everything will be linked to the root folder of the study. - authorizationManager.checkStudyPermission(study.getUid(), userId, StudyPermissions.Permissions.WRITE_FILES); + authorizationManager.checkStudyPermission(organizationId, study.getUid(), userId, StudyPermissions.Permissions.WRITE_FILES); } else { // Check if the folder exists query = new Query() .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()) .append(FileDBAdaptor.QueryParams.PATH.key(), params.getPath()); - if (fileDBAdaptor.count(query).getNumMatches() == 0) { + if (getFileDBAdaptor(organizationId).count(query).getNumMatches() == 0) { if (parents) { // Get the base URI where the files are located in the study URI studyURI = study.getUri(); - createParents(study, userId, studyURI, catalogPath, true); + createParents(organizationId, study, userId, studyURI, catalogPath, true); // Create them in the disk // URI directory = Paths.get(studyURI).resolve(catalogPath).toUri(); // catalogIOManagerFactory.get(directory).createDirectory(directory, true); @@ -3416,8 +3511,8 @@ private OpenCGAResult privateLink(Study study, FileLinkParams params, bool } } else { // Check if the user has permissions to link files in the directory - long fileId = fileDBAdaptor.getId(study.getUid(), params.getPath()); - authorizationManager.checkFilePermission(study.getUid(), fileId, userId, FilePermissions.WRITE); + long fileId = getFileDBAdaptor(organizationId).getId(study.getUid(), params.getPath()); + authorizationManager.checkFilePermission(organizationId, study.getUid(), fileId, userId, FilePermissions.WRITE); } } @@ -3454,15 +3549,15 @@ public FileVisitResult preVisitDirectory(URI dir, BasicFileAttributes attrs) thr .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()) .append(FileDBAdaptor.QueryParams.PATH.key(), destinyPath); - if (fileDBAdaptor.count(query).getNumMatches() == 0) { + if (getFileDBAdaptor(organizationId).count(query).getNumMatches() == 0) { // If the folder does not exist, we create it String parentPath = getParentPath(destinyPath); - long parentFileId = fileDBAdaptor.getId(study.getUid(), parentPath); + long parentFileId = getFileDBAdaptor(organizationId).getId(study.getUid(), parentPath); // We obtain the permissions set in the parent folder and set them to the file or folder being created OpenCGAResult> allFileAcls; try { - allFileAcls = authorizationManager.getAcls(study.getUid(), parentFileId, Enums.Resource.FILE, + allFileAcls = authorizationManager.getAcls(organizationId, study.getUid(), parentFileId, Enums.Resource.FILE, FilePermissions.class); } catch (CatalogException e) { throw new RuntimeException(e); @@ -3471,19 +3566,19 @@ public FileVisitResult preVisitDirectory(URI dir, BasicFileAttributes attrs) thr File folder = new File(Paths.get(dir).getFileName().toString(), File.Type.DIRECTORY, File.Format.PLAIN, File.Bioformat.NONE, dir, destinyPath, null, creationDate, modificationDate, params.getDescription(), true, 0, new Software(), new FileExperiment(), - Collections.emptyList(), relatedFiles, "", studyManager.getCurrentRelease(study), Collections.emptyList(), - Collections.emptyList(), new FileQualityControl(), Collections.emptyMap(), + Collections.emptyList(), relatedFiles, "", studyManager.getCurrentRelease(study), + Collections.emptyList(), Collections.emptyList(), new FileQualityControl(), Collections.emptyMap(), params.getStatus() != null ? params.getStatus().toStatus() : new Status(), FileInternal.init(), Collections.emptyMap()); folder.setUuid(UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.FILE)); checkHooks(folder, study.getFqn(), HookConfiguration.Stage.CREATE); - fileDBAdaptor.insert(study.getUid(), folder, Collections.emptyList(), Collections.emptyList(), + getFileDBAdaptor(organizationId).insert(study.getUid(), folder, Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), new QueryOptions()); - OpenCGAResult queryResult = getFile(study.getUid(), folder.getUuid(), QueryOptions.empty()); + OpenCGAResult queryResult = getFile(organizationId, study.getUid(), folder.getUuid(), QueryOptions.empty()); // Propagate ACLs if (allFileAcls != null && allFileAcls.getNumResults() > 0) { - authorizationManager.replicateAcls(Collections.singletonList(queryResult.first().getUid()), + authorizationManager.replicateAcls(organizationId, Collections.singletonList(queryResult.first().getUid()), allFileAcls.getResults().get(0), Enums.Resource.FILE); } } @@ -3508,15 +3603,15 @@ public FileVisitResult visitFile(URI fileUri, BasicFileAttributes attrs) throws .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()) .append(FileDBAdaptor.QueryParams.PATH.key(), destinyPath); - if (fileDBAdaptor.count(query).getNumMatches() == 0) { + if (getFileDBAdaptor(organizationId).count(query).getNumMatches() == 0) { long size = ioManager.getFileSize(fileUri); // If the file does not exist, we create it String parentPath = getParentPath(destinyPath); - long parentFileId = fileDBAdaptor.getId(study.getUid(), parentPath); + long parentFileId = getFileDBAdaptor(organizationId).getId(study.getUid(), parentPath); // We obtain the permissions set in the parent folder and set them to the file or folder being created OpenCGAResult> allFileAcls; try { - allFileAcls = authorizationManager.getAcls(study.getUid(), parentFileId, Enums.Resource.FILE, + allFileAcls = authorizationManager.getAcls(organizationId, study.getUid(), parentFileId, Enums.Resource.FILE, FilePermissions.class); } catch (CatalogException e) { throw new RuntimeException(e); @@ -3530,8 +3625,8 @@ public FileVisitResult visitFile(URI fileUri, BasicFileAttributes attrs) throws File subfile = new File(Paths.get(fileUri).getFileName().toString(), File.Type.FILE, File.Format.UNKNOWN, File.Bioformat.NONE, fileUri, destinyPath, null, creationDate, modificationDate, params.getDescription(), true, size, new Software(), new FileExperiment(), - Collections.emptyList(), relatedFiles, "", studyManager.getCurrentRelease(study), Collections.emptyList(), - Collections.emptyList(), new FileQualityControl(), Collections.emptyMap(), + Collections.emptyList(), relatedFiles, "", studyManager.getCurrentRelease(study), + Collections.emptyList(), Collections.emptyList(), new FileQualityControl(), Collections.emptyMap(), params.getStatus() != null ? params.getStatus().toStatus() : new Status(), internal, new HashMap<>()); subfile.setUuid(UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.FILE)); @@ -3542,7 +3637,7 @@ public FileVisitResult visitFile(URI fileUri, BasicFileAttributes attrs) throws List existingSamples = new LinkedList<>(); List nonExistingSamples = new LinkedList<>(); - validateNewSamples(study, subfile, existingSamples, nonExistingSamples, token); + validateNewSamples(organizationId, study, subfile, existingSamples, nonExistingSamples, jwtPayload); File virtualFile; if (StringUtils.isNotEmpty(params.getVirtualFileName())) { @@ -3561,7 +3656,8 @@ public FileVisitResult visitFile(URI fileUri, BasicFileAttributes attrs) throws FileDBAdaptor.QueryParams.TYPE.key(), FileDBAdaptor.QueryParams.SAMPLE_IDS.key(), FileDBAdaptor.QueryParams.INTERNAL_MISSING_SAMPLES.key())); - OpenCGAResult vFileResult = fileDBAdaptor.get(study.getUid(), tmpQuery, tmpOptions, userId); + OpenCGAResult vFileResult = getFileDBAdaptor(organizationId).get(study.getUid(), tmpQuery, tmpOptions, + userId); if (vFileResult.getNumResults() == 1) { if (!vFileResult.first().getType().equals(File.Type.VIRTUAL)) { @@ -3584,7 +3680,7 @@ public FileVisitResult visitFile(URI fileUri, BasicFileAttributes attrs) throws + "of samples differ."); } } else { - validateNewFile(study, virtualFile, true); + validateNewFile(organizationId, study, virtualFile, true); virtualFile.setSampleIds(subfile.getSampleIds()); virtualFile.setUri(null); virtualFile.setType(File.Type.VIRTUAL); @@ -3593,19 +3689,19 @@ public FileVisitResult visitFile(URI fileUri, BasicFileAttributes attrs) throws } subfile.setSampleIds(null); - fileDBAdaptor.insertWithVirtualFile(study.getUid(), subfile, virtualFile, existingSamples, nonExistingSamples, - Collections.emptyList(), new QueryOptions()); + getFileDBAdaptor(organizationId).insertWithVirtualFile(study.getUid(), subfile, virtualFile, existingSamples, + nonExistingSamples, Collections.emptyList(), new QueryOptions()); } else { - fileDBAdaptor.insert(study.getUid(), subfile, existingSamples, nonExistingSamples, Collections.emptyList(), - new QueryOptions()); + getFileDBAdaptor(organizationId).insert(study.getUid(), subfile, existingSamples, nonExistingSamples, + Collections.emptyList(), new QueryOptions()); } - subfile = getFile(study.getUid(), subfile.getUuid(), QueryOptions.empty()).first(); + subfile = getFile(organizationId, study.getUid(), subfile.getUuid(), QueryOptions.empty()).first(); // Propagate ACLs if (allFileAcls != null && allFileAcls.getNumResults() > 0) { - authorizationManager.replicateAcls(Collections.singletonList(subfile.getUid()), allFileAcls.getResults().get(0), - Enums.Resource.FILE); + authorizationManager.replicateAcls(organizationId, Collections.singletonList(subfile.getUid()), + allFileAcls.getResults().get(0), Enums.Resource.FILE); } if (isTransformedFile(subfile.getName())) { @@ -3636,8 +3732,8 @@ public FileVisitResult postVisitDirectory(URI dir, IOException exc) throws IOExc // Try to link transformed files with their corresponding original files if any try { - if (transformedFiles.size() > 0) { - matchUpVariantFiles(study.getFqn(), transformedFiles, token); + if (!transformedFiles.isEmpty()) { + matchUpVariantFiles(study.getFqn(), transformedFiles, jwtPayload.getToken()); } } catch (CatalogException e) { logger.warn("Matching avro to variant file: {}", e.getMessage()); @@ -3652,11 +3748,12 @@ public FileVisitResult postVisitDirectory(URI dir, IOException exc) throws IOExc // Limit the number of results and only some fields QueryOptions queryOptions = new QueryOptions() .append(QueryOptions.LIMIT, 100); - return fileDBAdaptor.get(query, queryOptions); + return getFileDBAdaptor(organizationId).get(query, queryOptions); } - OpenCGAResult registerFile(Study study, String filePath, URI fileUri, String jobId, String token) throws CatalogException { - String userId = userManager.getUserId(token); + OpenCGAResult registerFile(String organizationId, Study study, String filePath, URI fileUri, String jobId, + JwtPayload tokenPayload) throws CatalogException { + String userId = tokenPayload.getUserId(organizationId); IOManager ioManager; try { ioManager = ioManagerFactory.get(fileUri); @@ -3668,10 +3765,11 @@ OpenCGAResult registerFile(Study study, String filePath, URI fileUri, Stri long size = ioManager.getFileSize(fileUri); String parentPath = getParentPath(filePath); - File parentFile = internalGet(study.getUid(), parentPath, INCLUDE_FILE_URI_PATH, userId).first(); + File parentFile = internalGet(organizationId, study.getUid(), parentPath, INCLUDE_FILE_URI_PATH, userId).first(); // We obtain the permissions set in the parent folder and set them to the file or folder being created OpenCGAResult> allFileAcls = - authorizationManager.getAcls(study.getUid(), parentFile.getUid(), Enums.Resource.FILE, FilePermissions.class); + authorizationManager.getAcls(organizationId, study.getUid(), parentFile.getUid(), Enums.Resource.FILE, + FilePermissions.class); File.Type type = filePath.endsWith("/") ? File.Type.DIRECTORY : File.Type.FILE; @@ -3694,22 +3792,23 @@ OpenCGAResult registerFile(Study study, String filePath, URI fileUri, Stri } List existingSamples = new LinkedList<>(); List nonExistingSamples = new LinkedList<>(); - validateNewSamples(study, subfile, existingSamples, nonExistingSamples, token); + validateNewSamples(organizationId, study, subfile, existingSamples, nonExistingSamples, tokenPayload); - fileDBAdaptor.insert(study.getUid(), subfile, existingSamples, nonExistingSamples, Collections.emptyList(), new QueryOptions()); - OpenCGAResult result = getFile(study.getUid(), subfile.getUuid(), QueryOptions.empty()); + getFileDBAdaptor(organizationId).insert(study.getUid(), subfile, existingSamples, nonExistingSamples, Collections.emptyList(), + new QueryOptions()); + OpenCGAResult result = getFile(organizationId, study.getUid(), subfile.getUuid(), QueryOptions.empty()); subfile = result.first(); // Propagate ACLs if (allFileAcls != null && allFileAcls.getNumResults() > 0) { - authorizationManager.replicateAcls(Collections.singletonList(subfile.getUid()), allFileAcls.getResults().get(0), + authorizationManager.replicateAcls(organizationId, Collections.singletonList(subfile.getUid()), allFileAcls.getResults().get(0), Enums.Resource.FILE); } // If it is a transformed file, we will try to link it with the correspondent original file try { if (isTransformedFile(subfile.getName())) { - matchUpVariantFiles(study.getFqn(), Arrays.asList(subfile), token); + matchUpVariantFiles(study.getFqn(), Arrays.asList(subfile), tokenPayload.getToken()); } } catch (CatalogException e1) { logger.warn("Matching avro to variant file: {}", e1.getMessage()); @@ -3724,10 +3823,10 @@ private void checkHooks(File file, String fqn, HookConfiguration.Stage stage) th if (hooks != null && hooks.containsKey(fqn)) { Map> entityHookMap = hooks.get(fqn); List hookList = null; - if (entityHookMap.containsKey(MongoDBAdaptorFactory.FILE_COLLECTION)) { - hookList = entityHookMap.get(MongoDBAdaptorFactory.FILE_COLLECTION); - } else if (entityHookMap.containsKey(MongoDBAdaptorFactory.FILE_COLLECTION.toUpperCase())) { - hookList = entityHookMap.get(MongoDBAdaptorFactory.FILE_COLLECTION.toUpperCase()); + if (entityHookMap.containsKey(OrganizationMongoDBAdaptorFactory.FILE_COLLECTION)) { + hookList = entityHookMap.get(OrganizationMongoDBAdaptorFactory.FILE_COLLECTION); + } else if (entityHookMap.containsKey(OrganizationMongoDBAdaptorFactory.FILE_COLLECTION.toUpperCase())) { + hookList = entityHookMap.get(OrganizationMongoDBAdaptorFactory.FILE_COLLECTION.toUpperCase()); } // We check the hook list @@ -3973,11 +4072,11 @@ private void checkHooks(File file, String fqn, HookConfiguration.Stage stage) th } } - private URI getStudyUri(long studyId) throws CatalogException { - return studyDBAdaptor.get(studyId, INCLUDE_STUDY_URI).first().getUri(); + private URI getStudyUri(String organizationId, long studyId) throws CatalogException { + return getStudyDBAdaptor(organizationId).get(studyId, INCLUDE_STUDY_URI).first().getUri(); } - private CheckPath checkPathExists(String path, long studyId) throws CatalogException { + private CheckPath checkPathExists(String organizationId, long studyId, String path) throws CatalogException { String myPath = path; if (myPath.endsWith("/")) { myPath = myPath.substring(0, myPath.length() - 1); @@ -3987,7 +4086,7 @@ private CheckPath checkPathExists(String path, long studyId) throws CatalogExcep Query query = new Query() .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), studyId) .append(FileDBAdaptor.QueryParams.PATH.key(), myPath); - OpenCGAResult fileDataResult = fileDBAdaptor.count(query); + OpenCGAResult fileDataResult = getFileDBAdaptor(organizationId).count(query); if (fileDataResult.getNumMatches() > 0) { return CheckPath.FILE_EXISTS; } @@ -3995,17 +4094,17 @@ private CheckPath checkPathExists(String path, long studyId) throws CatalogExcep query = new Query() .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), studyId) .append(FileDBAdaptor.QueryParams.PATH.key(), myPath + "/"); - fileDataResult = fileDBAdaptor.count(query); + fileDataResult = getFileDBAdaptor(organizationId).count(query); return fileDataResult.getNumMatches() > 0 ? CheckPath.DIRECTORY_EXISTS : CheckPath.FREE_PATH; } - public int getFileSampleLinkThreshold() { - return fileDBAdaptor.getFileSampleLinkThreshold(); + public int getFileSampleLinkThreshold(String organizationId) throws CatalogDBException { + return getFileDBAdaptor(organizationId).getFileSampleLinkThreshold(); } - public void setFileSampleLinkThreshold(int numSamples) { - fileDBAdaptor.setFileSampleLinkThreshold(numSamples); + public void setFileSampleLinkThreshold(String organizationId, int numSamples) throws CatalogDBException { + getFileDBAdaptor(organizationId).setFileSampleLinkThreshold(numSamples); } private enum CheckPath { diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileUtils.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileUtils.java index 686b91cc6af..04d2352065b 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileUtils.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileUtils.java @@ -24,8 +24,10 @@ import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogIOException; import org.opencb.opencga.catalog.io.IOManager; +import org.opencb.opencga.catalog.utils.CatalogFqn; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.common.UriUtils; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.file.File; import org.opencb.opencga.core.models.file.FileRelatedFile; import org.opencb.opencga.core.models.file.FileStatus; @@ -76,6 +78,10 @@ public FileUtils(CatalogManager catalogManager) { * @throws CatalogException CatalogException */ public File checkFile(String studyStr, File file, boolean calculateChecksum, String sessionId) throws CatalogException { + JwtPayload payload = catalogManager.getUserManager().validateToken(sessionId); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, payload); + String organizationId = studyFqn.getOrganizationId(); + if (!file.getType().equals(File.Type.FILE)) { return file; } @@ -84,7 +90,7 @@ public File checkFile(String studyStr, File file, boolean calculateChecksum, Str switch (file.getInternal().getStatus().getId()) { case FileStatus.READY: case FileStatus.MISSING: { - URI fileUri = catalogManager.getFileManager().getUri(file); + URI fileUri = catalogManager.getFileManager().getUri(organizationId, file); IOManager ioManager; try { ioManager = catalogManager.getIoManagerFactory().get(fileUri); @@ -99,15 +105,17 @@ public File checkFile(String studyStr, File file, boolean calculateChecksum, Str ObjectMap params = new ObjectMap(FileDBAdaptor.UpdateParams.INTERNAL_STATUS.key(), new FileStatus(FileStatus.MISSING)); catalogManager.getFileManager().update(studyStr, file.getPath(), params, null, sessionId); - modifiedFile = catalogManager.getFileManager().get(studyStr, file.getPath(), null, sessionId).first(); + modifiedFile = catalogManager.getFileManager().get(studyStr, file.getPath(), null, sessionId) + .first(); } } else if (file.getInternal().getStatus().getId().equals(FileStatus.MISSING)) { logger.info("File { path:\"" + file.getPath() + "\" } recover tracking from file " + fileUri); logger.info("Set status to " + FileStatus.READY); - ObjectMap params = getModifiedFileAttributes(file, fileUri, calculateChecksum); + ObjectMap params = getModifiedFileAttributes(organizationId, file, fileUri, calculateChecksum); params.put(FileDBAdaptor.UpdateParams.INTERNAL_STATUS.key(), new FileStatus(FileStatus.READY)); - catalogManager.getFileManager().update(studyStr, file.getPath(), params, QueryOptions.empty(), sessionId); + catalogManager.getFileManager().update(studyStr, file.getPath(), params, QueryOptions.empty(), + sessionId); modifiedFile = catalogManager.getFileManager().get(studyStr, file.getPath(), null, sessionId).first(); } break; @@ -135,17 +143,19 @@ public File checkFile(String studyStr, File file, boolean calculateChecksum, Str * checksum * uri * + * @param organizationId Organization id. * @param file file * @param fileUri If null, calls to getFileUri() *

- * TODO: Lazy checksum: Only calculate checksum if the size has changed. + * TODO: Lazy checksum: Only calculate checksum if the size has changed. * @param calculateChecksum Calculate checksum to check if have changed * @return ObjectMap ObjectMap * @throws CatalogException CatalogException */ - public ObjectMap getModifiedFileAttributes(File file, URI fileUri, boolean calculateChecksum) throws CatalogException { + public ObjectMap getModifiedFileAttributes(String organizationId, File file, URI fileUri, boolean calculateChecksum) + throws CatalogException { if (fileUri == null) { - fileUri = catalogManager.getFileManager().getUri(file); + fileUri = catalogManager.getFileManager().getUri(organizationId, file); } String checksum = null; if (calculateChecksum) { @@ -155,7 +165,7 @@ public ObjectMap getModifiedFileAttributes(File file, URI fileUri, boolean calcu throw CatalogIOException.ioManagerException(fileUri, e); } } - return getModifiedFileAttributes(file, checksum, fileUri, null); + return getModifiedFileAttributes(organizationId, file, checksum, fileUri, null); } /** @@ -166,10 +176,11 @@ public ObjectMap getModifiedFileAttributes(File file, URI fileUri, boolean calcu * * @throws CatalogException CatalogException */ - private ObjectMap getModifiedFileAttributes(File file, String checksum, URI fileUri, ObjectMap parameters) throws CatalogException { + private ObjectMap getModifiedFileAttributes(String organizationId, File file, String checksum, URI fileUri, ObjectMap parameters) + throws CatalogException { parameters = ParamUtils.defaultObject(parameters, ObjectMap::new); if (fileUri == null) { - fileUri = catalogManager.getFileManager().getUri(file); + fileUri = catalogManager.getFileManager().getUri(organizationId, file); } IOManager ioManager; try { diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/IndividualManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/IndividualManager.java index 97057677b0d..cfe73a2788c 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/IndividualManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/IndividualManager.java @@ -23,7 +23,10 @@ import org.opencb.biodata.models.core.OntologyTermAnnotation; import org.opencb.biodata.models.core.SexOntologyTermAnnotation; import org.opencb.biodata.models.pedigree.IndividualProperty; -import org.opencb.commons.datastore.core.*; +import org.opencb.commons.datastore.core.Event; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.result.Error; import org.opencb.commons.utils.ListUtils; import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager; @@ -32,15 +35,12 @@ import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.models.InternalGetDataResult; -import org.opencb.opencga.catalog.stats.solr.CatalogSolrManager; -import org.opencb.opencga.catalog.utils.AnnotationUtils; -import org.opencb.opencga.catalog.utils.Constants; -import org.opencb.opencga.catalog.utils.ParamUtils; -import org.opencb.opencga.catalog.utils.UuidUtils; +import org.opencb.opencga.catalog.utils.*; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.models.AclEntryList; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.audit.AuditRecord; import org.opencb.opencga.core.models.common.AnnotationSet; import org.opencb.opencga.core.models.common.Enums; @@ -56,7 +56,6 @@ import org.slf4j.LoggerFactory; import javax.annotation.Nullable; -import java.io.IOException; import java.util.*; import java.util.function.Function; import java.util.stream.Collectors; @@ -127,9 +126,10 @@ Enums.Resource getEntity() { // queryCopy.put(IndividualDBAdaptor.QueryParams.ID.key(), entry); // } // QueryOptions queryOptions = options != null ? new QueryOptions(options) : new QueryOptions(); -// OpenCGAResult individualDataResult = individualDBAdaptor.get(studyUid, queryCopy, queryOptions, user); +// OpenCGAResult individualDataResult = getIndividualDBAdaptor(organizationId).get(studyUid, queryCopy, queryOptions, +// user); // if (individualDataResult.getNumResults() == 0) { -// individualDataResult = individualDBAdaptor.get(queryCopy, queryOptions); +// individualDataResult = getIndividualDBAdaptor(organizationId).get(queryCopy, queryOptions); // if (individualDataResult.getNumResults() == 0) { // throw new CatalogException("Individual " + entry + " not found"); // } else { @@ -143,8 +143,8 @@ Enums.Resource getEntity() { // } @Override - InternalGetDataResult internalGet(long studyUid, List entryList, @Nullable Query query, QueryOptions options, - String user, boolean ignoreException) throws CatalogException { + InternalGetDataResult internalGet(String organizationId, long studyUid, List entryList, @Nullable Query query, + QueryOptions options, String user, boolean ignoreException) throws CatalogException { if (ListUtils.isEmpty(entryList)) { throw new CatalogException("Missing individual entries."); } @@ -166,7 +166,8 @@ InternalGetDataResult internalGet(long studyUid, List entryL // Ensure the field by which we are querying for will be kept in the results queryOptions = keepFieldInQueryOptions(queryOptions, idQueryParam.key()); - OpenCGAResult individualDataResult = individualDBAdaptor.get(studyUid, queryCopy, queryOptions, user); + OpenCGAResult individualDataResult = getIndividualDBAdaptor(organizationId).get(studyUid, queryCopy, queryOptions, + user); Function individualStringFunction = Individual::getId; if (idQueryParam.equals(IndividualDBAdaptor.QueryParams.UUID)) { @@ -176,7 +177,7 @@ InternalGetDataResult internalGet(long studyUid, List entryL return keepOriginalOrder(uniqueList, individualStringFunction, individualDataResult, ignoreException, versioned); } // Query without adding the user check - OpenCGAResult resultsNoCheck = individualDBAdaptor.get(queryCopy, queryOptions); + OpenCGAResult resultsNoCheck = getIndividualDBAdaptor(organizationId).get(queryCopy, queryOptions); if (resultsNoCheck.getNumResults() == individualDataResult.getNumResults()) { throw CatalogException.notFound("individuals", @@ -204,15 +205,16 @@ IndividualDBAdaptor.QueryParams getFieldFilter(List idList) throws Catal return idQueryParam; } - private OpenCGAResult getIndividual(long studyUid, String individualUuid, QueryOptions options) throws CatalogException { + private OpenCGAResult getIndividual(String organizationId, long studyUid, String individualUuid, QueryOptions options) + throws CatalogException { Query query = new Query() .append(IndividualDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(IndividualDBAdaptor.QueryParams.UUID.key(), individualUuid); - return individualDBAdaptor.get(query, options); + return getIndividualDBAdaptor(organizationId).get(query, options); } - void validateNewIndividual(Study study, Individual individual, List samples, String userId, boolean linkParents) - throws CatalogException { + void validateNewIndividual(String organizationId, Study study, Individual individual, List samples, String userId, + boolean linkParents) throws CatalogException { ParamUtils.checkIdentifier(individual.getId(), "id"); individual.setName(StringUtils.isEmpty(individual.getName()) ? individual.getId() : individual.getName()); individual.setLocation(ParamUtils.defaultObject(individual.getLocation(), Location::new)); @@ -249,37 +251,38 @@ void validateNewIndividual(Study study, Individual individual, List samp Query query = new Query() .append(IndividualDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()) .append(IndividualDBAdaptor.QueryParams.ID.key(), individual.getId()); - if (individualDBAdaptor.count(query).getNumMatches() > 0) { + if (getIndividualDBAdaptor(organizationId).count(query).getNumMatches() > 0) { throw new CatalogException("Individual '" + individual.getId() + "' already exists."); } validateNewAnnotationSets(study.getVariableSets(), individual.getAnnotationSets()); - validateSamples(study, individual, samples, userId); + validateSamples(organizationId, study, individual, samples, userId); if (linkParents) { if (individual.getFather() != null && StringUtils.isNotEmpty(individual.getFather().getId())) { - OpenCGAResult fatherResult = internalGet(study.getUid(), individual.getFather().getId(), INCLUDE_INDIVIDUAL_IDS, - userId); + OpenCGAResult fatherResult = internalGet(organizationId, study.getUid(), individual.getFather().getId(), + INCLUDE_INDIVIDUAL_IDS, userId); individual.setFather(fatherResult.first()); } if (individual.getMother() != null && StringUtils.isNotEmpty(individual.getMother().getId())) { - OpenCGAResult motherResult = internalGet(study.getUid(), individual.getMother().getId(), INCLUDE_INDIVIDUAL_IDS, - userId); + OpenCGAResult motherResult = internalGet(organizationId, study.getUid(), individual.getMother().getId(), + INCLUDE_INDIVIDUAL_IDS, userId); individual.setMother(motherResult.first()); } } } - private void validateSamples(Study study, Individual individual, List samples, String userId) throws CatalogException { + private void validateSamples(String organizationId, Study study, Individual individual, List samples, String userId) + throws CatalogException { List sampleList = new ArrayList<>(); if (individual.getSamples() != null && !individual.getSamples().isEmpty()) { // Check the user can create new samples - authorizationManager.checkStudyPermission(study.getUid(), userId, StudyPermissions.Permissions.WRITE_SAMPLES); + authorizationManager.checkStudyPermission(organizationId, study.getUid(), userId, StudyPermissions.Permissions.WRITE_SAMPLES); // Validate the samples can be created and are valid for (Sample sample : individual.getSamples()) { - catalogManager.getSampleManager().validateNewSample(study, sample, userId); + catalogManager.getSampleManager().validateNewSample(organizationId, study, sample, userId); sampleList.add(sample); } } @@ -288,13 +291,13 @@ private void validateSamples(Study study, Individual individual, List sa // We remove any possible duplicate ArrayList deduplicatedSampleIds = new ArrayList<>(new HashSet<>(samples)); - InternalGetDataResult sampleDataResult = catalogManager.getSampleManager().internalGet(study.getUid(), + InternalGetDataResult sampleDataResult = catalogManager.getSampleManager().internalGet(organizationId, study.getUid(), deduplicatedSampleIds, SampleManager.INCLUDE_SAMPLE_IDS, userId, false); // Check the samples are not attached to other individual Set sampleUidSet = sampleDataResult.getResults().stream().map(Sample::getUid).collect(Collectors.toSet()); - checkSamplesNotInUseInOtherIndividual(sampleUidSet, study.getUid(), null); + checkSamplesNotInUseInOtherIndividual(organizationId, sampleUidSet, study.getUid(), null); sampleList.addAll(sampleDataResult.getResults()); } @@ -312,8 +315,11 @@ public OpenCGAResult create(String studyStr, Individual individual, String token) throws CatalogException { options = ParamUtils.defaultObject(options, QueryOptions::new); - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId, StudyManager.INCLUDE_VARIABLE_SET); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, StudyManager.INCLUDE_VARIABLE_SET, userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("study", studyStr) @@ -322,28 +328,30 @@ public OpenCGAResult create(String studyStr, Individual individual, .append("options", options) .append("token", token); try { - authorizationManager.checkStudyPermission(study.getUid(), userId, StudyPermissions.Permissions.WRITE_INDIVIDUALS); - validateNewIndividual(study, individual, sampleIds, userId, true); + authorizationManager.checkStudyPermission(organizationId, study.getUid(), userId, + StudyPermissions.Permissions.WRITE_INDIVIDUALS); + validateNewIndividual(organizationId, study, individual, sampleIds, userId, true); // Create the individual - OpenCGAResult insert = individualDBAdaptor.insert(study.getUid(), individual, study.getVariableSets(), options); + OpenCGAResult insert = getIndividualDBAdaptor(organizationId).insert(study.getUid(), individual, + study.getVariableSets(), options); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { // Fetch created individual - OpenCGAResult queryResult = getIndividual(study.getUid(), individual.getUuid(), options); + OpenCGAResult queryResult = getIndividual(organizationId, study.getUid(), individual.getUuid(), options); insert.setResults(queryResult.getResults()); } - auditManager.auditCreate(userId, Enums.Resource.INDIVIDUAL, individual.getId(), individual.getUuid(), study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.auditCreate(organizationId, userId, Enums.Resource.INDIVIDUAL, individual.getId(), individual.getUuid(), + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return insert; } catch (CatalogException e) { - auditManager.auditCreate(userId, Enums.Resource.INDIVIDUAL, individual.getId(), "", study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditCreate(organizationId, userId, Enums.Resource.INDIVIDUAL, individual.getId(), "", study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } - private void checkSamplesNotInUseInOtherIndividual(Set sampleIds, long studyId, Long individualId) + private void checkSamplesNotInUseInOtherIndividual(String organizationId, Set sampleIds, long studyId, Long individualId) throws CatalogException { // Check if any of the existing samples already belong to an individual Query query = new Query() @@ -351,7 +359,7 @@ private void checkSamplesNotInUseInOtherIndividual(Set sampleIds, long stu .append(IndividualDBAdaptor.QueryParams.STUDY_UID.key(), studyId); QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList( IndividualDBAdaptor.QueryParams.SAMPLES.key(), IndividualDBAdaptor.QueryParams.UID.key())); - OpenCGAResult queryResult = individualDBAdaptor.get(query, options); + OpenCGAResult queryResult = getIndividualDBAdaptor(organizationId).get(query, options); if (queryResult.getNumResults() > 0) { // Check which of the samples are already associated to an individual List usedSamples = new ArrayList<>(); @@ -376,22 +384,25 @@ private void checkSamplesNotInUseInOtherIndividual(Set sampleIds, long stu } @Override - public DBIterator iterator(String studyStr, Query query, QueryOptions options, String sessionId) throws CatalogException { - ParamUtils.checkObj(sessionId, "sessionId"); + public DBIterator iterator(String studyStr, Query query, QueryOptions options, String token) throws CatalogException { + ParamUtils.checkObj(token, "token"); query = ParamUtils.defaultObject(query, Query::new); options = ParamUtils.defaultObject(options, QueryOptions::new); - String userId = userManager.getUserId(sessionId); - Study study = catalogManager.getStudyManager().resolveId(studyStr, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyStr, userId, organizationId); Query finalQuery = new Query(query); - fixQuery(study, finalQuery, userId); + fixQuery(organizationId, study, finalQuery, userId); // Fix query if it contains any annotation - AnnotationUtils.fixQueryAnnotationSearch(study, finalQuery); + AnnotationUtils.fixQueryAnnotationSearch(organizationId, study, finalQuery); AnnotationUtils.fixQueryOptionAnnotation(options); finalQuery.append(IndividualDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - return individualDBAdaptor.iterator(study.getUid(), finalQuery, options, userId); + return getIndividualDBAdaptor(organizationId).iterator(study.getUid(), finalQuery, options, userId); } @Override @@ -400,9 +411,12 @@ public OpenCGAResult search(String studyId, Query query, QueryOption Query finalQuery = new Query(query); options = ParamUtils.defaultObject(options, QueryOptions::new); - String userId = userManager.getUserId(token); - Study study = catalogManager.getStudyManager().resolveId(studyId, userId, new QueryOptions(QueryOptions.INCLUDE, - StudyDBAdaptor.QueryParams.VARIABLE_SET.key())); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyId, new QueryOptions(QueryOptions.INCLUDE, + StudyDBAdaptor.QueryParams.VARIABLE_SET.key()), userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) @@ -410,22 +424,22 @@ public OpenCGAResult search(String studyId, Query query, QueryOption .append("options", options) .append("token", token); try { - fixQuery(study, finalQuery, userId); + fixQuery(organizationId, study, finalQuery, userId); // Fix query if it contains any annotation - AnnotationUtils.fixQueryAnnotationSearch(study, finalQuery); + AnnotationUtils.fixQueryAnnotationSearch(organizationId, study, finalQuery); AnnotationUtils.fixQueryOptionAnnotation(options); finalQuery.append(IndividualDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - OpenCGAResult queryResult = individualDBAdaptor.get(study.getUid(), finalQuery, options, userId); + OpenCGAResult queryResult = getIndividualDBAdaptor(organizationId).get(study.getUid(), finalQuery, options, userId); - auditManager.auditSearch(userId, Enums.Resource.INDIVIDUAL, study.getId(), study.getUuid(), auditParams, + auditManager.auditSearch(organizationId, userId, Enums.Resource.INDIVIDUAL, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return queryResult; } catch (CatalogException e) { - auditManager.auditSearch(userId, Enums.Resource.INDIVIDUAL, study.getId(), study.getUuid(), auditParams, + auditManager.auditSearch(organizationId, userId, Enums.Resource.INDIVIDUAL, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -435,9 +449,12 @@ public OpenCGAResult search(String studyId, Query query, QueryOption public OpenCGAResult distinct(String studyId, List fields, Query query, String token) throws CatalogException { query = ParamUtils.defaultObject(query, Query::new); - String userId = userManager.getUserId(token); - Study study = catalogManager.getStudyManager().resolveId(studyId, userId, new QueryOptions(QueryOptions.INCLUDE, - StudyDBAdaptor.QueryParams.VARIABLE_SET.key())); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyId, new QueryOptions(QueryOptions.INCLUDE, + StudyDBAdaptor.QueryParams.VARIABLE_SET.key()), userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) @@ -445,19 +462,19 @@ public OpenCGAResult distinct(String studyId, List fields, Query quer .append("query", new Query(query)) .append("token", token); try { - fixQuery(study, query, userId); + fixQuery(organizationId, study, query, userId); // Fix query if it contains any annotation - AnnotationUtils.fixQueryAnnotationSearch(study, query); + AnnotationUtils.fixQueryAnnotationSearch(organizationId, study, query); query.append(IndividualDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - OpenCGAResult result = individualDBAdaptor.distinct(study.getUid(), fields, query, userId); + OpenCGAResult result = getIndividualDBAdaptor(organizationId).distinct(study.getUid(), fields, query, userId); - auditManager.auditDistinct(userId, Enums.Resource.INDIVIDUAL, study.getId(), study.getUuid(), auditParams, + auditManager.auditDistinct(organizationId, userId, Enums.Resource.INDIVIDUAL, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return result; } catch (CatalogException e) { - auditManager.auditDistinct(userId, Enums.Resource.INDIVIDUAL, study.getId(), study.getUuid(), auditParams, + auditManager.auditDistinct(organizationId, userId, Enums.Resource.INDIVIDUAL, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -468,29 +485,32 @@ public OpenCGAResult count(String studyId, Query query, String token query = ParamUtils.defaultObject(query, Query::new); Query finalQuery = new Query(query); - String userId = userManager.getUserId(token); - Study study = catalogManager.getStudyManager().resolveId(studyId, userId, new QueryOptions(QueryOptions.INCLUDE, - StudyDBAdaptor.QueryParams.VARIABLE_SET.key())); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyId, new QueryOptions(QueryOptions.INCLUDE, + StudyDBAdaptor.QueryParams.VARIABLE_SET.key()), userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) .append("query", query) .append("token", token); try { - fixQuery(study, finalQuery, userId); + fixQuery(organizationId, study, finalQuery, userId); // Fix query if it contains any annotation - AnnotationUtils.fixQueryAnnotationSearch(study, finalQuery); + AnnotationUtils.fixQueryAnnotationSearch(organizationId, study, finalQuery); finalQuery.append(IndividualDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - OpenCGAResult queryResultAux = individualDBAdaptor.count(finalQuery, userId); + OpenCGAResult queryResultAux = getIndividualDBAdaptor(organizationId).count(finalQuery, userId); - auditManager.auditCount(userId, Enums.Resource.INDIVIDUAL, study.getId(), study.getUuid(), auditParams, + auditManager.auditCount(organizationId, userId, Enums.Resource.INDIVIDUAL, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return new OpenCGAResult<>(queryResultAux.getTime(), queryResultAux.getEvents(), 0, Collections.emptyList(), queryResultAux.getNumMatches()); } catch (CatalogException e) { - auditManager.auditCount(userId, Enums.Resource.INDIVIDUAL, study.getId(), study.getUuid(), auditParams, + auditManager.auditCount(organizationId, userId, Enums.Resource.INDIVIDUAL, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -498,8 +518,11 @@ public OpenCGAResult count(String studyId, Query query, String token public OpenCGAResult relatives(String studyId, String individualId, int degree, QueryOptions options, String token) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(token); - Study study = studyManager.resolveId(studyId, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyId, userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) @@ -512,36 +535,36 @@ public OpenCGAResult relatives(String studyId, String individualId, try { long startTime = System.currentTimeMillis(); - QueryOptions queryOptions = individualDBAdaptor.fixOptionsForRelatives(options); + QueryOptions queryOptions = getIndividualDBAdaptor(organizationId).fixOptionsForRelatives(options); if (degree < 0 || degree > 2) { throw new CatalogException("Unsupported degree value. Degree must be 0, 1 or 2"); } List individualList = new LinkedList<>(); - Individual proband = internalGet(study.getUid(), individualId, queryOptions, userId).first(); - individualDBAdaptor.addRelativeToList(proband, Family.FamiliarRelationship.PROBAND, 0, individualList); + Individual proband = internalGet(organizationId, study.getUid(), individualId, queryOptions, userId).first(); + getIndividualDBAdaptor(organizationId).addRelativeToList(proband, Family.FamiliarRelationship.PROBAND, 0, individualList); individualId = proband.getId(); individualUuid = proband.getUuid(); if (degree == 0) { - auditManager.audit(userId, Enums.Action.RELATIVES, Enums.Resource.INDIVIDUAL, individualId, individualUuid, study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.audit(organizationId, userId, Enums.Action.RELATIVES, Enums.Resource.INDIVIDUAL, individualId, individualUuid, + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return new OpenCGAResult<>((int) (System.currentTimeMillis() - startTime), Collections.emptyList(), individualList.size(), individualList, individualList.size()); } - individualList.addAll(individualDBAdaptor.calculateRelationship(study.getUid(), proband, degree, userId)); + individualList.addAll(getIndividualDBAdaptor(organizationId).calculateRelationship(study.getUid(), proband, degree, userId)); - auditManager.audit(userId, Enums.Action.RELATIVES, Enums.Resource.INDIVIDUAL, individualId, individualUuid, study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.audit(organizationId, userId, Enums.Action.RELATIVES, Enums.Resource.INDIVIDUAL, individualId, individualUuid, + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return new OpenCGAResult<>((int) (System.currentTimeMillis() - startTime), Collections.emptyList(), individualList.size(), individualList, individualList.size()); } catch (CatalogException e) { - auditManager.audit(userId, Enums.Action.RELATIVES, Enums.Resource.INDIVIDUAL, individualId, individualUuid, study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.audit(organizationId, userId, Enums.Action.RELATIVES, Enums.Resource.INDIVIDUAL, individualId, individualUuid, + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -554,9 +577,12 @@ public OpenCGAResult delete(String studyStr, List individualIds, QueryOp public OpenCGAResult delete(String studyStr, List individualIds, ObjectMap params, boolean ignoreException, String token) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(token); - Study study = studyManager.resolveId(studyStr, userId, new QueryOptions(QueryOptions.INCLUDE, - StudyDBAdaptor.QueryParams.VARIABLE_SET.key())); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, new QueryOptions(QueryOptions.INCLUDE, + StudyDBAdaptor.QueryParams.VARIABLE_SET.key()), userId, organizationId); String operationUuid = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -570,10 +596,11 @@ public OpenCGAResult delete(String studyStr, List individualIds, ObjectM boolean checkPermissions; try { // If the user is the owner or the admin, we won't check if he has permissions for every single entry - checkPermissions = !authorizationManager.isOwnerOrAdmin(study.getUid(), userId); + long studyId = study.getUid(); + checkPermissions = !authorizationManager.isAtLeastStudyAdministrator(organizationId, studyId, userId); } catch (CatalogException e) { - auditManager.auditDelete(operationUuid, userId, Enums.Resource.INDIVIDUAL, "", "", study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.INDIVIDUAL, "", "", study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -584,7 +611,7 @@ public OpenCGAResult delete(String studyStr, List individualIds, ObjectM String individualUuid = ""; try { - OpenCGAResult internalResult = internalGet(study.getUid(), id, INCLUDE_INDIVIDUAL_IDS, userId); + OpenCGAResult internalResult = internalGet(organizationId, study.getUid(), id, INCLUDE_INDIVIDUAL_IDS, userId); if (internalResult.getNumResults() == 0) { throw new CatalogException("Individual '" + id + "' not found"); } @@ -594,13 +621,14 @@ public OpenCGAResult delete(String studyStr, List individualIds, ObjectM individualId = individual.getId(); individualUuid = individual.getUuid(); - OpenCGAResult deleteResult = delete(study, individual, params, userId, checkPermissions); + OpenCGAResult deleteResult = delete(organizationId, study, individual, params, userId, checkPermissions); // Add the results to the current write result result.append(deleteResult); - auditManager.auditDelete(operationUuid, userId, Enums.Resource.INDIVIDUAL, individual.getId(), individual.getUuid(), - study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.INDIVIDUAL, individual.getId(), + individual.getUuid(), study.getId(), study.getUuid(), auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { String errorMsg = "Cannot delete individual " + individualId + ": " + e.getMessage(); @@ -609,11 +637,11 @@ public OpenCGAResult delete(String studyStr, List individualIds, ObjectM result.setNumErrors(result.getNumErrors() + 1); logger.error(errorMsg, e); - auditManager.auditDelete(operationUuid, userId, Enums.Resource.INDIVIDUAL, individualId, individualUuid, + auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.INDIVIDUAL, individualId, individualUuid, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); } } - auditManager.finishAuditBatch(operationUuid); + auditManager.finishAuditBatch(organizationId, operationUuid); return endResult(result, ignoreException); } @@ -628,8 +656,11 @@ public OpenCGAResult delete(String studyStr, Query query, ObjectMap params, bool Query finalQuery = new Query(ParamUtils.defaultObject(query, Query::new)); OpenCGAResult result = OpenCGAResult.empty(); - String userId = catalogManager.getUserManager().getUserId(token); - Study study = catalogManager.getStudyManager().resolveId(studyStr, userId, StudyManager.INCLUDE_VARIABLE_SET); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyStr, StudyManager.INCLUDE_VARIABLE_SET, userId, organizationId); String operationUuid = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -647,19 +678,20 @@ public OpenCGAResult delete(String studyStr, Query query, ObjectMap params, bool DBIterator iterator; try { // Fix query if it contains any annotation - fixQuery(study, finalQuery, userId); + fixQuery(organizationId, study, finalQuery, userId); // Fix query if it contains any annotation - AnnotationUtils.fixQueryAnnotationSearch(study, finalQuery); + AnnotationUtils.fixQueryAnnotationSearch(organizationId, study, finalQuery); finalQuery.append(FamilyDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - iterator = individualDBAdaptor.iterator(study.getUid(), finalQuery, INCLUDE_INDIVIDUAL_IDS, userId); + iterator = getIndividualDBAdaptor(organizationId).iterator(study.getUid(), finalQuery, INCLUDE_INDIVIDUAL_IDS, userId); // If the user is the owner or the admin, we won't check if he has permissions for every single entry - checkPermissions = !authorizationManager.isOwnerOrAdmin(study.getUid(), userId); + long studyId = study.getUid(); + checkPermissions = !authorizationManager.isAtLeastStudyAdministrator(organizationId, studyId, userId); } catch (CatalogException e) { - auditManager.auditDelete(operationUuid, userId, Enums.Resource.INDIVIDUAL, "", "", study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.INDIVIDUAL, "", "", study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -668,13 +700,14 @@ public OpenCGAResult delete(String studyStr, Query query, ObjectMap params, bool Individual individual = iterator.next(); try { - OpenCGAResult deleteResult = delete(study, individual, params, userId, checkPermissions); + OpenCGAResult deleteResult = delete(organizationId, study, individual, params, userId, checkPermissions); // Add the results to the current write result result.append(deleteResult); - auditManager.auditDelete(operationUuid, userId, Enums.Resource.INDIVIDUAL, individual.getId(), individual.getUuid(), - study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.INDIVIDUAL, individual.getId(), + individual.getUuid(), study.getId(), study.getUuid(), auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { String errorMsg = "Cannot delete individual " + individual.getId() + ": " + e.getMessage(); @@ -683,19 +716,20 @@ public OpenCGAResult delete(String studyStr, Query query, ObjectMap params, bool result.setNumErrors(result.getNumErrors() + 1); logger.error(errorMsg); - auditManager.auditDelete(operationUuid, userId, Enums.Resource.INDIVIDUAL, individual.getId(), individual.getUuid(), - study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.INDIVIDUAL, individual.getId(), + individual.getUuid(), study.getId(), study.getUuid(), auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); } } - auditManager.finishAuditBatch(operationUuid); + auditManager.finishAuditBatch(organizationId, operationUuid); return endResult(result, ignoreException); } - private OpenCGAResult delete(Study study, Individual individual, ObjectMap params, String userId, boolean checkPermissions) - throws CatalogException { + private OpenCGAResult delete(String organizationId, Study study, Individual individual, ObjectMap params, String userId, + boolean checkPermissions) throws CatalogException { if (checkPermissions) { - authorizationManager.checkIndividualPermission(study.getUid(), individual.getUid(), userId, + authorizationManager.checkIndividualPermission(organizationId, study.getUid(), individual.getUid(), userId, IndividualPermissions.DELETE); } @@ -703,7 +737,7 @@ private OpenCGAResult delete(Study study, Individual individual, ObjectMap param Query tmpQuery = new Query() .append(FamilyDBAdaptor.QueryParams.MEMBER_UID.key(), individual.getUid()) .append(FamilyDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - OpenCGAResult familyDataResult = familyDBAdaptor.get(tmpQuery, new QueryOptions(QueryOptions.INCLUDE, + OpenCGAResult familyDataResult = getFamilyDBAdaptor(organizationId).get(tmpQuery, new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(FamilyDBAdaptor.QueryParams.UID.key(), FamilyDBAdaptor.QueryParams.ID.key(), FamilyDBAdaptor.QueryParams.MEMBERS.key()))); @@ -717,7 +751,7 @@ private OpenCGAResult delete(Study study, Individual individual, ObjectMap param logger.info("Forcing deletion of individuals belonging to families"); } - return individualDBAdaptor.delete(individual); + return getIndividualDBAdaptor(organizationId).delete(individual); } public OpenCGAResult updateAnnotationSet(String studyStr, String individualStr, List annotationSetList, @@ -747,7 +781,8 @@ public OpenCGAResult setAnnotationSet(String studyStr, String indivi public OpenCGAResult setAnnotationSets(String studyStr, String individualStr, List annotationSetList, QueryOptions options, String token) throws CatalogException { - return updateAnnotationSet(studyStr, individualStr, annotationSetList, ParamUtils.BasicUpdateAction.SET, options, token); + return updateAnnotationSet(studyStr, individualStr, annotationSetList, ParamUtils.BasicUpdateAction.SET, options, + token); } public OpenCGAResult removeAnnotationSet(String studyStr, String individualStr, String annotationSetId, @@ -761,7 +796,8 @@ public OpenCGAResult removeAnnotationSets(String studyStr, String in .stream() .map(id -> new AnnotationSet().setId(id)) .collect(Collectors.toList()); - return updateAnnotationSet(studyStr, individualStr, annotationSetList, ParamUtils.BasicUpdateAction.REMOVE, options, token); + return updateAnnotationSet(studyStr, individualStr, annotationSetList, ParamUtils.BasicUpdateAction.REMOVE, options, + token); } public OpenCGAResult updateAnnotations(String studyStr, String individualStr, String annotationSetId, @@ -781,15 +817,15 @@ public OpenCGAResult updateAnnotations(String studyStr, String indiv public OpenCGAResult removeAnnotations(String studyStr, String individualStr, String annotationSetId, List annotations, QueryOptions options, String token) throws CatalogException { - return updateAnnotations(studyStr, individualStr, annotationSetId, new ObjectMap("remove", StringUtils.join(annotations, ",")), - ParamUtils.CompleteUpdateAction.REMOVE, options, token); + return updateAnnotations(studyStr, individualStr, annotationSetId, + new ObjectMap("remove", StringUtils.join(annotations, ",")), ParamUtils.CompleteUpdateAction.REMOVE, options, token); } public OpenCGAResult resetAnnotations(String studyStr, String individualStr, String annotationSetId, List annotations, QueryOptions options, String token) throws CatalogException { - return updateAnnotations(studyStr, individualStr, annotationSetId, new ObjectMap("reset", StringUtils.join(annotations, ",")), - ParamUtils.CompleteUpdateAction.RESET, options, token); + return updateAnnotations(studyStr, individualStr, annotationSetId, + new ObjectMap("reset", StringUtils.join(annotations, ",")), ParamUtils.CompleteUpdateAction.RESET, options, token); } public OpenCGAResult update(String studyStr, Query query, IndividualUpdateParams updateParams, QueryOptions options, @@ -799,8 +835,11 @@ public OpenCGAResult update(String studyStr, Query query, Individual public OpenCGAResult update(String studyStr, Query query, IndividualUpdateParams updateParams, boolean ignoreException, QueryOptions options, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId, StudyManager.INCLUDE_VARIABLE_SET); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, StudyManager.INCLUDE_VARIABLE_SET, userId, organizationId); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -823,16 +862,16 @@ public OpenCGAResult update(String studyStr, Query query, Individual DBIterator iterator; try { - fixQuery(study, finalQuery, userId); + fixQuery(organizationId, study, finalQuery, userId); // Fix query if it contains any annotation - AnnotationUtils.fixQueryAnnotationSearch(study, finalQuery); + AnnotationUtils.fixQueryAnnotationSearch(organizationId, study, finalQuery); finalQuery.append(IndividualDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - iterator = individualDBAdaptor.iterator(study.getUid(), finalQuery, INCLUDE_INDIVIDUAL_IDS, userId); + iterator = getIndividualDBAdaptor(organizationId).iterator(study.getUid(), finalQuery, INCLUDE_INDIVIDUAL_IDS, userId); } catch (CatalogException e) { - auditManager.auditUpdate(operationId, userId, Enums.Resource.INDIVIDUAL, "", "", study.getId(), study.getUuid(), + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.INDIVIDUAL, "", "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -842,30 +881,34 @@ public OpenCGAResult update(String studyStr, Query query, Individual while (iterator.hasNext()) { Individual individual = iterator.next(); try { - OpenCGAResult updateResult = update(study, individual, updateParams, options, userId); + OpenCGAResult updateResult = update(organizationId, study, individual, updateParams, options, userId); result.append(updateResult); - auditManager.auditUpdate(userId, Enums.Resource.INDIVIDUAL, individual.getId(), individual.getUuid(), study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.auditUpdate(organizationId, userId, Enums.Resource.INDIVIDUAL, individual.getId(), individual.getUuid(), + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { Event event = new Event(Event.Type.ERROR, individual.getId(), e.getMessage()); result.getEvents().add(event); result.setNumErrors(result.getNumErrors() + 1); logger.error("Cannot update individual {}: {}", individual.getId(), e.getMessage(), e); - auditManager.auditUpdate(operationId, userId, Enums.Resource.INDIVIDUAL, individual.getId(), individual.getUuid(), - study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.INDIVIDUAL, individual.getId(), + individual.getUuid(), study.getId(), study.getUuid(), auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); } } - auditManager.finishAuditBatch(operationId); + auditManager.finishAuditBatch(organizationId, operationId); return endResult(result, ignoreException); } - public OpenCGAResult update(String studyStr, String individualId, IndividualUpdateParams updateParams, QueryOptions options, - String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId, StudyManager.INCLUDE_VARIABLE_SET); + public OpenCGAResult update(String studyStr, String individualId, IndividualUpdateParams updateParams, + QueryOptions options, String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, StudyManager.INCLUDE_VARIABLE_SET, userId, organizationId); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -887,7 +930,8 @@ public OpenCGAResult update(String studyStr, String individualId, In String individualUuid = ""; try { - OpenCGAResult internalResult = internalGet(study.getUid(), individualId, QueryOptions.empty(), userId); + OpenCGAResult internalResult = internalGet(organizationId, study.getUid(), individualId, QueryOptions.empty(), + userId); if (internalResult.getNumResults() == 0) { throw new CatalogException("Individual '" + individualId + "' not found"); } @@ -897,19 +941,19 @@ public OpenCGAResult update(String studyStr, String individualId, In individualId = individual.getId(); individualUuid = individual.getUuid(); - OpenCGAResult updateResult = update(study, individual, updateParams, options, userId); + OpenCGAResult updateResult = update(organizationId, study, individual, updateParams, options, userId); result.append(updateResult); - auditManager.auditUpdate(userId, Enums.Resource.INDIVIDUAL, individual.getId(), individual.getUuid(), study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.auditUpdate(organizationId, userId, Enums.Resource.INDIVIDUAL, individual.getId(), individual.getUuid(), + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { Event event = new Event(Event.Type.ERROR, individualId, e.getMessage()); result.getEvents().add(event); result.setNumErrors(result.getNumErrors() + 1); logger.error("Cannot update individual {}: {}", individualId, e.getMessage()); - auditManager.auditUpdate(operationId, userId, Enums.Resource.INDIVIDUAL, individualId, individualUuid, study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.INDIVIDUAL, individualId, individualUuid, + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -919,7 +963,8 @@ public OpenCGAResult update(String studyStr, String individualId, In /** * Update an Individual from catalog. * - * @param studyStr Study id in string format. Could be one of [id|user@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy]. + * @param studyStr Study id in string format. Could be one of + * [id|organization@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy] * @param individualIds List of individual ids. Could be either the id or uuid. * @param updateParams Data model filled only with the parameters to be updated. * @param options QueryOptions object. @@ -935,8 +980,11 @@ public OpenCGAResult update(String studyStr, List individual public OpenCGAResult update(String studyStr, List individualIds, IndividualUpdateParams updateParams, boolean ignoreException, QueryOptions options, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId, StudyManager.INCLUDE_VARIABLE_SET); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, StudyManager.INCLUDE_VARIABLE_SET, userId, organizationId); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -962,7 +1010,7 @@ public OpenCGAResult update(String studyStr, List individual String individualUuid = ""; try { - OpenCGAResult internalResult = internalGet(study.getUid(), id, QueryOptions.empty(), userId); + OpenCGAResult internalResult = internalGet(organizationId, study.getUid(), id, QueryOptions.empty(), userId); if (internalResult.getNumResults() == 0) { throw new CatalogException("Individual '" + id + "' not found"); } @@ -972,28 +1020,28 @@ public OpenCGAResult update(String studyStr, List individual individualId = individual.getId(); individualUuid = individual.getUuid(); - OpenCGAResult updateResult = update(study, individual, updateParams, options, userId); + OpenCGAResult updateResult = update(organizationId, study, individual, updateParams, options, userId); result.append(updateResult); - auditManager.auditUpdate(userId, Enums.Resource.INDIVIDUAL, individual.getId(), individual.getUuid(), study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.auditUpdate(organizationId, userId, Enums.Resource.INDIVIDUAL, individual.getId(), individual.getUuid(), + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { Event event = new Event(Event.Type.ERROR, id, e.getMessage()); result.getEvents().add(event); result.setNumErrors(result.getNumErrors() + 1); logger.error("Cannot update individual {}: {}", individualId, e.getMessage()); - auditManager.auditUpdate(operationId, userId, Enums.Resource.INDIVIDUAL, individualId, individualUuid, study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.INDIVIDUAL, individualId, individualUuid, + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); } } - auditManager.finishAuditBatch(operationId); + auditManager.finishAuditBatch(organizationId, operationId); return endResult(result, ignoreException); } - private OpenCGAResult update(Study study, Individual individual, IndividualUpdateParams updateParams, QueryOptions options, - String userId) throws CatalogException { + private OpenCGAResult update(String organizationId, Study study, Individual individual, IndividualUpdateParams updateParams, + QueryOptions options, String userId) throws CatalogException { ObjectMap parameters = new ObjectMap(); if (updateParams != null) { try { @@ -1032,13 +1080,13 @@ private OpenCGAResult update(Study study, Individual individual, IndividualUpdat // Check permissions... // Only check write annotation permissions if the user wants to update the annotation sets if (updateParams != null && updateParams.getAnnotationSets() != null) { - authorizationManager.checkIndividualPermission(studyUid, individualUid, userId, + authorizationManager.checkIndividualPermission(organizationId, studyUid, individualUid, userId, IndividualPermissions.WRITE_ANNOTATIONS); } // Only check update permissions if the user wants to update anything apart from the annotation sets if ((parameters.size() == 1 && !parameters.containsKey(IndividualDBAdaptor.QueryParams.ANNOTATION_SETS.key())) || parameters.size() > 1) { - authorizationManager.checkIndividualPermission(studyUid, individualUid, userId, + authorizationManager.checkIndividualPermission(organizationId, studyUid, individualUid, userId, IndividualPermissions.WRITE); } @@ -1048,7 +1096,7 @@ private OpenCGAResult update(Study study, Individual individual, IndividualUpdat Query query = new Query() .append(IndividualDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(IndividualDBAdaptor.QueryParams.ID.key(), updateParams.getId()); - if (individualDBAdaptor.count(query).getNumMatches() > 0) { + if (getIndividualDBAdaptor(organizationId).count(query).getNumMatches() > 0) { throw new CatalogException("Individual id " + updateParams.getId() + " already in use"); } } @@ -1073,12 +1121,12 @@ private OpenCGAResult update(Study study, Individual individual, IndividualUpdat throw new CatalogException("Missing id or uuid in 'samples'"); } } - List sampleList = catalogManager.getSampleManager().internalGet(studyUid, idList, SampleManager.INCLUDE_SAMPLE_IDS, - userId, false).getResults(); + List sampleList = catalogManager.getSampleManager().internalGet(organizationId, studyUid, idList, + SampleManager.INCLUDE_SAMPLE_IDS, userId, false).getResults(); // Check those samples are not in use by other individuals - checkSamplesNotInUseInOtherIndividual(sampleList.stream().map(Sample::getUid).collect(Collectors.toSet()), studyUid, - individualUid); + checkSamplesNotInUseInOtherIndividual(organizationId, sampleList.stream().map(Sample::getUid).collect(Collectors.toSet()), + studyUid, individualUid); // Pass the DBAdaptor the corresponding list of Sample objects parameters.put(IndividualDBAdaptor.QueryParams.SAMPLES.key(), sampleList); @@ -1088,7 +1136,7 @@ private OpenCGAResult update(Study study, Individual individual, IndividualUpdat if (StringUtils.isNotEmpty(updateParams.getFather().getId()) || StringUtils.isNotEmpty(updateParams.getFather().getUuid())) { String fatherId = StringUtils.isNotEmpty(updateParams.getFather().getId()) ? updateParams.getFather().getId() : updateParams.getFather().getUuid(); - OpenCGAResult queryResult = internalGet(studyUid, fatherId, INCLUDE_INDIVIDUAL_IDS, userId); + OpenCGAResult queryResult = internalGet(organizationId, studyUid, fatherId, INCLUDE_INDIVIDUAL_IDS, userId); parameters.put(IndividualDBAdaptor.QueryParams.FATHER_UID.key(), queryResult.first().getUid()); } else { parameters.put(IndividualDBAdaptor.QueryParams.FATHER_UID.key(), -1L); @@ -1099,7 +1147,7 @@ private OpenCGAResult update(Study study, Individual individual, IndividualUpdat if (StringUtils.isNotEmpty(updateParams.getMother().getId()) || StringUtils.isNotEmpty(updateParams.getMother().getUuid())) { String motherId = StringUtils.isNotEmpty(updateParams.getMother().getId()) ? updateParams.getMother().getId() : updateParams.getMother().getUuid(); - OpenCGAResult queryResult = internalGet(studyUid, motherId, INCLUDE_INDIVIDUAL_IDS, userId); + OpenCGAResult queryResult = internalGet(organizationId, studyUid, motherId, INCLUDE_INDIVIDUAL_IDS, userId); parameters.put(IndividualDBAdaptor.QueryParams.MOTHER_UID.key(), queryResult.first().getUid()); } else { parameters.put(IndividualDBAdaptor.QueryParams.MOTHER_UID.key(), -1L); @@ -1107,13 +1155,14 @@ private OpenCGAResult update(Study study, Individual individual, IndividualUpdat parameters.remove(IndividualDBAdaptor.QueryParams.MOTHER.key()); } - checkUpdateAnnotations(study, individual, parameters, options, VariableSet.AnnotableDataModels.INDIVIDUAL, individualDBAdaptor, - userId); + checkUpdateAnnotations(organizationId, study, individual, parameters, options, VariableSet.AnnotableDataModels.INDIVIDUAL, + getIndividualDBAdaptor(organizationId), userId); - OpenCGAResult update = individualDBAdaptor.update(individual.getUid(), parameters, study.getVariableSets(), options); + OpenCGAResult update = getIndividualDBAdaptor(organizationId).update(individual.getUid(), parameters, + study.getVariableSets(), options); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { // Fetch updated individual - OpenCGAResult result = individualDBAdaptor.get(study.getUid(), + OpenCGAResult result = getIndividualDBAdaptor(organizationId).get(study.getUid(), new Query(IndividualDBAdaptor.QueryParams.UID.key(), individual.getUid()), options, userId); update.setResults(result.getResults()); } @@ -1121,19 +1170,22 @@ private OpenCGAResult update(Study study, Individual individual, IndividualUpdat } @Override - public OpenCGAResult rank(String studyStr, Query query, String field, int numResults, boolean asc, String sessionId) + public OpenCGAResult rank(String studyStr, Query query, String field, int numResults, boolean asc, String token) throws CatalogException { query = ParamUtils.defaultObject(query, Query::new); ParamUtils.checkObj(field, "field"); - ParamUtils.checkObj(sessionId, "sessionId"); + ParamUtils.checkObj(token, "sessionId"); - String userId = userManager.getUserId(sessionId); - Study study = catalogManager.getStudyManager().resolveId(studyStr, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyStr, userId, organizationId); - authorizationManager.checkStudyPermission(study.getUid(), userId, StudyPermissions.Permissions.VIEW_INDIVIDUALS); + authorizationManager.checkStudyPermission(organizationId, study.getUid(), userId, StudyPermissions.Permissions.VIEW_INDIVIDUALS); // Fix query if it contains any annotation - AnnotationUtils.fixQueryAnnotationSearch(study, userId, query, authorizationManager); + AnnotationUtils.fixQueryAnnotationSearch(organizationId, study, userId, query, authorizationManager); // TODO: In next release, we will have to check the count parameter from the queryOptions object. boolean count = true; @@ -1141,30 +1193,33 @@ public OpenCGAResult rank(String studyStr, Query query, String field, int numRes OpenCGAResult queryResult = null; if (count) { // We do not need to check for permissions when we show the count of files - queryResult = individualDBAdaptor.rank(query, field, numResults, asc); + queryResult = getIndividualDBAdaptor(organizationId).rank(query, field, numResults, asc); } return ParamUtils.defaultObject(queryResult, OpenCGAResult::new); } @Override - public OpenCGAResult groupBy(@Nullable String studyStr, Query query, List fields, QueryOptions options, String sessionId) + public OpenCGAResult groupBy(@Nullable String studyStr, Query query, List fields, QueryOptions options, String token) throws CatalogException { query = ParamUtils.defaultObject(query, Query::new); options = ParamUtils.defaultObject(options, QueryOptions::new); ParamUtils.checkObj(fields, "fields"); - String userId = userManager.getUserId(sessionId); - Study study = catalogManager.getStudyManager().resolveId(studyStr, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyStr, userId, organizationId); Query finalQuery = new Query(query); // Fix query if it contains any annotation - AnnotationUtils.fixQueryAnnotationSearch(study, userId, finalQuery, authorizationManager); + AnnotationUtils.fixQueryAnnotationSearch(organizationId, study, userId, finalQuery, authorizationManager); AnnotationUtils.fixQueryOptionAnnotation(options); try { - fixQuery(study, finalQuery, userId); + fixQuery(organizationId, study, finalQuery, userId); } catch (CatalogException e) { // Any of mother, father or sample ids or names do not exist or were not found return OpenCGAResult.empty(); @@ -1173,26 +1228,27 @@ public OpenCGAResult groupBy(@Nullable String studyStr, Query query, List> getAcls(String studyId, List individualList, - String member, boolean ignoreException, - String token) throws CatalogException { + public OpenCGAResult> getAcls(String studyId, List individualList, String member, + boolean ignoreException, String token) throws CatalogException { return getAcls(studyId, individualList, StringUtils.isNotEmpty(member) ? Collections.singletonList(member) : Collections.emptyList(), ignoreException, token); } - public OpenCGAResult> getAcls(String studyId, List individualList, - List members, boolean ignoreException, - String token) throws CatalogException { - String user = userManager.getUserId(token); - Study study = studyManager.resolveId(studyId, user); + public OpenCGAResult> getAcls(String studyId, List individualList, List members, + boolean ignoreException, String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyId, userId, organizationId); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); ObjectMap auditParams = new ObjectMap() @@ -1206,8 +1262,8 @@ public OpenCGAResult> getAcls(String studyId Map missingMap = new HashMap<>(); try { auditManager.initAuditBatch(operationId); - InternalGetDataResult queryResult = internalGet(study.getUid(), individualList, INCLUDE_INDIVIDUAL_IDS, user, - ignoreException); + InternalGetDataResult queryResult = internalGet(organizationId, study.getUid(), individualList, + INCLUDE_INDIVIDUAL_IDS, userId, ignoreException); if (queryResult.getMissing() != null) { missingMap = queryResult.getMissing().stream() @@ -1216,11 +1272,11 @@ public OpenCGAResult> getAcls(String studyId List individualUids = queryResult.getResults().stream().map(Individual::getUid).collect(Collectors.toList()); if (CollectionUtils.isNotEmpty(members)) { - individualAcls = authorizationManager.getAcl(user, study.getUid(), individualUids, members, Enums.Resource.INDIVIDUAL, - IndividualPermissions.class); + individualAcls = authorizationManager.getAcl(organizationId, study.getUid(), individualUids, members, + Enums.Resource.INDIVIDUAL, IndividualPermissions.class, userId); } else { - individualAcls = authorizationManager.getAcl(user, study.getUid(), individualUids, Enums.Resource.INDIVIDUAL, - IndividualPermissions.class); + individualAcls = authorizationManager.getAcl(organizationId, study.getUid(), individualUids, Enums.Resource.INDIVIDUAL, + IndividualPermissions.class, userId); } // Include non-existing samples to the result list @@ -1231,16 +1287,17 @@ public OpenCGAResult> getAcls(String studyId if (!missingMap.containsKey(individualId)) { Individual individual = queryResult.getResults().get(counter); resultList.add(individualAcls.getResults().get(counter)); - auditManager.audit(operationId, user, Enums.Action.FETCH_ACLS, Enums.Resource.INDIVIDUAL, individual.getId(), - individual.getUuid(), study.getId(), study.getUuid(), auditParams, + auditManager.audit(organizationId, operationId, userId, Enums.Action.FETCH_ACLS, Enums.Resource.INDIVIDUAL, + individual.getId(), individual.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS), new ObjectMap()); counter++; } else { resultList.add(new AclEntryList<>()); eventList.add(new Event(Event.Type.ERROR, individualId, missingMap.get(individualId).getErrorMsg())); - auditManager.audit(operationId, user, Enums.Action.FETCH_ACLS, Enums.Resource.INDIVIDUAL, individualId, "", - study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, - new Error(0, "", missingMap.get(individualId).getErrorMsg())), new ObjectMap()); + auditManager.audit(organizationId, operationId, userId, Enums.Action.FETCH_ACLS, Enums.Resource.INDIVIDUAL, + individualId, "", study.getId(), study.getUuid(), auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(0, "", + missingMap.get(individualId).getErrorMsg())), new ObjectMap()); } } for (int i = 0; i < queryResult.getResults().size(); i++) { @@ -1250,9 +1307,9 @@ public OpenCGAResult> getAcls(String studyId individualAcls.setEvents(eventList); } catch (CatalogException e) { for (String individualId : individualList) { - auditManager.audit(operationId, user, Enums.Action.FETCH_ACLS, Enums.Resource.INDIVIDUAL, individualId, "", - study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError()), - new ObjectMap()); + auditManager.audit(organizationId, operationId, userId, Enums.Action.FETCH_ACLS, Enums.Resource.INDIVIDUAL, individualId, + "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, + e.getError()), new ObjectMap()); } if (!ignoreException) { throw e; @@ -1263,18 +1320,20 @@ public OpenCGAResult> getAcls(String studyId } } } finally { - auditManager.finishAuditBatch(operationId); + auditManager.finishAuditBatch(organizationId, operationId); } return individualAcls; } - public OpenCGAResult> updateAcl(String studyId, List individualStrList, - String memberList, IndividualAclParams aclParams, - ParamUtils.AclAction action, boolean propagate, - String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyId, userId, StudyManager.INCLUDE_STUDY_UID); + public OpenCGAResult> updateAcl(String studyId, List individualStrList, String memberList, + IndividualAclParams aclParams, ParamUtils.AclAction action, + boolean propagate, String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyId, StudyManager.INCLUDE_STUDY_UID, userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) @@ -1312,16 +1371,17 @@ public OpenCGAResult> updateAcl(String study if (StringUtils.isNotEmpty(aclParams.getSample())) { Query query = new Query(IndividualDBAdaptor.QueryParams.SAMPLES.key(), aclParams.getSample()); QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, IndividualDBAdaptor.QueryParams.ID.key()); - OpenCGAResult indDataResult = catalogManager.getIndividualManager().search(studyId, query, options, token); + OpenCGAResult indDataResult = catalogManager.getIndividualManager().search(studyId, query, + options, token); individualStrList = indDataResult.getResults().stream().map(Individual::getId).collect(Collectors.toList()); } // Obtain the resource ids - List individualList = internalGet(study.getUid(), individualStrList, INCLUDE_INDIVIDUAL_IDS, userId, false) - .getResults(); + List individualList = internalGet(organizationId, study.getUid(), individualStrList, INCLUDE_INDIVIDUAL_IDS, userId, + false).getResults(); - authorizationManager.checkCanAssignOrSeePermissions(study.getUid(), userId); + authorizationManager.checkCanAssignOrSeePermissions(organizationId, study.getUid(), userId); // Validate that the members are actually valid members List members; @@ -1331,111 +1391,73 @@ public OpenCGAResult> updateAcl(String study members = Collections.emptyList(); } authorizationManager.checkNotAssigningPermissionsToAdminsGroup(members); - checkMembers(study.getUid(), members); + checkMembers(organizationId, study.getUid(), members); List individualUids = individualList.stream().map(Individual::getUid).collect(Collectors.toList()); List aclParamsList = new LinkedList<>(); aclParamsList.add(new AuthorizationManager.CatalogAclParams(individualUids, permissions, Enums.Resource.INDIVIDUAL)); if (propagate) { - List sampleUids = getSampleUidsFromIndividuals(study.getUid(), individualUids); + List sampleUids = getSampleUidsFromIndividuals(organizationId, study.getUid(), individualUids); aclParamsList.add(new AuthorizationManager.CatalogAclParams(sampleUids, permissions, Enums.Resource.SAMPLE)); } switch (action) { case SET: - authorizationManager.setAcls(study.getUid(), members, aclParamsList); + authorizationManager.setAcls(organizationId, study.getUid(), members, aclParamsList); break; case ADD: - authorizationManager.addAcls(study.getUid(), members, aclParamsList); + authorizationManager.addAcls(organizationId, study.getUid(), members, aclParamsList); break; case REMOVE: - authorizationManager.removeAcls(members, aclParamsList); + authorizationManager.removeAcls(organizationId, members, aclParamsList); break; case RESET: for (AuthorizationManager.CatalogAclParams catalogAclParams : aclParamsList) { catalogAclParams.setPermissions(null); } - authorizationManager.removeAcls(members, aclParamsList); + authorizationManager.removeAcls(organizationId, members, aclParamsList); break; default: throw new CatalogException("Unexpected error occurred. No valid action found."); } OpenCGAResult> queryResults = authorizationManager - .getAcls(study.getUid(), individualUids, members, Enums.Resource.INDIVIDUAL, + .getAcls(organizationId, study.getUid(), individualUids, members, Enums.Resource.INDIVIDUAL, IndividualPermissions.class); for (int i = 0; i < queryResults.getResults().size(); i++) { queryResults.getResults().get(i).setId(individualList.get(i).getId()); } for (Individual individual : individualList) { - auditManager.audit(operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.INDIVIDUAL, individual.getId(), - individual.getUuid(), study.getId(), study.getUuid(), auditParams, + auditManager.audit(organizationId, operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.INDIVIDUAL, + individual.getId(), individual.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS), new ObjectMap()); } return queryResults; } catch (CatalogException e) { if (individualStrList != null) { for (String individualId : individualStrList) { - auditManager.audit(operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.INDIVIDUAL, individualId, - "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, - e.getError()), new ObjectMap()); + auditManager.audit(organizationId, operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.INDIVIDUAL, + individualId, "", study.getId(), study.getUuid(), auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError()), new ObjectMap()); } } throw e; } finally { - auditManager.finishAuditBatch(operationId); - } - } - - public DataResult facet(String studyId, Query query, QueryOptions options, boolean defaultStats, String token) - throws CatalogException, IOException { - ParamUtils.defaultObject(query, Query::new); - ParamUtils.defaultObject(options, QueryOptions::new); - - String userId = userManager.getUserId(token); - // We need to add variableSets and groups to avoid additional queries as it will be used in the catalogSolrManager - Study study = catalogManager.getStudyManager().resolveId(studyId, userId, new QueryOptions(QueryOptions.INCLUDE, - Arrays.asList(StudyDBAdaptor.QueryParams.VARIABLE_SET.key(), StudyDBAdaptor.QueryParams.GROUPS.key()))); - - ObjectMap auditParams = new ObjectMap() - .append("studyId", studyId) - .append("query", new Query(query)) - .append("options", options) - .append("defaultStats", defaultStats) - .append("token", token); - try { - if (defaultStats || StringUtils.isEmpty(options.getString(QueryOptions.FACET))) { - String facet = options.getString(QueryOptions.FACET); - options.put(QueryOptions.FACET, StringUtils.isNotEmpty(facet) ? defaultFacet + ";" + facet : defaultFacet); - } - - try (CatalogSolrManager catalogSolrManager = new CatalogSolrManager(catalogManager)) { - AnnotationUtils.fixQueryAnnotationSearch(study, userId, query, authorizationManager); - - DataResult result = catalogSolrManager.facetedQuery(study, CatalogSolrManager.INDIVIDUAL_SOLR_COLLECTION, query, - options, userId); - auditManager.auditFacet(userId, Enums.Resource.INDIVIDUAL, study.getId(), study.getUuid(), auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); - - return result; - } - } catch (CatalogException e) { - auditManager.auditFacet(userId, Enums.Resource.INDIVIDUAL, study.getId(), study.getUuid(), auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(0, "", e.getMessage()))); - throw e; + auditManager.finishAuditBatch(organizationId, operationId); } } // ************************** Private methods ******************************** // - private List getSampleUidsFromIndividuals(long studyUid, List individualUidList) throws CatalogException { + private List getSampleUidsFromIndividuals(String organizationId, long studyUid, List individualUidList) + throws CatalogException { // Look for all the samples belonging to the individual Query query = new Query() .append(IndividualDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(IndividualDBAdaptor.QueryParams.UID.key(), individualUidList); - OpenCGAResult individualDataResult = individualDBAdaptor.get(query, + OpenCGAResult individualDataResult = getIndividualDBAdaptor(organizationId).get(query, new QueryOptions(QueryOptions.INCLUDE, IndividualDBAdaptor.QueryParams.SAMPLES.key())); List sampleUids = new ArrayList<>(); @@ -1448,7 +1470,7 @@ private List getSampleUidsFromIndividuals(long studyUid, List indivi // Checks if father or mother are in query and transforms them into father.id and mother.id respectively - private void fixQuery(Study study, Query query, String userId) throws CatalogException { + private void fixQuery(String organizationId, Study study, Query query, String userId) throws CatalogException { super.fixQueryObject(query); changeQueryId(query, ParamConstants.INDIVIDUAL_SEX_PARAM, IndividualDBAdaptor.QueryParams.SEX_ID.key()); changeQueryId(query, ParamConstants.INDIVIDUAL_ETHNICITY_PARAM, IndividualDBAdaptor.QueryParams.ETHNICITY_ID.key()); @@ -1460,7 +1482,7 @@ private void fixQuery(Study study, Query query, String userId) throws CatalogExc if (query.containsKey(IndividualDBAdaptor.QueryParams.FATHER.key())) { if (StringUtils.isNotEmpty(query.getString(IndividualDBAdaptor.QueryParams.FATHER.key()))) { - Individual ind = internalGet(study.getUid(), query.getString(IndividualDBAdaptor.QueryParams.FATHER.key()), + Individual ind = internalGet(organizationId, study.getUid(), query.getString(IndividualDBAdaptor.QueryParams.FATHER.key()), INCLUDE_INDIVIDUAL_IDS, userId).first(); query.append(IndividualDBAdaptor.QueryParams.FATHER_UID.key(), ind.getUid()); } @@ -1468,7 +1490,7 @@ private void fixQuery(Study study, Query query, String userId) throws CatalogExc } if (query.containsKey(IndividualDBAdaptor.QueryParams.MOTHER.key())) { if (StringUtils.isNotEmpty(query.getString(IndividualDBAdaptor.QueryParams.MOTHER.key()))) { - Individual ind = internalGet(study.getUid(), query.getString(IndividualDBAdaptor.QueryParams.MOTHER.key()), + Individual ind = internalGet(organizationId, study.getUid(), query.getString(IndividualDBAdaptor.QueryParams.MOTHER.key()), INCLUDE_INDIVIDUAL_IDS, userId).first(); query.append(IndividualDBAdaptor.QueryParams.MOTHER_UID.key(), ind.getUid()); } @@ -1476,7 +1498,7 @@ private void fixQuery(Study study, Query query, String userId) throws CatalogExc } if (query.containsKey(IndividualDBAdaptor.QueryParams.SAMPLES.key())) { if (StringUtils.isNotEmpty(query.getString(IndividualDBAdaptor.QueryParams.SAMPLES.key()))) { - OpenCGAResult sampleDataResult = catalogManager.getSampleManager().internalGet(study.getUid(), + OpenCGAResult sampleDataResult = catalogManager.getSampleManager().internalGet(organizationId, study.getUid(), query.getAsStringList(IndividualDBAdaptor.QueryParams.SAMPLES.key()), SampleManager.INCLUDE_SAMPLE_IDS, userId, true); query.append(IndividualDBAdaptor.QueryParams.SAMPLE_UIDS.key(), sampleDataResult.getResults().stream().map(Sample::getUid) diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/InterpretationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/InterpretationManager.java index 0ab582184d0..417978cef13 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/InterpretationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/InterpretationManager.java @@ -27,7 +27,6 @@ import org.opencb.biodata.models.clinical.interpretation.DiseasePanel; import org.opencb.biodata.models.clinical.interpretation.InterpretationMethod; import org.opencb.biodata.models.clinical.interpretation.InterpretationStats; -import org.opencb.biodata.models.common.Status; import org.opencb.commons.datastore.core.Event; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; @@ -40,12 +39,15 @@ import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.models.InternalGetDataResult; +import org.opencb.opencga.catalog.utils.CatalogFqn; import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.catalog.utils.UuidUtils; import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.common.GitRepositoryState; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.Configuration; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.audit.AuditRecord; import org.opencb.opencga.core.models.clinical.*; import org.opencb.opencga.core.models.common.Enums; @@ -67,18 +69,16 @@ public class InterpretationManager extends ResourceManager { public static final QueryOptions INCLUDE_CLINICAL_ANALYSIS = keepFieldsInQueryOptions(ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS, - Arrays.asList(ClinicalAnalysisDBAdaptor.QueryParams.LOCKED.key(), ClinicalAnalysisDBAdaptor.QueryParams.PANEL_LOCK.key())); + Arrays.asList(ClinicalAnalysisDBAdaptor.QueryParams.LOCKED.key(), ClinicalAnalysisDBAdaptor.QueryParams.PANEL_LOCKED.key(), + ClinicalAnalysisDBAdaptor.QueryParams.STATUS.key())); public static final QueryOptions INCLUDE_INTERPRETATION_IDS = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList( InterpretationDBAdaptor.QueryParams.ID.key(), InterpretationDBAdaptor.QueryParams.UID.key(), InterpretationDBAdaptor.QueryParams.UUID.key(), InterpretationDBAdaptor.QueryParams.CLINICAL_ANALYSIS_ID.key(), - InterpretationDBAdaptor.QueryParams.LOCKED.key(), + InterpretationDBAdaptor.QueryParams.LOCKED.key(), InterpretationDBAdaptor.QueryParams.STATUS.key(), InterpretationDBAdaptor.QueryParams.VERSION.key(), InterpretationDBAdaptor.QueryParams.STUDY_UID.key())); - public static final QueryOptions INCLUDE_INTERPRETATION_FINDING_IDS = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList( - InterpretationDBAdaptor.QueryParams.ID.key(), InterpretationDBAdaptor.QueryParams.UID.key(), - InterpretationDBAdaptor.QueryParams.UUID.key(), InterpretationDBAdaptor.QueryParams.CLINICAL_ANALYSIS_ID.key(), - InterpretationDBAdaptor.QueryParams.VERSION.key(), InterpretationDBAdaptor.QueryParams.STUDY_UID.key(), - InterpretationDBAdaptor.QueryParams.LOCKED.key(), InterpretationDBAdaptor.QueryParams.PRIMARY_FINDINGS_ID.key(), - InterpretationDBAdaptor.QueryParams.SECONDARY_FINDINGS_ID.key())); + public static final QueryOptions INCLUDE_INTERPRETATION_FINDING_IDS = keepFieldsInQueryOptions(INCLUDE_INTERPRETATION_IDS, + Arrays.asList(InterpretationDBAdaptor.QueryParams.PRIMARY_FINDINGS_ID.key(), + InterpretationDBAdaptor.QueryParams.SECONDARY_FINDINGS_ID.key())); protected static Logger logger = LoggerFactory.getLogger(InterpretationManager.class); private UserManager userManager; private StudyManager studyManager; @@ -98,8 +98,8 @@ Enums.Resource getEntity() { } @Override - InternalGetDataResult internalGet(long studyUid, List entryList, @Nullable Query query, QueryOptions options, - String user, boolean ignoreException) throws CatalogException { + InternalGetDataResult internalGet(String organizationId, long studyUid, List entryList, @Nullable Query query, + QueryOptions options, String user, boolean ignoreException) throws CatalogException { if (ListUtils.isEmpty(entryList)) { throw new CatalogException("Missing interpretation entries."); } @@ -135,7 +135,8 @@ InternalGetDataResult internalGet(long studyUid, List en // Ensure the field by which we are querying for will be kept in the results queryOptions = keepFieldInQueryOptions(queryOptions, idQueryParam.key()); - OpenCGAResult interpretationDataResult = interpretationDBAdaptor.get(studyUid, queryCopy, queryOptions, user); + OpenCGAResult interpretationDataResult = getInterpretationDBAdaptor(organizationId).get(studyUid, queryCopy, + queryOptions, user); if (!versioned && interpretationDataResult.getNumResults() != uniqueList.size() && !ignoreException) { throw CatalogException.notFound("interpretations", @@ -152,7 +153,7 @@ InternalGetDataResult internalGet(long studyUid, List en Interpretation interpretation = iterator.next(); // Check if the user has access to the corresponding clinical analysis try { - catalogManager.getClinicalAnalysisManager().internalGet(studyUid, + catalogManager.getClinicalAnalysisManager().internalGet(organizationId, studyUid, interpretation.getClinicalAnalysisId(), ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS, user); } catch (CatalogAuthorizationException e) { if (ignoreException) { @@ -169,7 +170,7 @@ InternalGetDataResult internalGet(long studyUid, List en interpretationList = interpretationDataResult.getResults(); Interpretation interpretation = interpretationDataResult.first(); try { - catalogManager.getClinicalAnalysisManager().internalGet(studyUid, + catalogManager.getClinicalAnalysisManager().internalGet(organizationId, studyUid, interpretation.getClinicalAnalysisId(), ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS, user); } catch (CatalogAuthorizationException e) { if (!ignoreException) { @@ -190,7 +191,7 @@ InternalGetDataResult internalGet(long studyUid, List en } @Override - public OpenCGAResult create(String studyStr, Interpretation entry, QueryOptions options, String sessionId) + public OpenCGAResult create(String studyStr, Interpretation entry, QueryOptions options, String token) throws CatalogException { throw new CatalogException("Non-supported. Use other create method"); } @@ -199,8 +200,11 @@ public OpenCGAResult create(String studyStr, String clinicalAnal ParamUtils.SaveInterpretationAs saveInterpretationAs, QueryOptions options, String token) throws CatalogException { // We check if the user can create interpretations in the clinical analysis - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId, StudyManager.INCLUDE_CONFIGURATION); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, StudyManager.INCLUDE_CONFIGURATION, userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("study", studyStr) @@ -213,41 +217,41 @@ public OpenCGAResult create(String studyStr, String clinicalAnal try { QueryOptions clinicalOptions = keepFieldsInQueryOptions(ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS, Arrays.asList(ClinicalAnalysisDBAdaptor.QueryParams.PANELS.key(), - ClinicalAnalysisDBAdaptor.QueryParams.PANEL_LOCK.key(), + ClinicalAnalysisDBAdaptor.QueryParams.PANEL_LOCKED.key(), ClinicalAnalysisDBAdaptor.QueryParams.AUDIT.key(), ClinicalAnalysisDBAdaptor.QueryParams.INTERPRETATION_ID.key(), ClinicalAnalysisDBAdaptor.QueryParams.SECONDARY_INTERPRETATIONS_ID.key())); - ClinicalAnalysis clinicalAnalysis = catalogManager.getClinicalAnalysisManager().internalGet(study.getUid(), clinicalAnalysisStr, - clinicalOptions, userId).first(); + ClinicalAnalysis clinicalAnalysis = catalogManager.getClinicalAnalysisManager().internalGet(organizationId, study.getUid(), + clinicalAnalysisStr, clinicalOptions, userId).first(); - authorizationManager.checkClinicalAnalysisPermission(study.getUid(), clinicalAnalysis.getUid(), + authorizationManager.checkClinicalAnalysisPermission(organizationId, study.getUid(), clinicalAnalysis.getUid(), userId, ClinicalAnalysisPermissions.WRITE); - validateNewInterpretation(study, interpretation, clinicalAnalysis, userId); + validateNewInterpretation(organizationId, study, interpretation, clinicalAnalysis, userId); ClinicalAudit clinicalAudit = new ClinicalAudit(userId, ClinicalAudit.Action.CREATE_INTERPRETATION, "Create interpretation '" + interpretation.getId() + "'", TimeUtils.getTime()); - OpenCGAResult result = interpretationDBAdaptor.insert(study.getUid(), interpretation, saveInterpretationAs, - Collections.singletonList(clinicalAudit)); + OpenCGAResult result = getInterpretationDBAdaptor(organizationId).insert(study.getUid(), interpretation, + saveInterpretationAs, Collections.singletonList(clinicalAudit)); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { // Fetch created Interpretation - OpenCGAResult queryResult = interpretationDBAdaptor.get(study.getUid(), interpretation.getId(), - QueryOptions.empty()); + OpenCGAResult queryResult = getInterpretationDBAdaptor(organizationId).get(study.getUid(), + interpretation.getId(), QueryOptions.empty()); result.setResults(queryResult.getResults()); } - auditManager.auditCreate(userId, Enums.Resource.INTERPRETATION, interpretation.getId(), "", study.getId(), + auditManager.auditCreate(organizationId, userId, Enums.Resource.INTERPRETATION, interpretation.getId(), "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return result; } catch (CatalogException e) { - auditManager.auditCreate(userId, Enums.Resource.INTERPRETATION, interpretation.getId(), "", study.getId(), + auditManager.auditCreate(organizationId, userId, Enums.Resource.INTERPRETATION, interpretation.getId(), "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } - void validateNewInterpretation(Study study, Interpretation interpretation, ClinicalAnalysis clinicalAnalysis, String userId) - throws CatalogException { + void validateNewInterpretation(String organizationId, Study study, Interpretation interpretation, ClinicalAnalysis clinicalAnalysis, + String userId) throws CatalogException { if (study.getInternal() == null || study.getInternal().getConfiguration() == null || study.getInternal().getConfiguration().getClinical() == null || study.getInternal().getConfiguration().getClinical().getInterpretation() == null) { @@ -271,6 +275,7 @@ void validateNewInterpretation(Study study, Interpretation interpretation, Clini } interpretation.setId(clinicalAnalysis.getId() + "." + count); + interpretation.setName(ParamUtils.defaultString(interpretation.getName(), interpretation.getId())); interpretation.setClinicalAnalysisId(clinicalAnalysis.getId()); @@ -284,7 +289,7 @@ void validateNewInterpretation(Study study, Interpretation interpretation, Clini interpretation.setPrimaryFindings(ParamUtils.defaultObject(interpretation.getPrimaryFindings(), Collections.emptyList())); interpretation.setSecondaryFindings(ParamUtils.defaultObject(interpretation.getSecondaryFindings(), Collections.emptyList())); interpretation.setComments(ParamUtils.defaultObject(interpretation.getComments(), Collections.emptyList())); - interpretation.setStatus(ParamUtils.defaultObject(interpretation.getStatus(), Status::new)); + interpretation.setStatus(ParamUtils.defaultObject(interpretation.getStatus(), ClinicalStatus::new)); interpretation.setRelease(studyManager.getCurrentRelease(study)); interpretation.setVersion(1); interpretation.setAttributes(ParamUtils.defaultObject(interpretation.getAttributes(), Collections.emptyMap())); @@ -294,7 +299,7 @@ void validateNewInterpretation(Study study, Interpretation interpretation, Clini if (CollectionUtils.isEmpty(interpretation.getPanels())) { interpretation.setPanels(clinicalAnalysis.getPanels()); } else { - if (clinicalAnalysis.isPanelLock()) { + if (clinicalAnalysis.isPanelLocked()) { // Check the panels are the same provided in the Clinical Analysis Map clinicalPanelIds = clinicalAnalysis.getPanels().stream() .collect(Collectors.toMap(DiseasePanel::getId, panel -> panel)); @@ -302,7 +307,8 @@ void validateNewInterpretation(Study study, Interpretation interpretation, Clini List panelList = new ArrayList<>(clinicalPanelIds.size()); for (Panel panel : interpretation.getPanels()) { if (!clinicalPanelIds.containsKey(panel.getId())) { - throw new CatalogException("'panelLock' from ClinicalAnalysis is set to True. Please, leave list of panels empty" + throw new CatalogException("'" + ClinicalAnalysisDBAdaptor.QueryParams.PANEL_LOCKED.key() + + "' from ClinicalAnalysis is set to True. Please, leave list of panels empty" + " so they can be inherited or pass at least a subset of the panels defined in the Clinical Analysis."); } panelList.add(clinicalPanelIds.get(panel.getId())); @@ -315,7 +321,7 @@ void validateNewInterpretation(Study study, Interpretation interpretation, Clini Set panelIds = interpretation.getPanels().stream().map(Panel::getId).collect(Collectors.toSet()); Query query = new Query(PanelDBAdaptor.QueryParams.ID.key(), panelIds); OpenCGAResult panelResult = - panelDBAdaptor.get(study.getUid(), query, PanelManager.INCLUDE_PANEL_IDS, userId); + getPanelDBAdaptor(organizationId).get(study.getUid(), query, PanelManager.INCLUDE_PANEL_IDS, userId); if (panelResult.getNumResults() < panelIds.size()) { throw new CatalogException("Some panels were not found or user doesn't have permissions to see them"); } @@ -325,9 +331,9 @@ void validateNewInterpretation(Study study, Interpretation interpretation, Clini } // Validate status - validateStatusParameter(interpretation, clinicalAnalysis.getType(), interpretationConfiguration); + validateStatusParameter(interpretation, interpretationConfiguration, userId, true); if (StringUtils.isNotEmpty(interpretation.getStatus().getId())) { - List clinicalStatusValues = interpretationConfiguration.getStatus().get(clinicalAnalysis.getType()); + List clinicalStatusValues = interpretationConfiguration.getStatus(); for (ClinicalStatusValue clinicalStatusValue : clinicalStatusValues) { if (interpretation.getStatus().getId().equals(clinicalStatusValue.getId()) && clinicalStatusValue.getType() == ClinicalStatusValue.ClinicalStatusType.CLOSED) { @@ -379,10 +385,10 @@ void validateNewInterpretation(Study study, Interpretation interpretation, Clini UserDBAdaptor.QueryParams.NAME.key(), UserDBAdaptor.QueryParams.EMAIL.key())); User user; if (interpretation.getAnalyst() == null || StringUtils.isEmpty(interpretation.getAnalyst().getId())) { - user = userDBAdaptor.get(userId, userInclude).first(); + user = getUserDBAdaptor(organizationId).get(userId, userInclude).first(); } else { // Validate user - OpenCGAResult result = userDBAdaptor.get(interpretation.getAnalyst().getId(), userInclude); + OpenCGAResult result = getUserDBAdaptor(organizationId).get(interpretation.getAnalyst().getId(), userInclude); if (result.getNumResults() == 0) { throw new CatalogException("User '" + interpretation.getAnalyst().getId() + "' not found"); } @@ -393,8 +399,11 @@ void validateNewInterpretation(Study study, Interpretation interpretation, Clini public OpenCGAResult clear(String studyStr, String clinicalAnalysisId, List interpretationList, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId, StudyManager.INCLUDE_CONFIGURATION); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, StudyManager.INCLUDE_CONFIGURATION, userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("study", studyStr) @@ -404,6 +413,16 @@ public OpenCGAResult clear(String studyStr, String clinicalAnaly String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); auditManager.initAuditBatch(operationId); + InterpretationStudyConfiguration interpretationConfiguration = + study.getInternal().getConfiguration().getClinical().getInterpretation(); + ClinicalStatusValue initStatus = null; + for (ClinicalStatusValue status : interpretationConfiguration.getStatus()) { + if (status.getType().equals(ClinicalStatusValue.ClinicalStatusType.NOT_STARTED)) { + initStatus = status; + break; + } + } + OpenCGAResult result = OpenCGAResult.empty(); for (String interpretationStr : interpretationList) { String interpretationId = interpretationStr; @@ -411,8 +430,8 @@ public OpenCGAResult clear(String studyStr, String clinicalAnaly try { QueryOptions clinicalOptions = keepFieldInQueryOptions(INCLUDE_CLINICAL_ANALYSIS, ClinicalAnalysisDBAdaptor.QueryParams.PANELS.key()); - OpenCGAResult clinicalResult = catalogManager.getClinicalAnalysisManager().internalGet(study.getUid(), - clinicalAnalysisId, clinicalOptions, userId); + OpenCGAResult clinicalResult = catalogManager.getClinicalAnalysisManager().internalGet(organizationId, + study.getUid(), clinicalAnalysisId, clinicalOptions, userId); if (clinicalResult.getNumResults() == 0) { throw new CatalogException("ClinicalAnalysis '" + clinicalAnalysisId + "' not found"); } @@ -422,8 +441,8 @@ public OpenCGAResult clear(String studyStr, String clinicalAnaly + "made to the Interpretation."); } - OpenCGAResult tmpResult = internalGet(study.getUid(), interpretationStr, INCLUDE_INTERPRETATION_IDS, - userId); + OpenCGAResult tmpResult = internalGet(organizationId, study.getUid(), interpretationStr, + INCLUDE_INTERPRETATION_IDS, userId); if (tmpResult.getNumResults() == 0) { throw new CatalogException("Interpretation '" + interpretationStr + "' not found."); } @@ -448,33 +467,33 @@ public OpenCGAResult clear(String studyStr, String clinicalAnaly actionMap.put(InterpretationDBAdaptor.QueryParams.PANELS.key(), ParamUtils.BasicUpdateAction.SET); QueryOptions options = new QueryOptions(Constants.ACTIONS, actionMap); - InterpretationUpdateParams params = new InterpretationUpdateParams("", new ClinicalAnalystParam(), + InterpretationUpdateParams params = new InterpretationUpdateParams("", "", new ClinicalAnalystParam(), InterpretationMethod.init(), null, null, Collections.emptyList(), Collections.emptyList(), clinicalAnalysis.getPanels() != null ? clinicalAnalysis.getPanels().stream() .map(p -> new PanelReferenceParam().setId(p.getId())).collect(Collectors.toList()) : null, - Collections.emptyList(), new StatusParam(), false, new ObjectMap()); + Collections.emptyList(), new StatusParam(initStatus.getId()), false, new ObjectMap()); ClinicalAudit clinicalAudit = new ClinicalAudit(userId, ClinicalAudit.Action.CLEAR_INTERPRETATION, "Clear interpretation '" + interpretationId + "'", TimeUtils.getTime()); - OpenCGAResult writeResult = update(study, interpretation, params, Collections.singletonList(clinicalAudit), null, options, - userId); + OpenCGAResult writeResult = update(organizationId, study, interpretation, params, Collections.singletonList(clinicalAudit), + null, options, userId); result.append(writeResult); - auditManager.audit(operationId, userId, Enums.Action.CLEAR, Enums.Resource.INTERPRETATION, interpretationId, + auditManager.audit(organizationId, operationId, userId, Enums.Action.CLEAR, Enums.Resource.INTERPRETATION, interpretationId, interpretationUuid, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS), new ObjectMap()); return result; } catch (CatalogException e) { - auditManager.audit(operationId, userId, Enums.Action.CLEAR, Enums.Resource.INTERPRETATION, interpretationId, + auditManager.audit(organizationId, operationId, userId, Enums.Action.CLEAR, Enums.Resource.INTERPRETATION, interpretationId, interpretationUuid, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError()), new ObjectMap()); throw e; } } - auditManager.finishAuditBatch(operationId); + auditManager.finishAuditBatch(organizationId, operationId); return result; } @@ -541,7 +560,8 @@ public OpenCGAResult clear(String studyStr, String clinicalAnaly // ClinicalAudit clinicalAudit = new ClinicalAudit(userId, ClinicalAudit.Action.MERGE_INTERPRETATION, // "Merge interpretation '" + interpretation2.getId() + "' in interpretation '" + interpretation.getId() + "'", // TimeUtils.getTime()); -// OpenCGAResult mergeResult = interpretationDBAdaptor.merge(interpretation.getUid(), interpretation2, +// OpenCGAResult mergeResult = getInterpretationDBAdaptor(organizationId).merge(interpretation.getUid(), +// interpretation2, // Collections.singletonList(clinicalAudit), clinicalVariantList); // auditManager.audit(userId, Enums.Action.MERGE, Enums.Resource.INTERPRETATION, interpretationId, interpretationUuid, // study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); @@ -605,7 +625,8 @@ public OpenCGAResult clear(String studyStr, String clinicalAnaly // ClinicalAudit clinicalAudit = new ClinicalAudit(userId, ClinicalAudit.Action.MERGE_INTERPRETATION, // "Merge external interpretation in interpretation '" + interpretation.getId() + "'", // TimeUtils.getTime()); -// OpenCGAResult mergeResult = interpretationDBAdaptor.merge(interpretation.getUid(), interpretation2, +// OpenCGAResult mergeResult = getInterpretationDBAdaptor(organizationId).merge(interpretation.getUid(), +// interpretation2, // Collections.singletonList(clinicalAudit), clinicalVariantList); // auditManager.audit(userId, Enums.Action.MERGE, Enums.Resource.INTERPRETATION, interpretationId, interpretationUuid, // study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); @@ -618,92 +639,100 @@ public OpenCGAResult clear(String studyStr, String clinicalAnaly // } // } - public OpenCGAResult update(String studyStr, Query query, InterpretationUpdateParams updateParams, - ParamUtils.SaveInterpretationAs as, QueryOptions options, String token) - throws CatalogException { - return update(studyStr, query, updateParams, as, false, options, token); - } - - public OpenCGAResult update(String studyStr, Query query, InterpretationUpdateParams updateParams, - ParamUtils.SaveInterpretationAs as, boolean ignoreException, QueryOptions options, - String token) throws CatalogException { - options = ParamUtils.defaultObject(options, QueryOptions::new); - - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId, StudyManager.INCLUDE_CONFIGURATION); - - String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); - - ObjectMap updateMap; - try { - updateMap = updateParams != null ? updateParams.getUpdateMap() : null; - } catch (JsonProcessingException e) { - throw new CatalogException("Could not parse InterpretationUpdateParams object: " + e.getMessage(), e); - } - - ObjectMap auditParams = new ObjectMap() - .append("study", studyStr) - .append("query", query) - .append("updateParams", updateMap) - .append("as", as) - .append("ignoreException", ignoreException) - .append("options", options) - .append("token", token); - - Query finalQuery = new Query(ParamUtils.defaultObject(query, Query::new)); - fixQueryObject(study, finalQuery, userId); - - DBIterator iterator; - try { - finalQuery.append(InterpretationDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - iterator = interpretationDBAdaptor.iterator(study.getUid(), finalQuery, INCLUDE_INTERPRETATION_FINDING_IDS, userId); - } catch (CatalogException e) { - auditManager.auditUpdate(operationId, userId, Enums.Resource.INTERPRETATION, "", "", study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); - throw e; - } - - auditManager.initAuditBatch(operationId); - OpenCGAResult result = OpenCGAResult.empty(); - while (iterator.hasNext()) { - Interpretation interpretation = iterator.next(); - try { - List clinicalAuditList = new ArrayList<>(); - clinicalAuditList.add(new ClinicalAudit(userId, ClinicalAudit.Action.UPDATE_INTERPRETATION, - "Update interpretation '" + interpretation.getId() + "'", TimeUtils.getTime())); - if (as != null) { - clinicalAuditList.add(new ClinicalAudit(userId, ClinicalAudit.Action.SWAP_INTERPRETATION, - "Swap interpretation '" + interpretation.getId() + "' to " + as, TimeUtils.getTime())); - } - OpenCGAResult writeResult = update(study, interpretation, updateParams, clinicalAuditList, as, options, userId); - auditManager.auditUpdate(operationId, userId, Enums.Resource.INTERPRETATION, interpretation.getId(), - interpretation.getUuid(), study.getId(), study.getUuid(), auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); - - result.append(writeResult); - } catch (CatalogException e) { - Event event = new Event(Event.Type.ERROR, interpretation.getId(), e.getMessage()); - result.getEvents().add(event); - result.setNumErrors(result.getNumErrors() + 1); - - logger.error("Cannot update interpretation {}: {}", interpretation.getId(), e.getMessage(), e); - auditManager.auditUpdate(operationId, userId, Enums.Resource.INTERPRETATION, interpretation.getId(), - interpretation.getUuid(), study.getId(), study.getUuid(), auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); - } - } - auditManager.finishAuditBatch(operationId); - - return endResult(result, ignoreException); - } +// public OpenCGAResult update(String studyStr, Query query, InterpretationUpdateParams updateParams, +// ParamUtils.SaveInterpretationAs as, QueryOptions options, String token) +// throws CatalogException { +// return update(studyStr, query, updateParams, as, false, options, token); +// } +// +// public OpenCGAResult update(String studyStr, Query query, InterpretationUpdateParams updateParams, +// ParamUtils.SaveInterpretationAs as, boolean ignoreException, QueryOptions options, +// String token) throws CatalogException { +// options = ParamUtils.defaultObject(options, QueryOptions::new); +// +// JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); +// CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); +// String organizationId = studyFqn.getOrganizationId(); +// String userId = tokenPayload.getUserId(organizationId); +// Study study = studyManager.resolveId(studyStr, StudyManager.INCLUDE_CONFIGURATION, userId, organizationId); +// +// String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); +// +// ObjectMap updateMap; +// try { +// updateMap = updateParams != null ? updateParams.getUpdateMap() : null; +// } catch (JsonProcessingException e) { +// throw new CatalogException("Could not parse InterpretationUpdateParams object: " + e.getMessage(), e); +// } +// +// ObjectMap auditParams = new ObjectMap() +// .append("study", studyStr) +// .append("query", query) +// .append("updateParams", updateMap) +// .append("as", as) +// .append("ignoreException", ignoreException) +// .append("options", options) +// .append("token", token); +// +// Query finalQuery = new Query(ParamUtils.defaultObject(query, Query::new)); +// fixQueryObject(organizationId, study, finalQuery, userId); +// +// DBIterator iterator; +// try { +// finalQuery.append(InterpretationDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); +// iterator = getInterpretationDBAdaptor(organizationId).iterator(study.getUid(), finalQuery, INCLUDE_INTERPRETATION_FINDING_IDS, +// userId); +// } catch (CatalogException e) { +// auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.INTERPRETATION, "", "", study.getId(), +// study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); +// throw e; +// } +// +// auditManager.initAuditBatch(operationId); +// OpenCGAResult result = OpenCGAResult.empty(); +// while (iterator.hasNext()) { +// Interpretation interpretation = iterator.next(); +// try { +// List clinicalAuditList = new ArrayList<>(); +// clinicalAuditList.add(new ClinicalAudit(userId, ClinicalAudit.Action.UPDATE_INTERPRETATION, +// "Update interpretation '" + interpretation.getId() + "'", TimeUtils.getTime())); +// if (as != null) { +// clinicalAuditList.add(new ClinicalAudit(userId, ClinicalAudit.Action.SWAP_INTERPRETATION, +// "Swap interpretation '" + interpretation.getId() + "' to " + as, TimeUtils.getTime())); +// } +// OpenCGAResult writeResult = update(organizationId, study, interpretation, updateParams, clinicalAuditList, as, options, +// userId); +// auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.INTERPRETATION, interpretation.getId(), +// interpretation.getUuid(), study.getId(), study.getUuid(), auditParams, +// new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); +// +// result.append(writeResult); +// } catch (CatalogException e) { +// Event event = new Event(Event.Type.ERROR, interpretation.getId(), e.getMessage()); +// result.getEvents().add(event); +// result.setNumErrors(result.getNumErrors() + 1); +// +// logger.error("Cannot update interpretation {}: {}", interpretation.getId(), e.getMessage(), e); +// auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.INTERPRETATION, interpretation.getId(), +// interpretation.getUuid(), study.getId(), study.getUuid(), auditParams, +// new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); +// } +// } +// auditManager.finishAuditBatch(organizationId, operationId); +// +// return endResult(result, ignoreException); +// } public OpenCGAResult update(String studyStr, String clinicalAnalysisId, String intepretationId, InterpretationUpdateParams updateParams, ParamUtils.SaveInterpretationAs as, QueryOptions options, String token) throws CatalogException { options = ParamUtils.defaultObject(options, QueryOptions::new); - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId, StudyManager.INCLUDE_CONFIGURATION); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, StudyManager.INCLUDE_CONFIGURATION, userId, organizationId); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -730,7 +759,7 @@ public OpenCGAResult update(String studyStr, String clinicalAnal ParamUtils.checkParameter(clinicalAnalysisId, "ClinicalAnalysisId"); ParamUtils.checkParameter(intepretationId, "InterpretationId"); - OpenCGAResult interpretationOpenCGAResult = internalGet(study.getUid(), intepretationId, + OpenCGAResult interpretationOpenCGAResult = internalGet(organizationId, study.getUid(), intepretationId, INCLUDE_INTERPRETATION_FINDING_IDS, userId); if (interpretationOpenCGAResult.getNumResults() == 0) { throw new CatalogException("Interpretation '" + interpretationId + "' not found."); @@ -753,10 +782,10 @@ public OpenCGAResult update(String studyStr, String clinicalAnal clinicalAuditList.add(new ClinicalAudit(userId, ClinicalAudit.Action.SWAP_INTERPRETATION, "Swap interpretation '" + interpretation.getId() + "' to " + as, TimeUtils.getTime())); } - OpenCGAResult writeResult = update(study, interpretation, updateParams, clinicalAuditList, as, options, userId); + OpenCGAResult writeResult = update(organizationId, study, interpretation, updateParams, clinicalAuditList, as, options, userId); result.append(writeResult); - auditManager.auditUpdate(operationId, userId, Enums.Resource.INTERPRETATION, interpretation.getId(), + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.INTERPRETATION, interpretation.getId(), interpretation.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { @@ -767,8 +796,9 @@ public OpenCGAResult update(String studyStr, String clinicalAnal result.setNumErrors(result.getNumErrors() + 1); logger.error("{}", e1.getMessage(), e); - auditManager.auditUpdate(operationId, userId, Enums.Resource.INTERPRETATION, interpretationId, interpretationUuid, - study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e1.getError())); + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.INTERPRETATION, interpretationId, + interpretationUuid, study.getId(), study.getUuid(), auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e1.getError())); throw e1; } @@ -778,7 +808,7 @@ public OpenCGAResult update(String studyStr, String clinicalAnal /** * Update interpretations from catalog. * - * @param studyStr Study id in string format. Could be one of [id|user@projectId:studyId|projectId:studyId]. + * @param studyStr Study id in string format. Could be one of [id|organization@projectId:studyId|projectId:studyId]. * @param clinicalAnalysisId ClinicalAnalysis id. * @param interpretationIds List of interpretation ids. Could be either the id or uuid. * @param updateParams Data model filled only with the parameters to be updated. @@ -791,7 +821,8 @@ public OpenCGAResult update(String studyStr, String clinicalAnal */ public OpenCGAResult update(String studyStr, String clinicalAnalysisId, List interpretationIds, InterpretationUpdateParams updateParams, ParamUtils.SaveInterpretationAs as, - QueryOptions options, String token) throws CatalogException { + QueryOptions options, String token) + throws CatalogException { return update(studyStr, clinicalAnalysisId, interpretationIds, updateParams, as, false, options, token); } @@ -800,8 +831,11 @@ public OpenCGAResult update(String studyStr, String clinicalAnal boolean ignoreException, QueryOptions options, String token) throws CatalogException { options = ParamUtils.defaultObject(options, QueryOptions::new); - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId, StudyManager.INCLUDE_CONFIGURATION); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, StudyManager.INCLUDE_CONFIGURATION, userId, organizationId); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -829,8 +863,8 @@ public OpenCGAResult update(String studyStr, String clinicalAnal String interpretationUuid = ""; try { - OpenCGAResult tmpResult = internalGet(study.getUid(), interpretationId, INCLUDE_INTERPRETATION_FINDING_IDS, - userId); + OpenCGAResult tmpResult = internalGet(organizationId, study.getUid(), interpretationId, + INCLUDE_INTERPRETATION_FINDING_IDS, userId); if (tmpResult.getNumResults() == 0) { throw new CatalogException("Interpretation '" + interpretationId + "' not found."); } @@ -852,10 +886,11 @@ public OpenCGAResult update(String studyStr, String clinicalAnal clinicalAuditList.add(new ClinicalAudit(userId, ClinicalAudit.Action.SWAP_INTERPRETATION, "Swap interpretation '" + interpretation.getId() + "' to " + as, TimeUtils.getTime())); } - OpenCGAResult writeResult = update(study, interpretation, updateParams, clinicalAuditList, as, options, userId); + OpenCGAResult writeResult = update(organizationId, study, interpretation, updateParams, clinicalAuditList, as, options, + userId); result.append(writeResult); - auditManager.auditUpdate(operationId, userId, Enums.Resource.INTERPRETATION, interpretation.getId(), + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.INTERPRETATION, interpretation.getId(), interpretation.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { @@ -864,16 +899,17 @@ public OpenCGAResult update(String studyStr, String clinicalAnal result.setNumErrors(result.getNumErrors() + 1); logger.error("Cannot update interpretation {}: {}", interpretationId, e.getMessage(), e); - auditManager.auditUpdate(operationId, userId, Enums.Resource.INTERPRETATION, interpretationId, interpretationUuid, - study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.INTERPRETATION, interpretationId, + interpretationUuid, study.getId(), study.getUuid(), auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); } } - auditManager.finishAuditBatch(operationId); + auditManager.finishAuditBatch(organizationId, operationId); return endResult(result, ignoreException); } - private OpenCGAResult update(Study study, Interpretation interpretation, InterpretationUpdateParams updateParams, + private OpenCGAResult update(String organizationId, Study study, Interpretation interpretation, InterpretationUpdateParams updateParams, List clinicalAuditList, ParamUtils.SaveInterpretationAs as, QueryOptions options, String userId) throws CatalogException { if (study.getInternal() == null || study.getInternal().getConfiguration() == null @@ -884,17 +920,89 @@ private OpenCGAResult update(Study study, Interpretation interpretation, Interpr InterpretationStudyConfiguration interpretationConfiguration = study.getInternal().getConfiguration().getClinical().getInterpretation(); + ObjectMap parameters = new ObjectMap(); + if (updateParams != null) { + try { + parameters = updateParams.getUpdateMap(); + } catch (JsonProcessingException e) { + throw new CatalogException("Could not parse InterpretationUpdateParams object: " + e.getMessage(), e); + } + } else { + throw new CatalogException("Missing interpretation update parameters"); + } + Map actionMap = options.getMap(Constants.ACTIONS); // Check if user has permissions to write clinical analysis - ClinicalAnalysis clinicalAnalysis = catalogManager.getClinicalAnalysisManager().internalGet(study.getUid(), + ClinicalAnalysis clinicalAnalysis = catalogManager.getClinicalAnalysisManager().internalGet(organizationId, study.getUid(), interpretation.getClinicalAnalysisId(), INCLUDE_CLINICAL_ANALYSIS, userId).first(); - authorizationManager.checkClinicalAnalysisPermission(study.getUid(), clinicalAnalysis.getUid(), userId, - ClinicalAnalysisPermissions.WRITE); +// if (clinicalAnalysis.isLocked()) { +// throw new CatalogException("Could not update the Interpretation. Case is locked so no further modifications can be made to" +// + " the Interpretation."); +// } + if (clinicalAnalysis.getStatus().getType() == ClinicalStatusValue.ClinicalStatusType.CLOSED) { + throw new CatalogException("Cannot update the Interpretation. Case status is " + ClinicalStatusValue.ClinicalStatusType.CLOSED); + } + + InterpretationStudyConfiguration interpretationStudyConfiguration = study.getInternal().getConfiguration().getClinical() + .getInterpretation(); + // Get the interpretation status that are CLOSED and DONE + Set closedStatus = new HashSet<>(); + Set doneStatus = new HashSet<>(); + for (ClinicalStatusValue clinicalStatusValue : interpretationStudyConfiguration.getStatus()) { + if (clinicalStatusValue.getType().equals(ClinicalStatusValue.ClinicalStatusType.CLOSED)) { + closedStatus.add(clinicalStatusValue.getId()); + } else if (clinicalStatusValue.getType().equals(ClinicalStatusValue.ClinicalStatusType.DONE)) { + doneStatus.add(clinicalStatusValue.getId()); + } + } + + // If the current interpretation: + // - is locked + // - the user wants to update the locked status + // - the user wants to update the status to/from a done|closed status + boolean adminPermissionsChecked = false; + if (interpretation.isLocked() + || interpretation.getStatus().getType() == ClinicalStatusValue.ClinicalStatusType.CLOSED + || interpretation.getStatus().getType() == ClinicalStatusValue.ClinicalStatusType.DONE + || updateParams.getLocked() != null + || (updateParams.getStatus() != null && (closedStatus.contains(updateParams.getStatus().getId()) + || doneStatus.contains(updateParams.getStatus().getId())))) { + authorizationManager.checkClinicalAnalysisPermission(organizationId, study.getUid(), clinicalAnalysis.getUid(), userId, + ClinicalAnalysisPermissions.ADMIN); + + // Current status is of type CLOSED + if (interpretation.getStatus().getType() == ClinicalStatusValue.ClinicalStatusType.CLOSED) { + // The only allowed action is to remove the CLOSED status + if (updateParams.getStatus() == null || StringUtils.isEmpty(updateParams.getStatus().getId())) { + throw new CatalogException("Cannot update a Interpretation with a " + ClinicalStatusValue.ClinicalStatusType.CLOSED + + " status. You need to remove the " + ClinicalStatusValue.ClinicalStatusType.CLOSED + " status to be able " + + "to perform further updates on the Interpretation."); + } else if (closedStatus.contains(updateParams.getStatus().getId())) { + // Users should be able to change from one CLOSED status to a different one but we should still control that no further + // modifications are made + if (parameters.size() > 1) { + throw new CatalogException("Cannot update a Interpretation with a " + ClinicalStatusValue.ClinicalStatusType.CLOSED + + " status. You need to remove the " + ClinicalStatusValue.ClinicalStatusType.CLOSED + " status to be able " + + "to perform further updates on the Interpretation."); + } else if (interpretation.getStatus().getId().equals(updateParams.getStatus().getId())) { + throw new CatalogException("Interpretation already have the status '" + interpretation.getStatus().getId() + + "' of type " + ClinicalStatusValue.ClinicalStatusType.CLOSED); + } + } + } + + adminPermissionsChecked = true; + } + + if (!adminPermissionsChecked) { + authorizationManager.checkClinicalAnalysisPermission(organizationId, study.getUid(), clinicalAnalysis.getUid(), userId, + ClinicalAnalysisPermissions.WRITE); + } - if (clinicalAnalysis.isLocked()) { - throw new CatalogException("Could not update the Interpretation. Case is locked so no further modifications can be made to" - + " the Interpretation."); + if (clinicalAnalysis.isLocked() && updateParams.getLocked() != null && !updateParams.getLocked()) { + throw new CatalogException("Could not unlock the Interpretation. Case is locked so unlocking the Interpretation is not" + + " allowed."); } List events = new ArrayList<>(); @@ -906,20 +1014,11 @@ private OpenCGAResult update(Study study, Interpretation interpretation, Interpr ParamUtils.checkDateFormat(updateParams.getModificationDate(), InterpretationDBAdaptor.QueryParams.MODIFICATION_DATE.key()); } - ObjectMap parameters = new ObjectMap(); - if (updateParams != null) { - try { - parameters = updateParams.getUpdateMap(); - } catch (JsonProcessingException e) { - throw new CatalogException("Could not parse InterpretationUpdateParams object: " + e.getMessage(), e); - } - } - - if (!parameters.isEmpty() && interpretation.isLocked() - && parameters.getBoolean(InterpretationDBAdaptor.QueryParams.LOCKED.key(), true)) { - throw new CatalogException("Could not update the Interpretation. Interpretation '" + interpretation.getId() - + " is locked. Please, unlock it first."); - } +// if (!parameters.isEmpty() && interpretation.isLocked() +// && parameters.getBoolean(InterpretationDBAdaptor.QueryParams.LOCKED.key(), true)) { +// throw new CatalogException("Could not update the Interpretation. Interpretation '" + interpretation.getId() +// + " is locked. Please, unlock it first."); +// } if (updateParams != null && updateParams.getComments() != null && !updateParams.getComments().isEmpty()) { List comments = new ArrayList<>(updateParams.getComments().size()); @@ -956,16 +1055,16 @@ private OpenCGAResult update(Study study, Interpretation interpretation, Interpr } if (updateParams != null && CollectionUtils.isNotEmpty(updateParams.getPanels())) { - if (clinicalAnalysis.isPanelLock()) { - throw new CatalogException("Updating panels from Interpretation is not allowed. 'panelLock' from ClinicalAnalysis is set " - + "to True."); + if (clinicalAnalysis.isPanelLocked()) { + throw new CatalogException("Updating panels from Interpretation is not allowed. '" + + ClinicalAnalysisDBAdaptor.QueryParams.PANEL_LOCKED.key() + "' from ClinicalAnalysis is set to True."); } // Validate and get panels List panelIds = updateParams.getPanels().stream().map(PanelReferenceParam::getId).collect(Collectors.toList()); Query query = new Query(PanelDBAdaptor.QueryParams.ID.key(), panelIds); - OpenCGAResult panelResult = - panelDBAdaptor.get(study.getUid(), query, PanelManager.INCLUDE_PANEL_IDS, userId); + OpenCGAResult panelResult = + getPanelDBAdaptor(organizationId).get(study.getUid(), query, PanelManager.INCLUDE_PANEL_IDS, userId); if (panelResult.getNumResults() < panelIds.size()) { throw new CatalogException("Some panels were not found or user doesn't have permissions to see them"); } @@ -979,7 +1078,7 @@ private OpenCGAResult update(Study study, Interpretation interpretation, Interpr QueryOptions userOptions = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(UserDBAdaptor.QueryParams.ID.key(), UserDBAdaptor.QueryParams.NAME.key(), UserDBAdaptor.QueryParams.EMAIL.key())); // Check user exists - OpenCGAResult userResult = userDBAdaptor.get(updateParams.getAnalyst().getId(), userOptions); + OpenCGAResult userResult = getUserDBAdaptor(organizationId).get(updateParams.getAnalyst().getId(), userOptions); if (userResult.getNumResults() == 0) { throw new CatalogException("User '" + updateParams.getAnalyst().getId() + "' not found"); } @@ -1042,12 +1141,12 @@ private OpenCGAResult update(Study study, Interpretation interpretation, Interpr } if (parameters.containsKey(InterpretationDBAdaptor.QueryParams.STATUS.key())) { - interpretation.setStatus(updateParams.getStatus().toStatus()); - validateStatusParameter(interpretation, clinicalAnalysis.getType(), interpretationConfiguration); + interpretation.setStatus(updateParams.getStatus().toClinicalStatus()); + validateStatusParameter(interpretation, interpretationConfiguration, userId, false); parameters.put(InterpretationDBAdaptor.QueryParams.STATUS.key(), interpretation.getStatus()); if (StringUtils.isNotEmpty(interpretation.getStatus().getId())) { - List clinicalStatusValues = interpretationConfiguration.getStatus().get(clinicalAnalysis.getType()); + List clinicalStatusValues = interpretationConfiguration.getStatus(); for (ClinicalStatusValue clinicalStatusValue : clinicalStatusValues) { if (interpretation.getStatus().getId().equals(clinicalStatusValue.getId()) && clinicalStatusValue.getType() == ClinicalStatusValue.ClinicalStatusType.CLOSED) { @@ -1061,21 +1160,25 @@ private OpenCGAResult update(Study study, Interpretation interpretation, Interpr } } - OpenCGAResult update = interpretationDBAdaptor.update(interpretation.getUid(), parameters, clinicalAuditList, as, - options); + OpenCGAResult update = getInterpretationDBAdaptor(organizationId).update(interpretation.getUid(), parameters, + clinicalAuditList, as, options); update.addEvents(events); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { // Fetch updated interpretation - OpenCGAResult result = interpretationDBAdaptor.get(study.getUid(), interpretation.getId(), options); + OpenCGAResult result = getInterpretationDBAdaptor(organizationId).get(study.getUid(), interpretation.getId(), + options); update.setResults(result.getResults()); } return update; } - public OpenCGAResult revert(String studyStr, String clinicalAnalysisId, String interpretationId, int version, - String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId, StudyManager.INCLUDE_CONFIGURATION); + public OpenCGAResult revert(String studyStr, String clinicalAnalysisId, String interpretationId, + int version, String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, StudyManager.INCLUDE_CONFIGURATION, userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("study", studyStr) @@ -1086,24 +1189,26 @@ public OpenCGAResult revert(String studyStr, String clinicalAnal String interpretationUuid = ""; try { - OpenCGAResult clinicalResult = catalogManager.getClinicalAnalysisManager().internalGet(study.getUid(), - clinicalAnalysisId, INCLUDE_CLINICAL_ANALYSIS, userId); + OpenCGAResult clinicalResult = catalogManager.getClinicalAnalysisManager().internalGet(organizationId, + study.getUid(), clinicalAnalysisId, INCLUDE_CLINICAL_ANALYSIS, userId); if (clinicalResult.getNumResults() == 0) { throw new CatalogException("Could not find ClinicalAnalysis '" + clinicalAnalysisId + "'"); } ClinicalAnalysis clinicalAnalysis = clinicalResult.first(); - authorizationManager.checkClinicalAnalysisPermission(study.getUid(), clinicalAnalysis.getUid(), userId, + authorizationManager.checkClinicalAnalysisPermission(organizationId, study.getUid(), clinicalAnalysis.getUid(), userId, ClinicalAnalysisPermissions.WRITE); if (clinicalAnalysis.isLocked()) { throw new CatalogException("Could not revert the Interpretation. Case is locked so no further modifications can be made to" + " the Interpretation."); } - if (clinicalAnalysis.isPanelLock()) { - throw new CatalogException("Could not revert the Interpretation. 'panelLock' is set to True, so no further modifications" + if (clinicalAnalysis.isPanelLocked()) { + throw new CatalogException("Could not revert the Interpretation. '" + + ClinicalAnalysisDBAdaptor.QueryParams.PANEL_LOCKED.key() + "' is set to True, so no further modifications" + " can be made to the Interpretation."); } - OpenCGAResult result = internalGet(study.getUid(), interpretationId, INCLUDE_INTERPRETATION_IDS, userId); + OpenCGAResult result = internalGet(organizationId, study.getUid(), interpretationId, INCLUDE_INTERPRETATION_IDS, + userId); if (result.getNumResults() == 0) { throw new CatalogException("Could not find interpretation '" + interpretationId + "'"); } @@ -1132,16 +1237,17 @@ public OpenCGAResult revert(String studyStr, String clinicalAnal List clinicalAuditList = new ArrayList<>(); clinicalAuditList.add(new ClinicalAudit(userId, ClinicalAudit.Action.REVERT_INTERPRETATION, "Revert interpretation '" + interpretation.getId() + "' to version '" + version + "'", TimeUtils.getTime())); - OpenCGAResult revert = interpretationDBAdaptor.revert(interpretation.getUid(), version, clinicalAuditList); + OpenCGAResult revert = getInterpretationDBAdaptor(organizationId).revert(interpretation.getUid(), version, + clinicalAuditList); - auditManager.audit(userId, Enums.Action.REVERT, Enums.Resource.INTERPRETATION, interpretation.getId(), + auditManager.audit(organizationId, userId, Enums.Action.REVERT, Enums.Resource.INTERPRETATION, interpretation.getId(), interpretation.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return revert; } catch (CatalogDBException e) { logger.error("Could not revert interpretation {}", interpretationId, e); - auditManager.audit(userId, Enums.Action.REVERT, Enums.Resource.INTERPRETATION, interpretationId, + auditManager.audit(organizationId, userId, Enums.Action.REVERT, Enums.Resource.INTERPRETATION, interpretationId, interpretationUuid, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); CatalogException exception = new CatalogException("Could not revert interpretation '" + interpretationId + "'"); @@ -1149,7 +1255,7 @@ public OpenCGAResult revert(String studyStr, String clinicalAnal throw exception; } catch (CatalogException e) { logger.error("Could not revert interpretation {}: {}", interpretationId, e.getMessage(), e); - auditManager.audit(userId, Enums.Action.REVERT, Enums.Resource.INTERPRETATION, interpretationId, + auditManager.audit(organizationId, userId, Enums.Action.REVERT, Enums.Resource.INTERPRETATION, interpretationId, interpretationUuid, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw new CatalogException("Could not revert interpretation '" + interpretationId + "': " + e.getMessage()); @@ -1163,33 +1269,36 @@ public DBIterator iterator(String studyStr, Query query, QueryOp } @Override - public OpenCGAResult search(String studyId, Query query, QueryOptions options, String token) - throws CatalogException { + public OpenCGAResult search(String studyId, Query query, QueryOptions options, String token) throws CatalogException { query = ParamUtils.defaultObject(query, Query::new); options = ParamUtils.defaultObject(options, QueryOptions::new); - String userId = catalogManager.getUserManager().getUserId(token); - Study study = catalogManager.getStudyManager().resolveId(studyId, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyId, userId, organizationId); - fixQueryObject(study, query, userId); + fixQueryObject(organizationId, study, query, userId); query.append(InterpretationDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - OpenCGAResult queryResult = interpretationDBAdaptor.get(study.getUid(), query, options, userId); + OpenCGAResult queryResult = getInterpretationDBAdaptor(organizationId).get(study.getUid(), query, options, userId); List results = new ArrayList<>(queryResult.getResults().size()); for (Interpretation interpretation : queryResult.getResults()) { if (StringUtils.isNotEmpty(interpretation.getClinicalAnalysisId())) { try { - catalogManager.getClinicalAnalysisManager().internalGet(study.getUid(), interpretation.getClinicalAnalysisId(), - ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS, userId); + catalogManager.getClinicalAnalysisManager().internalGet(organizationId, study.getUid(), + interpretation.getClinicalAnalysisId(), ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS, userId); results.add(interpretation); } catch (CatalogException e) { // Maybe the clinical analysis was deleted Query clinicalQuery = new Query(ClinicalAnalysisDBAdaptor.QueryParams.DELETED.key(), true); try { - catalogManager.getClinicalAnalysisManager().internalGet(study.getUid(), interpretation.getClinicalAnalysisId(), - clinicalQuery, ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS, userId); + catalogManager.getClinicalAnalysisManager().internalGet(organizationId, study.getUid(), + interpretation.getClinicalAnalysisId(), clinicalQuery, ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS, + userId); results.add(interpretation); } catch (CatalogException e1) { logger.debug("Removing interpretation " + interpretation.getUuid() + " from results. User " + userId @@ -1209,8 +1318,11 @@ public OpenCGAResult search(String studyId, Query query, QueryOp public OpenCGAResult distinct(String studyId, List fields, Query query, String token) throws CatalogException { query = ParamUtils.defaultObject(query, Query::new); - String userId = userManager.getUserId(token); - Study study = catalogManager.getStudyManager().resolveId(studyId, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyId, userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) @@ -1218,17 +1330,17 @@ public OpenCGAResult distinct(String studyId, List fields, Query quer .append("query", new Query(query)) .append("token", token); try { - fixQueryObject(study, query, userId); + fixQueryObject(organizationId, study, query, userId); query.append(InterpretationDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - OpenCGAResult result = interpretationDBAdaptor.distinct(study.getUid(), fields, query, userId); + OpenCGAResult result = getInterpretationDBAdaptor(organizationId).distinct(study.getUid(), fields, query, userId); - auditManager.auditDistinct(userId, Enums.Resource.INTERPRETATION, study.getId(), study.getUuid(), auditParams, + auditManager.auditDistinct(organizationId, userId, Enums.Resource.INTERPRETATION, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return result; } catch (CatalogException e) { - auditManager.auditDistinct(userId, Enums.Resource.INTERPRETATION, study.getId(), study.getUuid(), auditParams, + auditManager.auditDistinct(organizationId, userId, Enums.Resource.INTERPRETATION, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -1255,8 +1367,11 @@ public OpenCGAResult delete(String studyStr, String clinicalAnalysisId, List internalResult = internalGet(study.getUid(), id, INCLUDE_INTERPRETATION_IDS, userId); + OpenCGAResult internalResult = internalGet(organizationId, study.getUid(), id, INCLUDE_INTERPRETATION_IDS, + userId); if (internalResult.getNumResults() == 0) { throw new CatalogException("Interpretation '" + id + "' not found"); } @@ -1325,9 +1442,9 @@ public OpenCGAResult delete(String studyStr, String clinicalAnalysisId, List fields, QueryOptions options, String sessionId) + public OpenCGAResult groupBy(@Nullable String studyStr, Query query, List fields, QueryOptions options, String token) throws CatalogException { return null; } - protected void fixQueryObject(Study study, Query query, String user) throws CatalogException { + protected void fixQueryObject(String organizationId, Study study, Query query, String user) throws CatalogException { changeQueryId(query, ParamConstants.INTERPRETATION_ANALYST_ID_PARAM, InterpretationDBAdaptor.QueryParams.ANALYST_ID.key()); changeQueryId(query, ParamConstants.INTERPRETATION_METHOD_NAME_PARAM, InterpretationDBAdaptor.QueryParams.METHOD_NAME.key()); changeQueryId(query, ParamConstants.INTERPRETATION_PRIMARY_FINDINGS_IDS_PARAM, @@ -1458,7 +1577,8 @@ protected void fixQueryObject(Study study, Query query, String user) throws Cata PanelDBAdaptor.QueryParams fieldFilter = catalogManager.getPanelManager().getFieldFilter(panelList); Query tmpQuery = new Query(fieldFilter.key(), panelList); - OpenCGAResult result = panelDBAdaptor.get(study.getUid(), tmpQuery, PanelManager.INCLUDE_PANEL_IDS, user); + OpenCGAResult result = getPanelDBAdaptor(organizationId).get(study.getUid(), tmpQuery, PanelManager.INCLUDE_PANEL_IDS, + user); if (result.getNumResults() > 0) { query.put(InterpretationDBAdaptor.QueryParams.PANELS_UID.key(), result.getResults().stream().map(Panel::getUid).collect(Collectors.toList())); @@ -1469,17 +1589,15 @@ protected void fixQueryObject(Study study, Query query, String user) throws Cata } } - private void validateStatusParameter(Interpretation interpretation, ClinicalAnalysis.Type type, - InterpretationStudyConfiguration interpretationConfiguration) throws CatalogException { + private void validateStatusParameter(Interpretation interpretation, InterpretationStudyConfiguration interpretationConfiguration, + String userId, boolean initIfUndefined) throws CatalogException { // Status - if (interpretationConfiguration.getStatus() == null - || CollectionUtils.isEmpty(interpretationConfiguration.getStatus().get(type))) { - throw new CatalogException("Missing status configuration in study for type '" + type - + "'. Please add a proper set of valid statuses."); + if (CollectionUtils.isEmpty(interpretationConfiguration.getStatus())) { + throw new CatalogException("Missing status configuration in study. Please add a proper set of valid statuses."); } if (StringUtils.isNotEmpty(interpretation.getStatus().getId())) { Map statusMap = new HashMap<>(); - for (ClinicalStatusValue status : interpretationConfiguration.getStatus().get(type)) { + for (ClinicalStatusValue status : interpretationConfiguration.getStatus()) { statusMap.put(status.getId(), status); } if (!statusMap.containsKey(interpretation.getStatus().getId())) { @@ -1488,7 +1606,25 @@ private void validateStatusParameter(Interpretation interpretation, ClinicalAnal } ClinicalStatusValue clinicalStatusValue = statusMap.get(interpretation.getStatus().getId()); interpretation.getStatus().setDescription(clinicalStatusValue.getDescription()); - interpretation.getStatus().setDate(TimeUtils.getTime()); + interpretation.getStatus().setType(clinicalStatusValue.getType()); + } else { + if (initIfUndefined) { + // Look for first status of type NOT_STARTED + for (ClinicalStatusValue status : interpretationConfiguration.getStatus()) { + if (status.getType() == ClinicalStatusValue.ClinicalStatusType.NOT_STARTED) { + interpretation.getStatus().setId(status.getId()); + interpretation.getStatus().setDescription(status.getDescription()); + interpretation.getStatus().setType(status.getType()); + break; + } + } + } else { + throw new CatalogException("Missing status id in Interpretation"); + } } + interpretation.getStatus().setDate(TimeUtils.getTime()); + interpretation.getStatus().setVersion(GitRepositoryState.getInstance().getBuildVersion()); + interpretation.getStatus().setCommit(GitRepositoryState.getInstance().getCommitId()); + interpretation.getStatus().setAuthor(userId); } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/JobManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/JobManager.java index 96664425beb..fe7ffc18e44 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/JobManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/JobManager.java @@ -21,7 +21,10 @@ import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.StopWatch; -import org.opencb.commons.datastore.core.*; +import org.opencb.commons.datastore.core.Event; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.result.Error; import org.opencb.commons.utils.ListUtils; import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager; @@ -33,16 +36,17 @@ import org.opencb.opencga.catalog.io.IOManager; import org.opencb.opencga.catalog.io.IOManagerFactory; import org.opencb.opencga.catalog.models.InternalGetDataResult; -import org.opencb.opencga.catalog.stats.solr.CatalogSolrManager; -import org.opencb.opencga.catalog.utils.AnnotationUtils; +import org.opencb.opencga.catalog.utils.CatalogFqn; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.catalog.utils.UuidUtils; +import org.opencb.opencga.core.api.FieldConstants; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.config.Execution; import org.opencb.opencga.core.models.AclEntryList; import org.opencb.opencga.core.models.AclParams; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.audit.AuditRecord; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.file.File; @@ -108,9 +112,9 @@ Enums.Resource getEntity() { //// QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList( //// JobDBAdaptor.QueryParams.UUID.key(), JobDBAdaptor.QueryParams.UID.key(), JobDBAdaptor.QueryParams.STUDY_UID.key(), //// JobDBAdaptor.QueryParams.ID.key(), JobDBAdaptor.QueryParams.STATUS.key())); -// OpenCGAResult jobDataResult = jobDBAdaptor.get(studyUid, queryCopy, options, user); +// OpenCGAResult jobDataResult = getJobDBAdaptor(organizationId).get(studyUid, queryCopy, options, user); // if (jobDataResult.getNumResults() == 0) { -// jobDataResult = jobDBAdaptor.get(queryCopy, options); +// jobDataResult = getJobDBAdaptor(organizationId).get(queryCopy, options); // if (jobDataResult.getNumResults() == 0) { // throw new CatalogException("Job " + entry + " not found"); // } else { @@ -124,8 +128,8 @@ Enums.Resource getEntity() { // } @Override - InternalGetDataResult internalGet(long studyUid, List entryList, @Nullable Query query, QueryOptions options, String user, - boolean ignoreException) throws CatalogException { + InternalGetDataResult internalGet(String organizationId, long studyUid, List entryList, @Nullable Query query, + QueryOptions options, String user, boolean ignoreException) throws CatalogException { if (ListUtils.isEmpty(entryList)) { throw new CatalogException("Missing job entries."); } @@ -161,21 +165,21 @@ InternalGetDataResult internalGet(long studyUid, List entryList, @N } // If studyUid has not been provided, we will look for - OpenCGAResult jobDataResult = jobDBAdaptor.get(queryCopy, options); + OpenCGAResult jobDataResult = getJobDBAdaptor(organizationId).get(queryCopy, options); for (Job job : jobDataResult.getResults()) { // Check view permissions - authorizationManager.checkJobPermission(job.getStudyUid(), job.getUid(), user, JobPermissions.VIEW); + authorizationManager.checkJobPermission(organizationId, job.getStudyUid(), job.getUid(), user, JobPermissions.VIEW); } return keepOriginalOrder(uniqueList, jobStringFunction, jobDataResult, ignoreException, false); } queryCopy.put(JobDBAdaptor.QueryParams.STUDY_UID.key(), studyUid); - OpenCGAResult jobDataResult = jobDBAdaptor.get(studyUid, queryCopy, options, user); + OpenCGAResult jobDataResult = getJobDBAdaptor(organizationId).get(studyUid, queryCopy, options, user); if (ignoreException || jobDataResult.getNumResults() == uniqueList.size()) { return keepOriginalOrder(uniqueList, jobStringFunction, jobDataResult, ignoreException, false); } // Query without adding the user check - OpenCGAResult resultsNoCheck = jobDBAdaptor.get(queryCopy, queryOptions); + OpenCGAResult resultsNoCheck = getJobDBAdaptor(organizationId).get(queryCopy, queryOptions); if (resultsNoCheck.getNumResults() == jobDataResult.getNumResults()) { throw CatalogException.notFound("jobs", getMissingFields(uniqueList, jobDataResult.getResults(), jobStringFunction)); } else { @@ -183,40 +187,23 @@ InternalGetDataResult internalGet(long studyUid, List entryList, @N } } - private OpenCGAResult getJob(long studyUid, String jobUuid, QueryOptions options) throws CatalogException { + private OpenCGAResult getJob(String organizationId, long studyUid, String jobUuid, QueryOptions options) throws CatalogException { Query query = new Query() .append(JobDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(JobDBAdaptor.QueryParams.UUID.key(), jobUuid); - return jobDBAdaptor.get(query, options); + return getJobDBAdaptor(organizationId).get(query, options); } - public Long getStudyId(long jobId) throws CatalogException { - return jobDBAdaptor.getStudyId(jobId); - } - - public Study getStudy(Job job, String sessionId) throws CatalogException { - ParamUtils.checkObj(job, "job"); - ParamUtils.checkObj(sessionId, "session id"); - - if (job.getStudyUid() <= 0) { - throw new CatalogException("Missing study uid field in job"); - } - - String user = catalogManager.getUserManager().getUserId(sessionId); - - Query query = new Query(StudyDBAdaptor.QueryParams.UID.key(), job.getStudyUid()); - OpenCGAResult studyDataResult = studyDBAdaptor.get(query, QueryOptions.empty(), user); - if (studyDataResult.getNumResults() == 1) { - return studyDataResult.first(); - } else { - authorizationManager.checkCanViewStudy(job.getStudyUid(), user); - throw new CatalogException("Incorrect study uid"); - } + public Long getStudyId(String organizationId, long jobId) throws CatalogException { + return getJobDBAdaptor(organizationId).getStudyId(jobId); } public OpenCGAResult visit(String studyId, String jobId, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyId, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyId, userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) @@ -224,24 +211,27 @@ public OpenCGAResult visit(String studyId, String jobId, String token) thro .append("token", token); try { JobUpdateParams updateParams = new JobUpdateParams().setVisited(true); - Job job = internalGet(study.getUid(), jobId, INCLUDE_JOB_IDS, userId).first(); + Job job = internalGet(organizationId, study.getUid(), jobId, INCLUDE_JOB_IDS, userId).first(); - OpenCGAResult result = update(study, job, updateParams, QueryOptions.empty(), userId); - auditManager.audit(userId, Enums.Action.VISIT, Enums.Resource.JOB, job.getId(), job.getUuid(), study.getId(), + OpenCGAResult result = update(organizationId, study, job, updateParams, QueryOptions.empty(), userId); + auditManager.audit(organizationId, userId, Enums.Action.VISIT, Enums.Resource.JOB, job.getId(), job.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return result; - } catch (CatalogException e) { - auditManager.audit(userId, Enums.Action.VISIT, Enums.Resource.JOB, jobId, "", study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + } catch (Exception e) { + auditManager.audit(organizationId, userId, Enums.Action.VISIT, Enums.Resource.JOB, jobId, "", study.getId(), study.getUuid(), + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); throw e; } } @Override public OpenCGAResult create(String studyStr, Job job, QueryOptions options, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = catalogManager.getStudyManager().resolveId(studyStr, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyStr, userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("study", studyStr) @@ -251,7 +241,7 @@ public OpenCGAResult create(String studyStr, Job job, QueryOptions options, try { options = ParamUtils.defaultObject(options, QueryOptions::new); - authorizationManager.checkStudyPermission(study.getUid(), userId, StudyPermissions.Permissions.WRITE_JOBS); + authorizationManager.checkStudyPermission(organizationId, study.getUid(), userId, StudyPermissions.Permissions.WRITE_JOBS); ParamUtils.checkObj(job, "Job"); ParamUtils.checkIdentifier(job.getId(), "job id"); @@ -285,49 +275,49 @@ public OpenCGAResult create(String studyStr, Job job, QueryOptions options, if (ListUtils.isNotEmpty(job.getInput())) { List inputFiles = new ArrayList<>(job.getInput().size()); for (File file : job.getInput()) { - inputFiles.add(getFile(study.getUid(), file.getPath(), userId)); + inputFiles.add(getFile(organizationId, study.getUid(), file.getPath(), userId)); } job.setInput(inputFiles); } if (ListUtils.isNotEmpty(job.getOutput())) { List outputFiles = new ArrayList<>(job.getOutput().size()); for (File file : job.getOutput()) { - outputFiles.add(getFile(study.getUid(), file.getPath(), userId)); + outputFiles.add(getFile(organizationId, study.getUid(), file.getPath(), userId)); } job.setOutput(outputFiles); } if (job.getOutDir() != null && StringUtils.isNotEmpty(job.getOutDir().getPath())) { - job.setOutDir(getFile(study.getUid(), job.getOutDir().getPath(), userId)); + job.setOutDir(getFile(organizationId, study.getUid(), job.getOutDir().getPath(), userId)); if (job.getOutDir().getType() != File.Type.DIRECTORY) { throw new CatalogException("Unexpected outDir type. Expected " + File.Type.DIRECTORY); } } if (job.getStdout() != null && StringUtils.isNotEmpty(job.getStdout().getPath())) { - job.setStdout(getFile(study.getUid(), job.getStdout().getPath(), userId)); + job.setStdout(getFile(organizationId, study.getUid(), job.getStdout().getPath(), userId)); } if (job.getStderr() != null && StringUtils.isNotEmpty(job.getStderr().getPath())) { - job.setStderr(getFile(study.getUid(), job.getStderr().getPath(), userId)); + job.setStderr(getFile(organizationId, study.getUid(), job.getStderr().getPath(), userId)); } job.setUuid(UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.JOB)); - OpenCGAResult insert = jobDBAdaptor.insert(study.getUid(), job, options); + OpenCGAResult insert = getJobDBAdaptor(organizationId).insert(study.getUid(), job, options); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { // Fetch created job - OpenCGAResult queryResult = getJob(study.getUid(), job.getUuid(), options); + OpenCGAResult queryResult = getJob(organizationId, study.getUid(), job.getUuid(), options); insert.setResults(queryResult.getResults()); } - auditManager.auditCreate(userId, Enums.Resource.JOB, job.getId(), job.getUuid(), study.getId(), study.getUuid(), + auditManager.auditCreate(organizationId, userId, Enums.Resource.JOB, job.getId(), job.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return insert; - } catch (CatalogException e) { - auditManager.auditCreate(userId, Enums.Resource.JOB, job.getId(), "", study.getId(), study.getUuid(), auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + } catch (Exception e) { + auditManager.auditCreate(organizationId, userId, Enums.Resource.JOB, job.getId(), "", study.getId(), study.getUuid(), + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); throw e; } } - private void autoCompleteNewJob(Study study, Job job, String token) throws CatalogException { + private void autoCompleteNewJob(String organizationId, Study study, Job job, JwtPayload tokenPayload) throws CatalogException { ParamUtils.checkObj(job, "Job"); // Auto generate id @@ -345,6 +335,27 @@ private void autoCompleteNewJob(Study study, Job job, String token) throws Catal job.setInternal(JobInternal.init()); job.getInternal().setWebhook(new JobInternalWebhook(study.getNotification().getWebhook(), new HashMap<>())); + if (StringUtils.isNotEmpty(job.getParentId())) { + // Check parent job exists + try { + Job tmpJob = internalGet(organizationId, study.getUid(), job.getParentId(), INCLUDE_JOB_IDS, + tokenPayload.getUserId(organizationId)).first(); + job.setParentId(tmpJob.getId()); + } catch (CatalogException e) { + throw new CatalogException("Parent job '" + job.getParentId() + "' not found", e); + } + } else { + job.setParentId(""); + } + + if (StringUtils.isNotEmpty(job.getScheduledStartTime())) { + ParamUtils.checkDateFormat(job.getScheduledStartTime(), FieldConstants.JOB_SCHEDULED_START_TIME); + Date date = TimeUtils.toDate(job.getScheduledStartTime()); + if (date.before(new Date())) { + throw new CatalogException("'" + FieldConstants.JOB_SCHEDULED_START_TIME + "' must be a future date"); + } + } + if (job.getDependsOn() != null && !job.getDependsOn().isEmpty()) { boolean uuidProvided = job.getDependsOn().stream().map(Job::getId).anyMatch(UuidUtils::isOpenCgaUuid); @@ -352,10 +363,12 @@ private void autoCompleteNewJob(Study study, Job job, String token) throws Catal // If uuid is provided, we will remove the study uid from the query so it can be searched across any study InternalGetDataResult dependsOnResult; if (uuidProvided) { - dependsOnResult = internalGet(0, job.getDependsOn().stream().map(Job::getId).collect(Collectors.toList()), null, - INCLUDE_JOB_IDS, job.getUserId(), false); + dependsOnResult = internalGet(organizationId, 0, + job.getDependsOn().stream().map(Job::getId).collect(Collectors.toList()), + null, INCLUDE_JOB_IDS, job.getUserId(), false); } else { - dependsOnResult = internalGet(study.getUid(), job.getDependsOn().stream().map(Job::getId).collect(Collectors.toList()), + dependsOnResult = internalGet(organizationId, study.getUid(), + job.getDependsOn().stream().map(Job::getId).collect(Collectors.toList()), null, INCLUDE_JOB_IDS, job.getUserId(), false); } job.setDependsOn(dependsOnResult.getResults()); @@ -367,13 +380,61 @@ private void autoCompleteNewJob(Study study, Job job, String token) throws Catal } else { // We only check input files if the job does not depend on other job that might be creating the necessary file. - List inputFiles = getJobInputFilesFromParams(study.getFqn(), job, token); + List inputFiles = getJobInputFilesFromParams(study.getFqn(), job, tokenPayload.getToken()); job.setInput(inputFiles); } job.setAttributes(ParamUtils.defaultObject(job.getAttributes(), HashMap::new)); } + public OpenCGAResult kill(String studyStr, String jobId, String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyStr, userId, organizationId); + + ObjectMap auditParams = new ObjectMap() + .append("study", studyStr) + .append("jobId", jobId) + .append("token", token); + String jobUuid = ""; + try { + QueryOptions options = keepFieldInQueryOptions(INCLUDE_JOB_IDS, JobDBAdaptor.QueryParams.USER_ID.key()); + Job job = internalGet(organizationId, study.getUid(), jobId, options, userId).first(); + jobId = job.getId(); + jobUuid = job.getUuid(); + + try { + if (!job.getUserId().equals(userId)) { + // Check if the user is a study administrator + authorizationManager.checkIsAtLeastStudyAdministrator(organizationId, study.getUid(), userId); + } + } catch (CatalogException e) { + throw CatalogAuthorizationException.deny(userId, "job", jobId, CatalogAuthorizationException.ErrorCode.KILL_JOB, e); + } + + if (!job.getInternal().getStatus().getId().equals(Enums.ExecutionStatus.PENDING) + && !job.getInternal().getStatus().getId().equals(Enums.ExecutionStatus.QUEUED) + && !job.getInternal().getStatus().getId().equals(Enums.ExecutionStatus.RUNNING)) { + throw new CatalogException("Cannot kill job '" + jobId + "' in status " + job.getInternal().getStatus().getId()); + } + + ObjectMap params = new ObjectMap(JobDBAdaptor.QueryParams.INTERNAL_KILL_JOB_REQUESTED.key(), true); + OpenCGAResult update = getCatalogDBAdaptorFactory().getCatalogJobDBAdaptor(organizationId).update(job.getUid(), params, + QueryOptions.empty()); + + auditManager.audit(organizationId, userId, Enums.Action.KILL_JOB, Enums.Resource.JOB, jobId, jobUuid, study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + return update; + } catch (Exception e) { + auditManager.audit(organizationId, userId, Enums.Action.KILL_JOB, Enums.Resource.JOB, jobId, jobUuid, study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); + throw e; + } + } + + public List getJobInputFilesFromParams(String study, Job job, String token) throws CatalogException { // Look for input files String fileParamSuffix = "file"; @@ -417,8 +478,8 @@ public List getJobInputFilesFromParams(String study, Job job, String token return inputFiles; } - public OpenCGAResult retry(String studyStr, JobRetryParams jobRetry, Enums.Priority priority, - String jobId, String jobDescription, List jobDependsOn, List jobTags, String token) + public OpenCGAResult retry(String studyStr, JobRetryParams jobRetry, Enums.Priority priority, String jobId, String jobDescription, + List jobDependsOn, List jobTags, String jobScheduledStartTime, String token) throws CatalogException { Job job = get(studyStr, jobRetry.getJob(), new QueryOptions(), token).first(); if (jobRetry.isForce() @@ -433,8 +494,8 @@ public OpenCGAResult retry(String studyStr, JobRetryParams jobRetry, Enums. if (StringUtils.isEmpty(jobDescription)) { jobDescription = "Retry from job '" + jobRetry.getJob() + "'"; } - return submit(studyStr, job.getTool().getId(), priority, params, jobId, jobDescription, jobDependsOn, jobTags, - attributes, token); + return submit(studyStr, job.getTool().getId(), priority, params, jobId, jobDescription, jobDependsOn, jobTags, job.getId(), + jobScheduledStartTime, job.isDryRun(), attributes, token); } else { throw new CatalogException("Unable to retry job with status " + job.getInternal().getStatus().getId()); } @@ -442,12 +503,12 @@ public OpenCGAResult retry(String studyStr, JobRetryParams jobRetry, Enums. public OpenCGAResult submit(String studyStr, String toolId, Enums.Priority priority, Map params, String token) throws CatalogException { - return submit(studyStr, toolId, priority, params, null, null, null, null, token); + return submit(studyStr, toolId, priority, params, null, null, null, null, null, null, false, token); } - public OpenCGAResult submitProject(String projectStr, String toolId, Enums.Priority priority, Map params, - String jobId, String jobDescription, List jobDependsOn, List jobTags, - String token) throws CatalogException { + public OpenCGAResult submitProject(String projectStr, String toolId, Enums.Priority priority, + Map params, String jobId, String jobDescription, List jobDependsOn, + List jobTags, String token) throws CatalogException { // Project job QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, StudyDBAdaptor.QueryParams.FQN.key()); // Peek any study. The ExecutionDaemon will take care of filling up the rest of studies. @@ -460,33 +521,39 @@ public OpenCGAResult submitProject(String projectStr, String toolId, Enums. if (studies.isEmpty()) { throw new CatalogException("Project '" + projectStr + "' not found!"); } - return submit(studies.get(0), toolId, priority, params, jobId, jobDescription, jobDependsOn, jobTags, token); + return submit(studies.get(0), toolId, priority, params, jobId, jobDescription, jobDependsOn, jobTags, null, null, false, token); } public OpenCGAResult submit(String studyStr, String toolId, Enums.Priority priority, Map params, String jobId, - String jobDescription, List jobDependsOn, List jobTags, String token) - throws CatalogException { - return submit(studyStr, toolId, priority, params, jobId, jobDescription, jobDependsOn, jobTags, null, token); + String jobDescription, List jobDependsOn, List jobTags, @Nullable String jobParentId, + @Nullable String scheduledStartTime, Boolean dryRun, String token) throws CatalogException { + return submit(studyStr, toolId, priority, params, jobId, jobDescription, jobDependsOn, jobTags, jobParentId, scheduledStartTime, + dryRun, null, token); } public OpenCGAResult submit(String studyStr, String toolId, Enums.Priority priority, Map params, String jobId, String jobDescription, List jobDependsOn, List jobTags, - Map attributes, String token) - throws CatalogException { - String userId = userManager.getUserId(token); - Study study = catalogManager.getStudyManager().resolveId(studyStr, userId); + @Nullable String jobParentId, @Nullable String scheduledStartTime, Boolean dryRun, + Map attributes, String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyStr, userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("study", studyStr) .append("toolId", toolId) - .append("priority", priority) + .append("jobPriority", priority) .append("params", params) .append("jobId", jobId) .append("jobDescription", jobDescription) .append("jobDependsOn", jobDependsOn) .append("jobTags", jobTags) + .append("jobParentId", jobParentId) + .append("jobScheduledStartTime", scheduledStartTime) + .append("jobDryRun", dryRun) .append("token", token); - Job job = new Job(); job.setId(jobId); job.setDescription(jobDescription); @@ -495,15 +562,18 @@ public OpenCGAResult submit(String studyStr, String toolId, Enums.Priority job.setStudy(new JobStudyParam(study.getFqn())); job.setUserId(userId); job.setParams(params); + job.setParentId(jobParentId); + job.setScheduledStartTime(scheduledStartTime); job.setPriority(priority); + job.setDryRun(dryRun != null && dryRun); job.setDependsOn(jobDependsOn != null ? jobDependsOn.stream().map(j -> new Job().setId(j)).collect(Collectors.toList()) : Collections.emptyList()); job.setAttributes(attributes); try { - autoCompleteNewJob(study, job, token); + autoCompleteNewJob(organizationId, study, job, tokenPayload); - authorizationManager.checkStudyPermission(study.getUid(), userId, StudyPermissions.Permissions.EXECUTE_JOBS); + authorizationManager.checkStudyPermission(organizationId, study.getUid(), userId, StudyPermissions.Permissions.EXECUTE_JOBS); // Check params ParamUtils.checkObj(params, "params"); @@ -513,31 +583,31 @@ public OpenCGAResult submit(String studyStr, String toolId, Enums.Priority } } - OpenCGAResult reuseJob = getJobToReuse(study, jobId, job, token); + OpenCGAResult reuseJob = getJobToReuse(organizationId, study, jobId, job, token); if (reuseJob != null) { job = reuseJob.first(); - auditManager.audit(userId, Enums.Action.REUSE, Enums.Resource.JOB, job.getId(), job.getUuid(), + auditManager.audit(organizationId, userId, Enums.Action.REUSE, Enums.Resource.JOB, job.getId(), job.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return reuseJob; } else { - jobDBAdaptor.insert(study.getUid(), job, new QueryOptions()); - OpenCGAResult jobResult = jobDBAdaptor.get(job.getUid(), new QueryOptions()); + getJobDBAdaptor(organizationId).insert(study.getUid(), job, new QueryOptions()); + OpenCGAResult jobResult = getJobDBAdaptor(organizationId).get(job.getUid(), new QueryOptions()); - auditManager.auditCreate(userId, Enums.Resource.JOB, job.getId(), "", study.getId(), study.getUuid(), auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.auditCreate(organizationId, userId, Enums.Resource.JOB, job.getId(), "", study.getId(), study.getUuid(), + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return jobResult; } - } catch (CatalogException e) { - auditManager.auditCreate(userId, Enums.Resource.JOB, job.getId(), "", study.getId(), study.getUuid(), auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + } catch (Exception e) { + auditManager.auditCreate(organizationId, userId, Enums.Resource.JOB, job.getId(), "", study.getId(), study.getUuid(), + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); job.getInternal().setStatus(new Enums.ExecutionStatus(Enums.ExecutionStatus.ABORTED)); job.getInternal().getStatus().setDescription(e.toString()); - jobDBAdaptor.insert(study.getUid(), job, new QueryOptions()); + getJobDBAdaptor(organizationId).insert(study.getUid(), job, new QueryOptions()); throw e; } @@ -546,20 +616,22 @@ public OpenCGAResult submit(String studyStr, String toolId, Enums.Priority /** * Check if the job is eligible to be reused, and if so, try to find an equivalent job. * Eligible job: - * - Not enforced jobId - * - Tool.id within list of "tools-to-reuse" + * - Not enforced jobId + * - Tool.id within list of "tools-to-reuse" * Equivalent job: - * - Internal.status.id = PENDING | QUEUED - * - Params -> Same as the input job - * - Same job dependencies - * @param study Study - * @param inputJobId Enforced JobId - * @param job Job to be created - * @param token User token + * - Internal.status.id = PENDING | QUEUED + * - Params -> Same as the input job + * - Same job dependencies + * + * @param organizationId Organization id. + * @param study Study + * @param inputJobId Enforced JobId + * @param job Job to be created + * @param token User token * @return A job to be reused (if any) * @throws CatalogException on error */ - private OpenCGAResult getJobToReuse(Study study, String inputJobId, Job job, String token) + private OpenCGAResult getJobToReuse(String organizationId, Study study, String inputJobId, Job job, String token) throws CatalogException { // First check if the job can be reused @@ -577,13 +649,13 @@ private OpenCGAResult getJobToReuse(Study study, String inputJobId, Job job .append(QueryOptions.LIMIT, 10) .append(QueryOptions.INCLUDE, Arrays.asList(JobDBAdaptor.QueryParams.UID.key(), JobDBAdaptor.QueryParams.PARAMS.key())); - DBIterator it = iterator(study.getUuid(), query, options, token); + DBIterator it = iterator(study.getFqn(), query, options, token); while (it.hasNext()) { Job candidateJob = it.next(); // Compare params orderless if (new HashMap<>(job.getParams()).equals(new HashMap<>(candidateJob.getParams()))) { // This is a valid candidate! - OpenCGAResult result = jobDBAdaptor.get(candidateJob.getUid(), new QueryOptions()); + OpenCGAResult result = getJobDBAdaptor(organizationId).get(candidateJob.getUid(), new QueryOptions()); result.addEvent(new Event(Event.Type.WARNING, "reuse", "Another job which is pending execution was already created using the exact same parameters. " + "Returning that job instead of creating a new one.")); @@ -622,30 +694,7 @@ private boolean jobEligibleToReuse(String inputJobId, Job job) { return validTool; } - public OpenCGAResult count(Query query, String token) throws CatalogException { - String userId = userManager.getUserId(token); - authorizationManager.isInstallationAdministrator(userId); - - return jobDBAdaptor.count(query); - } - - public DBIterator iterator(Query query, QueryOptions options, String token) throws CatalogException { - String userId = userManager.getUserId(token); - authorizationManager.isInstallationAdministrator(userId); - - return jobDBAdaptor.iterator(query, options); - } - - public OpenCGAResult get(long jobId, QueryOptions options, String sessionId) throws CatalogException { - return get(null, String.valueOf(jobId), options, sessionId); - } - -// public OpenCGAResult get(List jobIds, QueryOptions options, boolean ignoreException, String sessionId) -// throws CatalogException { -// return get(null, jobIds, options, ignoreException, sessionId); -// } - - private void fixQueryObject(Study study, Query query, String userId) throws CatalogException { + private void fixQueryObject(String organizationId, Study study, Query query, String userId) throws CatalogException { super.fixQueryObject(query); changeQueryId(query, ParamConstants.JOB_TOOL_ID_PARAM, JobDBAdaptor.QueryParams.TOOL_ID.key()); changeQueryId(query, ParamConstants.JOB_TOOL_TYPE_PARAM, JobDBAdaptor.QueryParams.TOOL_TYPE.key()); @@ -653,7 +702,7 @@ private void fixQueryObject(Study study, Query query, String userId) throws Cata changeQueryId(query, ParamConstants.JOB_STATUS_PARAM, JobDBAdaptor.QueryParams.STATUS_ID.key()); if (query.containsKey(ParamConstants.JOB_INPUT_FILES_PARAM)) { - List inputFiles = catalogManager.getFileManager().internalGet(study.getUid(), + List inputFiles = catalogManager.getFileManager().internalGet(organizationId, study.getUid(), query.getAsStringList(ParamConstants.JOB_INPUT_FILES_PARAM), FileManager.INCLUDE_FILE_IDS, userId, true).getResults(); if (ListUtils.isNotEmpty(inputFiles)) { query.put(JobDBAdaptor.QueryParams.INPUT_UID.key(), inputFiles.stream().map(File::getUid).collect(Collectors.toList())); @@ -664,7 +713,7 @@ private void fixQueryObject(Study study, Query query, String userId) throws Cata query.remove(ParamConstants.JOB_INPUT_FILES_PARAM); } if (query.containsKey(ParamConstants.JOB_OUTPUT_FILES_PARAM)) { - List inputFiles = catalogManager.getFileManager().internalGet(study.getUid(), + List inputFiles = catalogManager.getFileManager().internalGet(organizationId, study.getUid(), query.getAsStringList(ParamConstants.JOB_OUTPUT_FILES_PARAM), FileManager.INCLUDE_FILE_IDS, userId, true).getResults(); if (ListUtils.isNotEmpty(inputFiles)) { query.put(JobDBAdaptor.QueryParams.OUTPUT_UID.key(), inputFiles.stream().map(File::getUid).collect(Collectors.toList())); @@ -681,8 +730,11 @@ public OpenCGAResult search(String studyId, Query query, QueryOptions optio query = ParamUtils.defaultObject(query, Query::new); options = ParamUtils.defaultObject(options, QueryOptions::new); - String userId = userManager.getUserId(token); - Study study = catalogManager.getStudyManager().resolveId(studyId, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyId, userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) @@ -690,17 +742,17 @@ public OpenCGAResult search(String studyId, Query query, QueryOptions optio .append("options", options) .append("token", token); try { - fixQueryObject(study, query, userId); + fixQueryObject(organizationId, study, query, userId); query.put(JobDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - OpenCGAResult queryResult = jobDBAdaptor.get(study.getUid(), query, options, userId); - auditManager.auditSearch(userId, Enums.Resource.JOB, study.getId(), study.getUuid(), auditParams, + OpenCGAResult queryResult = getJobDBAdaptor(organizationId).get(study.getUid(), query, options, userId); + auditManager.auditSearch(organizationId, userId, Enums.Resource.JOB, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return queryResult; - } catch (CatalogException e) { - auditManager.auditSearch(userId, Enums.Resource.JOB, study.getId(), study.getUuid(), auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + } catch (Exception e) { + auditManager.auditSearch(organizationId, userId, Enums.Resource.JOB, study.getId(), study.getUuid(), auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); throw e; } } @@ -709,8 +761,11 @@ public OpenCGAResult search(String studyId, Query query, QueryOptions optio public OpenCGAResult distinct(String studyId, List fields, Query query, String token) throws CatalogException { query = ParamUtils.defaultObject(query, Query::new); - String userId = userManager.getUserId(token); - Study study = catalogManager.getStudyManager().resolveId(studyId, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyId, userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) @@ -718,61 +773,80 @@ public OpenCGAResult distinct(String studyId, List fields, Query quer .append("query", new Query(query)) .append("token", token); try { - fixQueryObject(study, query, userId); + fixQueryObject(organizationId, study, query, userId); query.append(JobDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - OpenCGAResult result = jobDBAdaptor.distinct(study.getUid(), fields, query, userId); + OpenCGAResult result = getJobDBAdaptor(organizationId).distinct(study.getUid(), fields, query, userId); - auditManager.auditDistinct(userId, Enums.Resource.JOB, study.getId(), study.getUuid(), auditParams, + auditManager.auditDistinct(organizationId, userId, Enums.Resource.JOB, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return result; - } catch (CatalogException e) { - auditManager.auditDistinct(userId, Enums.Resource.JOB, study.getId(), study.getUuid(), auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + } catch (Exception e) { + auditManager.auditDistinct(organizationId, userId, Enums.Resource.JOB, study.getId(), study.getUuid(), auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); throw e; } } + public DBIterator iteratorInOrganization(String organizationId, Query query, QueryOptions options, String token) + throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + authorizationManager.isOpencgaAdministrator(tokenPayload); + return getJobDBAdaptor(organizationId).iterator(query, options); + } + @Override public DBIterator iterator(String studyId, Query query, QueryOptions options, String token) throws CatalogException { query = ParamUtils.defaultObject(query, Query::new); options = ParamUtils.defaultObject(options, QueryOptions::new); - String userId = userManager.getUserId(token); - Study study = catalogManager.getStudyManager().resolveId(studyId, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyId, userId, organizationId); - fixQueryObject(study, query, userId); + fixQueryObject(organizationId, study, query, userId); query.put(JobDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - return jobDBAdaptor.iterator(study.getUid(), query, options, userId); + return getJobDBAdaptor(organizationId).iterator(study.getUid(), query, options, userId); + } + + public OpenCGAResult countInOrganization(String organizationId, Query query, String token) throws CatalogException { + JwtPayload jwtPayload = userManager.validateToken(token); + authorizationManager.checkIsOpencgaAdministrator(jwtPayload); + return getCatalogDBAdaptorFactory().getCatalogJobDBAdaptor(organizationId).count(query); } @Override public OpenCGAResult count(String studyId, Query query, String token) throws CatalogException { query = ParamUtils.defaultObject(query, Query::new); - String userId = userManager.getUserId(token); - Study study = catalogManager.getStudyManager().resolveId(studyId, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyId, userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) .append("query", new Query(query)) .append("token", token); try { - fixQueryObject(study, query, userId); + fixQueryObject(organizationId, study, query, userId); query.append(JobDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - OpenCGAResult queryResultAux = jobDBAdaptor.count(query, userId); + OpenCGAResult queryResultAux = getJobDBAdaptor(organizationId).count(query, userId); - auditManager.auditCount(userId, Enums.Resource.JOB, study.getId(), study.getUuid(), auditParams, + auditManager.auditCount(organizationId, userId, Enums.Resource.JOB, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return new OpenCGAResult<>(queryResultAux.getTime(), queryResultAux.getEvents(), 0, Collections.emptyList(), queryResultAux.getNumMatches()); - } catch (CatalogException e) { - auditManager.auditCount(userId, Enums.Resource.JOB, study.getId(), study.getUuid(), auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + } catch (Exception e) { + auditManager.auditCount(organizationId, userId, Enums.Resource.JOB, study.getId(), study.getUuid(), auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); throw e; } } @@ -784,8 +858,11 @@ public OpenCGAResult delete(String studyStr, List jobIds, QueryOptions o public OpenCGAResult delete(String studyStr, List jobIds, ObjectMap params, boolean ignoreException, String token) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(token); - Study study = studyManager.resolveId(studyStr, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, userId, organizationId); String operationUuid = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -799,10 +876,11 @@ public OpenCGAResult delete(String studyStr, List jobIds, ObjectMap para boolean checkPermissions; try { // If the user is the owner or the admin, we won't check if he has permissions for every single entry - checkPermissions = !authorizationManager.isOwnerOrAdmin(study.getUid(), userId); - } catch (CatalogException e) { - auditManager.auditDelete(operationUuid, userId, Enums.Resource.JOB, "", "", study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + long studyId = study.getUid(); + checkPermissions = !authorizationManager.isAtLeastStudyAdministrator(organizationId, studyId, userId); + } catch (Exception e) { + auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.JOB, "", "", study.getId(), study.getUuid(), + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); throw e; } @@ -813,7 +891,7 @@ public OpenCGAResult delete(String studyStr, List jobIds, ObjectMap para String jobUuid = ""; try { - OpenCGAResult internalResult = internalGet(study.getUid(), id, INCLUDE_JOB_IDS, userId); + OpenCGAResult internalResult = internalGet(organizationId, study.getUid(), id, INCLUDE_JOB_IDS, userId); if (internalResult.getNumResults() == 0) { throw new CatalogException("Job '" + id + "' not found"); } @@ -824,27 +902,27 @@ public OpenCGAResult delete(String studyStr, List jobIds, ObjectMap para jobUuid = job.getUuid(); if (checkPermissions) { - authorizationManager.checkJobPermission(study.getUid(), job.getUid(), userId, JobPermissions.DELETE); + authorizationManager.checkJobPermission(organizationId, study.getUid(), job.getUid(), userId, JobPermissions.DELETE); } // Check if the job can be deleted checkJobCanBeDeleted(job); - result.append(jobDBAdaptor.delete(job)); + result.append(getJobDBAdaptor(organizationId).delete(job)); - auditManager.auditDelete(operationUuid, userId, Enums.Resource.JOB, job.getId(), job.getUuid(), study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); - } catch (CatalogException e) { + auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.JOB, job.getId(), job.getUuid(), + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + } catch (Exception e) { Event event = new Event(Event.Type.ERROR, jobId, e.getMessage()); result.getEvents().add(event); result.setNumErrors(result.getNumErrors() + 1); logger.error("Cannot delete job {}: {}", jobId, e.getMessage(), e); - auditManager.auditDelete(operationUuid, userId, Enums.Resource.FAMILY, jobId, jobUuid, - study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.FAMILY, jobId, jobUuid, + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); } } - auditManager.finishAuditBatch(operationUuid); + auditManager.finishAuditBatch(organizationId, operationUuid); return endResult(result, ignoreException); } @@ -859,8 +937,11 @@ public OpenCGAResult delete(String studyId, Query query, ObjectMap params, boole Query finalQuery = new Query(ParamUtils.defaultObject(query, Query::new)); OpenCGAResult result = OpenCGAResult.empty(); - String userId = catalogManager.getUserManager().getUserId(token); - Study study = catalogManager.getStudyManager().resolveId(studyId, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyId, userId, organizationId); String operationUuid = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -877,16 +958,17 @@ public OpenCGAResult delete(String studyId, Query query, ObjectMap params, boole // We try to get an iterator containing all the jobs to be deleted DBIterator iterator; try { - fixQueryObject(study, query, userId); + fixQueryObject(organizationId, study, query, userId); finalQuery.append(JobDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - iterator = jobDBAdaptor.iterator(study.getUid(), finalQuery, INCLUDE_JOB_IDS, userId); + iterator = getJobDBAdaptor(organizationId).iterator(study.getUid(), finalQuery, INCLUDE_JOB_IDS, userId); // If the user is the owner or the admin, we won't check if he has permissions for every single entry - checkPermissions = !authorizationManager.isOwnerOrAdmin(study.getUid(), userId); - } catch (CatalogException e) { - auditManager.auditDelete(operationUuid, userId, Enums.Resource.JOB, "", "", study.getId(), study.getUuid(), auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + long studyId1 = study.getUid(); + checkPermissions = !authorizationManager.isAtLeastStudyAdministrator(organizationId, studyId1, userId); + } catch (Exception e) { + auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.JOB, "", "", study.getId(), study.getUuid(), + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); throw e; } @@ -896,17 +978,17 @@ public OpenCGAResult delete(String studyId, Query query, ObjectMap params, boole try { if (checkPermissions) { - authorizationManager.checkJobPermission(study.getUid(), job.getUid(), userId, JobPermissions.DELETE); + authorizationManager.checkJobPermission(organizationId, study.getUid(), job.getUid(), userId, JobPermissions.DELETE); } // Check if the job can be deleted checkJobCanBeDeleted(job); - result.append(jobDBAdaptor.delete(job)); + result.append(getJobDBAdaptor(organizationId).delete(job)); - auditManager.auditDelete(operationUuid, userId, Enums.Resource.JOB, job.getId(), job.getUuid(), study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); - } catch (CatalogException e) { + auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.JOB, job.getId(), job.getUuid(), + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + } catch (Exception e) { String errorMsg = "Cannot delete job " + job.getId() + ": " + e.getMessage(); Event event = new Event(Event.Type.ERROR, job.getId(), e.getMessage()); @@ -914,11 +996,11 @@ public OpenCGAResult delete(String studyId, Query query, ObjectMap params, boole result.setNumErrors(result.getNumErrors() + 1); logger.error(errorMsg, e); - auditManager.auditDelete(operationUuid, userId, Enums.Resource.JOB, job.getId(), job.getUuid(), study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.JOB, job.getId(), job.getUuid(), + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); } } - auditManager.finishAuditBatch(operationUuid); + auditManager.finishAuditBatch(organizationId, operationUuid); return endResult(result, ignoreException); } @@ -941,8 +1023,11 @@ public OpenCGAResult log(String studyId, String jobId, long offset, throws CatalogException { long startTime = System.currentTimeMillis(); - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyId, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyId, userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) @@ -964,7 +1049,7 @@ public OpenCGAResult log(String studyId, String jobId, long offset, Arrays.asList(JobDBAdaptor.QueryParams.ID.key(), JobDBAdaptor.QueryParams.UUID.key(), JobDBAdaptor.QueryParams.INTERNAL_STATUS.key(), JobDBAdaptor.QueryParams.STDOUT.key(), JobDBAdaptor.QueryParams.OUT_DIR.key())); - Job job = internalGet(study.getUid(), jobId, options, userId).first(); + Job job = internalGet(organizationId, study.getUid(), jobId, options, userId).first(); Path logFile; if ("stderr".equalsIgnoreCase(type)) { @@ -972,23 +1057,17 @@ public OpenCGAResult log(String studyId, String jobId, long offset, logFile = Paths.get(job.getStderr().getUri()); } else { // The log file hasn't yet been registered - if (!Arrays.asList(Enums.ExecutionStatus.PENDING, Enums.ExecutionStatus.QUEUED, Enums.ExecutionStatus.ABORTED) - .contains(job.getInternal().getStatus().getId()) && job.getOutDir() != null) { - logFile = Paths.get(job.getOutDir().getUri()).resolve(job.getId() + ".err"); - } else { - throw CatalogAuthorizationException.deny(userId, "see stderr log file of job '" + jobId + "'"); - } + logFile = Paths.get(job.getOutDir().getUri()).resolve(job.getId() + ".err"); } } else { if (job.getStdout() != null && job.getStdout().getUri() != null) { logFile = Paths.get(job.getStdout().getUri()); } else { - // The log file hasn't yet been registered - if (!Arrays.asList(Enums.ExecutionStatus.PENDING, Enums.ExecutionStatus.QUEUED, Enums.ExecutionStatus.ABORTED) - .contains(job.getInternal().getStatus().getId()) && job.getOutDir() != null) { + if (job.getOutDir() != null && job.getOutDir().getUri() != null) { + // The log file hasn't yet been registered logFile = Paths.get(job.getOutDir().getUri()).resolve(job.getId() + ".log"); } else { - throw CatalogAuthorizationException.deny(userId, "see stdout log file of job '" + jobId + "'"); + throw CatalogAuthorizationException.deny(userId, "view log file"); } } } @@ -999,6 +1078,12 @@ public OpenCGAResult log(String studyId, String jobId, long offset, } catch (IOException e) { throw CatalogIOException.ioManagerException(logFile.toUri(), e); } + + if (!ioManager.exists(logFile.toUri())) { + String status = job.getInternal().getStatus().getId(); + throw new CatalogException("Job '" + jobId + "' in status '" + status + "'. Log file '" + type + "' not found"); + } + FileContent fileContent; if (tail) { fileContent = ioManager.tail(logFile, lines); @@ -1006,14 +1091,14 @@ public OpenCGAResult log(String studyId, String jobId, long offset, fileContent = ioManager.head(logFile, offset, lines); } - auditManager.audit(userId, Enums.Action.VIEW_LOG, Enums.Resource.JOB, job.getId(), job.getUuid(), study.getId(), + auditManager.audit(organizationId, userId, Enums.Action.VIEW_LOG, Enums.Resource.JOB, job.getId(), job.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return new OpenCGAResult<>((int) (System.currentTimeMillis() - startTime), Collections.emptyList(), 1, Collections.singletonList(fileContent), 1); - } catch (CatalogException e) { - auditManager.audit(userId, Enums.Action.VIEW_LOG, Enums.Resource.JOB, jobId, "", study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + } catch (Exception e) { + auditManager.audit(organizationId, userId, Enums.Action.VIEW_LOG, Enums.Resource.JOB, jobId, "", study.getId(), study.getUuid(), + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); throw e; } } @@ -1027,8 +1112,11 @@ public OpenCGAResult update(String studyStr, Query query, JobUpdateParams u QueryOptions options, String token) throws CatalogException { Query finalQuery = new Query(ParamUtils.defaultObject(query, Query::new)); - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, userId, organizationId); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -1049,13 +1137,13 @@ public OpenCGAResult update(String studyStr, Query query, JobUpdateParams u DBIterator iterator; try { - fixQueryObject(study, finalQuery, userId); + fixQueryObject(organizationId, study, finalQuery, userId); finalQuery.append(JobDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - iterator = jobDBAdaptor.iterator(study.getUid(), finalQuery, INCLUDE_JOB_IDS, userId); - } catch (CatalogException e) { - auditManager.auditUpdate(operationId, userId, Enums.Resource.JOB, "", "", study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + iterator = getJobDBAdaptor(organizationId).iterator(study.getUid(), finalQuery, INCLUDE_JOB_IDS, userId); + } catch (Exception e) { + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.JOB, "", "", study.getId(), study.getUuid(), + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); throw e; } @@ -1064,47 +1152,33 @@ public OpenCGAResult update(String studyStr, Query query, JobUpdateParams u while (iterator.hasNext()) { Job job = iterator.next(); try { - OpenCGAResult updateResult = update(study, job, updateParams, options, userId); + OpenCGAResult updateResult = update(organizationId, study, job, updateParams, options, userId); result.append(updateResult); - auditManager.auditUpdate(operationId, userId, Enums.Resource.JOB, job.getId(), job.getUuid(), study.getId(), + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.JOB, job.getId(), job.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); - } catch (CatalogException e) { + } catch (Exception e) { Event event = new Event(Event.Type.ERROR, job.getId(), e.getMessage()); result.getEvents().add(event); result.setNumErrors(result.getNumErrors() + 1); logger.error("Could not update job {}: {}", job.getId(), e.getMessage(), e); - auditManager.auditUpdate(operationId, userId, Enums.Resource.JOB, job.getId(), job.getUuid(), study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.JOB, job.getId(), job.getUuid(), study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); } } - auditManager.finishAuditBatch(operationId); + auditManager.finishAuditBatch(organizationId, operationId); return endResult(result, ignoreException); } - /** - * Update Job from catalog. - * - * @param studyStr Study id in string format. Could be one of [id|user@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy]. - * @param jobIds List of Job ids. Could be either the id or uuid. - * @param updateParams Data model filled only with the parameters to be updated. - * @param options QueryOptions object. - * @param token Session id of the user logged in. - * @return A OpenCGAResult with the objects updated. - * @throws CatalogException if there is any internal error, the user does not have proper permissions or a parameter passed does not - * exist or is not allowed to be updated. - */ - public OpenCGAResult update(String studyStr, List jobIds, JobUpdateParams updateParams, QueryOptions options, - String token) throws CatalogException { - return update(studyStr, jobIds, updateParams, false, options, token); - } - public OpenCGAResult update(String studyStr, List jobIds, JobUpdateParams updateParams, boolean ignoreException, QueryOptions options, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, userId, organizationId); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -1130,7 +1204,7 @@ public OpenCGAResult update(String studyStr, List jobIds, JobUpdate String jobUuid = ""; try { - OpenCGAResult internalResult = internalGet(study.getUid(), id, INCLUDE_JOB_IDS, userId); + OpenCGAResult internalResult = internalGet(organizationId, study.getUid(), id, INCLUDE_JOB_IDS, userId); if (internalResult.getNumResults() == 0) { throw new CatalogException("Job '" + id + "' not found"); } @@ -1140,30 +1214,51 @@ public OpenCGAResult update(String studyStr, List jobIds, JobUpdate jobId = job.getId(); jobUuid = job.getUuid(); - OpenCGAResult updateResult = update(study, job, updateParams, options, userId); + OpenCGAResult updateResult = update(organizationId, study, job, updateParams, options, userId); result.append(updateResult); - auditManager.auditUpdate(operationId, userId, Enums.Resource.JOB, job.getId(), job.getUuid(), study.getId(), + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.JOB, job.getId(), job.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); - } catch (CatalogException e) { + } catch (Exception e) { Event event = new Event(Event.Type.ERROR, jobId, e.getMessage()); result.getEvents().add(event); result.setNumErrors(result.getNumErrors() + 1); logger.error("Could not update job {}: {}", jobId, e.getMessage(), e); - auditManager.auditUpdate(operationId, userId, Enums.Resource.JOB, jobId, jobUuid, study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.JOB, jobId, jobUuid, study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); } } - auditManager.finishAuditBatch(operationId); + auditManager.finishAuditBatch(organizationId, operationId); return endResult(result, ignoreException); } + /** + * Update Job from catalog. + * + * @param studyStr Study id in string format. Could be one of + * [id|organization@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy] + * @param jobIds List of Job ids. Could be either the id or uuid. + * @param updateParams Data model filled only with the parameters to be updated. + * @param options QueryOptions object. + * @param token Session id of the user logged in. + * @return A OpenCGAResult with the objects updated. + * @throws CatalogException if there is any internal error, the user does not have proper permissions or a parameter passed does not + * exist or is not allowed to be updated. + */ + public OpenCGAResult update(String studyStr, List jobIds, JobUpdateParams updateParams, QueryOptions options, String token) + throws CatalogException { + return update(studyStr, jobIds, updateParams, false, options, token); + } + public OpenCGAResult update(String studyStr, String jobId, JobUpdateParams updateParams, QueryOptions options, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, userId, organizationId); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -1184,7 +1279,7 @@ public OpenCGAResult update(String studyStr, String jobId, JobUpdateParams OpenCGAResult result = OpenCGAResult.empty(); String jobUuid = ""; try { - OpenCGAResult internalResult = internalGet(study.getUid(), jobId, INCLUDE_JOB_IDS, userId); + OpenCGAResult internalResult = internalGet(organizationId, study.getUid(), jobId, INCLUDE_JOB_IDS, userId); if (internalResult.getNumResults() == 0) { throw new CatalogException("Job '" + jobId + "' not found"); } @@ -1194,27 +1289,27 @@ public OpenCGAResult update(String studyStr, String jobId, JobUpdateParams jobId = job.getId(); jobUuid = job.getUuid(); - OpenCGAResult updateResult = update(study, job, updateParams, options, userId); + OpenCGAResult updateResult = update(organizationId, study, job, updateParams, options, userId); result.append(updateResult); - auditManager.auditUpdate(operationId, userId, Enums.Resource.JOB, job.getId(), job.getUuid(), study.getId(), + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.JOB, job.getId(), job.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); - } catch (CatalogException e) { + } catch (Exception e) { Event event = new Event(Event.Type.ERROR, jobId, e.getMessage()); result.getEvents().add(event); result.setNumErrors(result.getNumErrors() + 1); logger.error("Could not update job {}: {}", jobId, e.getMessage(), e); - auditManager.auditUpdate(operationId, userId, Enums.Resource.JOB, jobId, jobUuid, study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.JOB, jobId, jobUuid, study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); throw e; } return result; } - private OpenCGAResult update(Study study, Job job, JobUpdateParams updateParams, QueryOptions options, String userId) - throws CatalogException { + private OpenCGAResult update(String organizationId, Study study, Job job, JobUpdateParams updateParams, QueryOptions options, + String userId) throws CatalogException { if (updateParams == null) { throw new CatalogException("Missing parameters to update"); } @@ -1230,7 +1325,7 @@ private OpenCGAResult update(Study study, Job job, JobUpdateParams updatePa } options = ParamUtils.defaultObject(options, QueryOptions::new); - authorizationManager.checkJobPermission(study.getUid(), job.getUid(), userId, JobPermissions.WRITE); + authorizationManager.checkJobPermission(organizationId, study.getUid(), job.getUid(), userId, JobPermissions.WRITE); // if (StringUtils.isNotEmpty(updateParams.getId())) { // ParamUtils.checkAlias(updateParams.getId(), JobDBAdaptor.QueryParams.ID.key()); @@ -1272,23 +1367,23 @@ private OpenCGAResult update(Study study, Job job, JobUpdateParams updatePa // updateParams.setErrorLog(getFile(study.getUid(), updateParams.getErrorLog().getPath(), userId)); // } - OpenCGAResult update = jobDBAdaptor.update(job.getUid(), updateMap, options); + OpenCGAResult update = getJobDBAdaptor(organizationId).update(job.getUid(), updateMap, options); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { // Fetch updated job - OpenCGAResult result = jobDBAdaptor.get(study.getUid(), new Query(JobDBAdaptor.QueryParams.UID.key(), job.getUid()), - options, userId); + OpenCGAResult result = getJobDBAdaptor(organizationId).get(study.getUid(), new Query(JobDBAdaptor.QueryParams.UID.key(), + job.getUid()), options, userId); update.setResults(result.getResults()); } return update; } - private File getFile(long studyUid, String path, String userId) throws CatalogException { + private File getFile(String organizationId, long studyUid, String path, String userId) throws CatalogException { if (StringUtils.isEmpty(path)) { throw new CatalogException("Missing file path"); } - OpenCGAResult fileResult = catalogManager.getFileManager().internalGet(studyUid, path, FileManager.INCLUDE_FILE_URI_PATH, - userId); + OpenCGAResult fileResult = catalogManager.getFileManager().internalGet(organizationId, studyUid, path, + FileManager.INCLUDE_FILE_URI_PATH, userId); if (fileResult.getNumResults() == 0) { throw new CatalogException("File/Folder '" + path + "' not found"); } @@ -1302,8 +1397,11 @@ public OpenCGAResult update(String studyId, Query query, ObjectMap paramete public OpenCGAResult update(String studyId, Query query, ObjectMap parameters, boolean ignoreException, QueryOptions options, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyId, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyId, userId, organizationId); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -1321,13 +1419,13 @@ public OpenCGAResult update(String studyId, Query query, ObjectMap paramete DBIterator iterator; try { - fixQueryObject(study, finalQuery, token); + fixQueryObject(organizationId, study, finalQuery, token); finalQuery.append(FamilyDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - iterator = jobDBAdaptor.iterator(study.getUid(), finalQuery, INCLUDE_JOB_IDS, userId); - } catch (CatalogException e) { - auditManager.auditUpdate(operationId, userId, Enums.Resource.JOB, "", "", study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + iterator = getJobDBAdaptor(organizationId).iterator(study.getUid(), finalQuery, INCLUDE_JOB_IDS, userId); + } catch (Exception e) { + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.JOB, "", "", study.getId(), study.getUuid(), + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); throw e; } @@ -1337,21 +1435,21 @@ public OpenCGAResult update(String studyId, Query query, ObjectMap paramete try { options = ParamUtils.defaultObject(options, QueryOptions::new); - authorizationManager.checkJobPermission(study.getUid(), job.getUid(), userId, JobPermissions.WRITE); + authorizationManager.checkJobPermission(organizationId, study.getUid(), job.getUid(), userId, JobPermissions.WRITE); - OpenCGAResult updateResult = jobDBAdaptor.update(job.getUid(), parameters, options); + OpenCGAResult updateResult = getJobDBAdaptor(organizationId).update(job.getUid(), parameters, options); result.append(updateResult); - auditManager.auditUpdate(operationId, userId, Enums.Resource.JOB, job.getId(), job.getUuid(), study.getId(), + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.JOB, job.getId(), job.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); - } catch (CatalogException e) { + } catch (Exception e) { Event event = new Event(Event.Type.ERROR, job.getId(), e.getMessage()); result.getEvents().add(event); result.setNumErrors(result.getNumErrors() + 1); logger.error("Cannot update job {}: {}", job.getId(), e.getMessage()); - auditManager.auditUpdate(operationId, userId, Enums.Resource.JOB, job.getId(), job.getUuid(), study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.JOB, job.getId(), job.getUuid(), study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); } } @@ -1360,8 +1458,11 @@ public OpenCGAResult update(String studyId, Query query, ObjectMap paramete public OpenCGAResult update(String studyId, String jobId, ObjectMap parameters, QueryOptions options, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyId, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyId, userId, organizationId); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -1377,7 +1478,7 @@ public OpenCGAResult update(String studyId, String jobId, ObjectMap paramet OpenCGAResult result = OpenCGAResult.empty(); String jobUuid = ""; try { - OpenCGAResult internalResult = internalGet(study.getUid(), jobId, QueryOptions.empty(), userId); + OpenCGAResult internalResult = internalGet(organizationId, study.getUid(), jobId, QueryOptions.empty(), userId); if (internalResult.getNumResults() == 0) { throw new CatalogException("Job '" + jobId + "' not found"); } @@ -1389,21 +1490,21 @@ public OpenCGAResult update(String studyId, String jobId, ObjectMap paramet options = ParamUtils.defaultObject(options, QueryOptions::new); - authorizationManager.checkJobPermission(study.getUid(), job.getUid(), userId, JobPermissions.WRITE); + authorizationManager.checkJobPermission(organizationId, study.getUid(), job.getUid(), userId, JobPermissions.WRITE); - OpenCGAResult updateResult = jobDBAdaptor.update(job.getUid(), parameters, options); + OpenCGAResult updateResult = getJobDBAdaptor(organizationId).update(job.getUid(), parameters, options); result.append(updateResult); - auditManager.auditUpdate(operationId, userId, Enums.Resource.JOB, job.getId(), job.getUuid(), study.getId(), + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.JOB, job.getId(), job.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); - } catch (CatalogException e) { + } catch (Exception e) { Event event = new Event(Event.Type.ERROR, jobId, e.getMessage()); result.getEvents().add(event); result.setNumErrors(result.getNumErrors() + 1); logger.error("Cannot update job {}: {}", jobId, e.getMessage()); - auditManager.auditUpdate(operationId, userId, Enums.Resource.JOB, jobId, jobUuid, study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.JOB, jobId, jobUuid, study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); throw e; } @@ -1417,8 +1518,11 @@ public OpenCGAResult update(String studyId, List jobIds, ObjectMap public OpenCGAResult update(String studyId, List jobIds, ObjectMap parameters, boolean ignoreException, QueryOptions options, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyId, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyId, userId, organizationId); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -1438,7 +1542,7 @@ public OpenCGAResult update(String studyId, List jobIds, ObjectMap String jobUuid = ""; try { - OpenCGAResult internalResult = internalGet(study.getUid(), id, QueryOptions.empty(), userId); + OpenCGAResult internalResult = internalGet(organizationId, study.getUid(), id, QueryOptions.empty(), userId); if (internalResult.getNumResults() == 0) { throw new CatalogException("Job '" + id + "' not found"); } @@ -1450,55 +1554,66 @@ public OpenCGAResult update(String studyId, List jobIds, ObjectMap options = ParamUtils.defaultObject(options, QueryOptions::new); - authorizationManager.checkJobPermission(study.getUid(), job.getUid(), userId, JobPermissions.WRITE); + authorizationManager.checkJobPermission(organizationId, study.getUid(), job.getUid(), userId, JobPermissions.WRITE); - OpenCGAResult updateResult = jobDBAdaptor.update(job.getUid(), parameters, options); + OpenCGAResult updateResult = getJobDBAdaptor(organizationId).update(job.getUid(), parameters, options); result.append(updateResult); - auditManager.auditUpdate(operationId, userId, Enums.Resource.JOB, job.getId(), job.getUuid(), study.getId(), + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.JOB, job.getId(), job.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); - } catch (CatalogException e) { + } catch (Exception e) { Event event = new Event(Event.Type.ERROR, jobId, e.getMessage()); result.getEvents().add(event); result.setNumErrors(result.getNumErrors() + 1); logger.error("Cannot update job {}: {}", jobId, e.getMessage()); - auditManager.auditUpdate(operationId, userId, Enums.Resource.JOB, jobId, jobUuid, study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.JOB, jobId, jobUuid, study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); } } return endResult(result, ignoreException); } - public OpenCGAResult top(Query baseQuery, int limit, String token) throws CatalogException { - String userId = userManager.getUserId(token); - List studies = studyManager.search(new Query(StudyDBAdaptor.QueryParams.OWNER.key(), userId), - new QueryOptions(QueryOptions.INCLUDE, StudyDBAdaptor.QueryParams.UUID.key()), token).getResults() + public OpenCGAResult top(String organizationId, Query baseQuery, int limit, String token) throws CatalogException { + JwtPayload jwtPayload = userManager.validateToken(token); + String userId = jwtPayload.getUserId(organizationId); + authorizationManager.checkIsAtLeastOrganizationOwnerOrAdmin(organizationId, userId); + List studies = studyManager.searchInOrganization(organizationId, new Query(), StudyManager.INCLUDE_STUDY_IDS, token) + .getResults() .stream() - .map(Study::getUuid) + .map(Study::getFqn) .collect(Collectors.toList()); return top(studies, baseQuery, limit, token); } - public OpenCGAResult top(String studyStr, Query baseQuery, int limit, String token) throws CatalogException { + public OpenCGAResult top(String organizationId, String studyStr, Query baseQuery, int limit, String token) + throws CatalogException { if (StringUtils.isEmpty(studyStr)) { - return top(baseQuery, limit, token); + return top(organizationId, baseQuery, limit, token); } else { return top(Collections.singletonList(studyStr), baseQuery, limit, token); } } public OpenCGAResult top(List studiesStr, Query baseQuery, int limit, String token) throws CatalogException { - String userId = userManager.getUserId(token); - fixQueryObject(null, baseQuery, userId); + JwtPayload payload = userManager.validateToken(token); List studies = new ArrayList<>(studiesStr.size()); + String organizationId = null; for (String studyStr : studiesStr) { - Study study = studyManager.resolveId(studyStr, userId); - authorizationManager.checkCanViewStudy(study.getUid(), userId); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, payload); + if (organizationId == null) { + organizationId = studyFqn.getOrganizationId(); + } else if (!organizationId.equals(studyFqn.getOrganizationId())) { + throw new CatalogException("Organization id should be the same for all the studies."); + } + Study study = studyManager.resolveId(studyFqn, QueryOptions.empty(), payload); studies.add(study); } + String userId = payload.getUserId(organizationId); + fixQueryObject(organizationId, null, baseQuery, userId); + StopWatch stopWatch = StopWatch.createStarted(); QueryOptions queryOptions = new QueryOptions() .append(QueryOptions.INCLUDE, Arrays.asList(JobDBAdaptor.QueryParams.ID.key(), JobDBAdaptor.QueryParams.TOOL.key(), @@ -1515,7 +1630,7 @@ public OpenCGAResult top(List studiesStr, Query baseQuery, int l if (jobsLimit == 0) { break; } - List results = jobDBAdaptor.get( + List results = getJobDBAdaptor(organizationId).get( study.getUid(), new Query(baseQuery) .append(JobDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()) @@ -1540,7 +1655,7 @@ public OpenCGAResult top(List studiesStr, Query baseQuery, int l if (jobsLimit == 0) { break; } - List results = jobDBAdaptor.get( + List results = getJobDBAdaptor(organizationId).get( study.getUid(), new Query(baseQuery) .append(JobDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()) @@ -1562,7 +1677,7 @@ public OpenCGAResult top(List studiesStr, Query baseQuery, int l if (jobsLimit == 0) { break; } - List results = jobDBAdaptor.get( + List results = getJobDBAdaptor(organizationId).get( study.getUid(), new Query(baseQuery) .append(JobDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()) @@ -1584,7 +1699,7 @@ public OpenCGAResult top(List studiesStr, Query baseQuery, int l if (jobsLimit == 0) { break; } - List results = jobDBAdaptor.get( + List results = getJobDBAdaptor(organizationId).get( study.getUid(), new Query(baseQuery) .append(JobDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()) @@ -1614,7 +1729,7 @@ public OpenCGAResult top(List studiesStr, Query baseQuery, int l JobTopStats stats = new JobTopStats(); for (Study study : studies) { - OpenCGAResult result = jobDBAdaptor.groupBy(new Query(baseQuery) + OpenCGAResult result = getJobDBAdaptor(organizationId).groupBy(new Query(baseQuery) .append(JobDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()), Collections.singletonList(JobDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key()), new QueryOptions(QueryOptions.COUNT, true), @@ -1655,11 +1770,14 @@ public OpenCGAResult rank(String studyId, Query query, String field, int numResu throws CatalogException { query = ParamUtils.defaultObject(query, Query::new); ParamUtils.checkObj(field, "field"); - ParamUtils.checkObj(token, "sessionId"); + ParamUtils.checkObj(token, "token"); - String userId = userManager.getUserId(token); - Study study = catalogManager.getStudyManager().resolveId(studyId, userId); - authorizationManager.checkStudyPermission(study.getUid(), userId, StudyPermissions.Permissions.VIEW_JOBS); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyId, userId, organizationId); + authorizationManager.checkStudyPermission(organizationId, study.getUid(), userId, StudyPermissions.Permissions.VIEW_JOBS); // TODO: In next release, we will have to check the count parameter from the queryOptions object. boolean count = true; @@ -1667,7 +1785,7 @@ public OpenCGAResult rank(String studyId, Query query, String field, int numResu OpenCGAResult queryResult = null; if (count) { // We do not need to check for permissions when we show the count of files - queryResult = jobDBAdaptor.rank(query, field, numResults, asc); + queryResult = getJobDBAdaptor(organizationId).rank(query, field, numResults, asc); } return ParamUtils.defaultObject(queryResult, OpenCGAResult::new); @@ -1683,13 +1801,16 @@ public OpenCGAResult groupBy(@Nullable String studyId, Query query, List throw new CatalogException("Empty fields parameter."); } - String userId = userManager.getUserId(token); - Study study = catalogManager.getStudyManager().resolveId(studyId, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyId, userId, organizationId); // Add study id to the query query.put(SampleDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - OpenCGAResult queryResult = jobDBAdaptor.groupBy(query, fields, options, userId); + OpenCGAResult queryResult = getJobDBAdaptor(organizationId).groupBy(query, fields, options, userId); return ParamUtils.defaultObject(queryResult, OpenCGAResult::new); } @@ -1697,14 +1818,18 @@ public OpenCGAResult groupBy(@Nullable String studyId, Query query, List // ************************** ACLs ******************************** // public OpenCGAResult> getAcls(String studyId, List jobList, String member, boolean ignoreException, String token) throws CatalogException { - return getAcls(studyId, jobList, StringUtils.isNotEmpty(member) ? Collections.singletonList(member) : Collections.emptyList(), + return getAcls(studyId, jobList, + StringUtils.isNotEmpty(member) ? Collections.singletonList(member) : Collections.emptyList(), ignoreException, token); } public OpenCGAResult> getAcls(String studyId, List jobList, List members, boolean ignoreException, String token) throws CatalogException { - String user = userManager.getUserId(token); - Study study = studyManager.resolveId(studyId, user); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyId, userId, organizationId); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); ObjectMap auditParams = new ObjectMap() @@ -1718,7 +1843,8 @@ public OpenCGAResult> getAcls(String studyId, List< Map missingMap = new HashMap<>(); try { auditManager.initAuditBatch(operationId); - InternalGetDataResult queryResult = internalGet(study.getUid(), jobList, INCLUDE_JOB_IDS, user, ignoreException); + InternalGetDataResult queryResult = internalGet(organizationId, study.getUid(), jobList, INCLUDE_JOB_IDS, userId, + ignoreException); if (queryResult.getMissing() != null) { missingMap = queryResult.getMissing().stream() @@ -1727,9 +1853,11 @@ public OpenCGAResult> getAcls(String studyId, List< List jobUids = queryResult.getResults().stream().map(Job::getUid).collect(Collectors.toList()); if (CollectionUtils.isNotEmpty(members)) { - jobAcls = authorizationManager.getAcl(user, study.getUid(), jobUids, members, Enums.Resource.JOB, JobPermissions.class); + jobAcls = authorizationManager.getAcl(organizationId, study.getUid(), jobUids, members, Enums.Resource.JOB, + JobPermissions.class, userId); } else { - jobAcls = authorizationManager.getAcl(user, study.getUid(), jobUids, Enums.Resource.JOB, JobPermissions.class); + jobAcls = authorizationManager.getAcl(organizationId, study.getUid(), jobUids, Enums.Resource.JOB, JobPermissions.class, + userId); } // Include non-existing jobs to the result list @@ -1740,15 +1868,15 @@ public OpenCGAResult> getAcls(String studyId, List< if (!missingMap.containsKey(jobId)) { Job job = queryResult.getResults().get(counter); resultList.add(jobAcls.getResults().get(counter)); - auditManager.audit(operationId, user, Enums.Action.FETCH_ACLS, Enums.Resource.JOB, job.getId(), job.getUuid(), - study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS), - new ObjectMap()); + auditManager.audit(organizationId, operationId, userId, Enums.Action.FETCH_ACLS, Enums.Resource.JOB, job.getId(), + job.getUuid(), study.getId(), study.getUuid(), auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS), new ObjectMap()); counter++; } else { resultList.add(new AclEntryList<>()); eventList.add(new Event(Event.Type.ERROR, jobId, missingMap.get(jobId).getErrorMsg())); - auditManager.audit(operationId, user, Enums.Action.FETCH_ACLS, Enums.Resource.JOB, jobId, "", study.getId(), - study.getUuid(), auditParams, + auditManager.audit(organizationId, operationId, userId, Enums.Action.FETCH_ACLS, Enums.Resource.JOB, jobId, "", + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(0, "", missingMap.get(jobId).getErrorMsg())), new ObjectMap()); } @@ -1758,10 +1886,10 @@ public OpenCGAResult> getAcls(String studyId, List< } jobAcls.setResults(resultList); jobAcls.setEvents(eventList); - } catch (CatalogException e) { + } catch (Exception e) { for (String jobId : jobList) { - auditManager.audit(operationId, user, Enums.Action.FETCH_ACLS, Enums.Resource.JOB, jobId, "", study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError()), + auditManager.audit(organizationId, operationId, userId, Enums.Action.FETCH_ACLS, Enums.Resource.JOB, jobId, "", + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e), new ObjectMap()); } if (!ignoreException) { @@ -1773,7 +1901,7 @@ public OpenCGAResult> getAcls(String studyId, List< } } } finally { - auditManager.finishAuditBatch(operationId); + auditManager.finishAuditBatch(organizationId, operationId); } return jobAcls; @@ -1782,8 +1910,11 @@ public OpenCGAResult> getAcls(String studyId, List< public OpenCGAResult> updateAcl(String studyId, List jobStrList, String memberList, AclParams aclParams, ParamUtils.AclAction action, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyId, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyId, userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) @@ -1811,9 +1942,9 @@ public OpenCGAResult> updateAcl(String studyId, Lis checkPermissions(permissions, JobPermissions::valueOf); } - List jobList = internalGet(study.getUid(), jobStrList, INCLUDE_JOB_IDS, userId, false).getResults(); + List jobList = internalGet(organizationId, study.getUid(), jobStrList, INCLUDE_JOB_IDS, userId, false).getResults(); - authorizationManager.checkCanAssignOrSeePermissions(study.getUid(), userId); + authorizationManager.checkCanAssignOrSeePermissions(organizationId, study.getUid(), userId); // Validate that the members are actually valid members List members; @@ -1823,7 +1954,7 @@ public OpenCGAResult> updateAcl(String studyId, Lis members = Collections.emptyList(); } authorizationManager.checkNotAssigningPermissionsToAdminsGroup(members); - checkMembers(study.getUid(), members); + checkMembers(organizationId, study.getUid(), members); List jobUids = jobList.stream().map(Job::getUid).collect(Collectors.toList()); AuthorizationManager.CatalogAclParams catalogAclParams = new AuthorizationManager.CatalogAclParams(jobUids, permissions, @@ -1831,82 +1962,43 @@ public OpenCGAResult> updateAcl(String studyId, Lis switch (action) { case SET: - authorizationManager.setAcls(study.getUid(), members, catalogAclParams); + authorizationManager.setAcls(organizationId, study.getUid(), members, catalogAclParams); break; case ADD: - authorizationManager.addAcls(study.getUid(), members, catalogAclParams); + authorizationManager.addAcls(organizationId, study.getUid(), members, catalogAclParams); break; case REMOVE: - authorizationManager.removeAcls(members, catalogAclParams); + authorizationManager.removeAcls(organizationId, members, catalogAclParams); break; case RESET: catalogAclParams.setPermissions(null); - authorizationManager.removeAcls(members, catalogAclParams); + authorizationManager.removeAcls(organizationId, members, catalogAclParams); break; default: throw new CatalogException("Unexpected error occurred. No valid action found."); } - OpenCGAResult> queryResultList = authorizationManager.getAcls(study.getUid(), jobUids, - members, Enums.Resource.JOB, JobPermissions.class); + OpenCGAResult> queryResultList = authorizationManager.getAcls(organizationId, study.getUid(), + jobUids, members, Enums.Resource.JOB, JobPermissions.class); for (int i = 0; i < queryResultList.getResults().size(); i++) { queryResultList.getResults().get(i).setId(jobList.get(i).getId()); } for (Job job : jobList) { - auditManager.audit(operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.JOB, job.getId(), + auditManager.audit(organizationId, operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.JOB, job.getId(), job.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS), new ObjectMap()); } return queryResultList; - } catch (CatalogException e) { + } catch (Exception e) { if (jobStrList != null) { for (String jobId : jobStrList) { - auditManager.audit(operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.JOB, jobId, "", - study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, - e.getError()), new ObjectMap()); + auditManager.audit(organizationId, operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.JOB, jobId, "", + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e), + new ObjectMap()); } } throw e; } finally { - auditManager.finishAuditBatch(operationId); - } - } - - public DataResult facet(String studyId, Query query, QueryOptions options, boolean defaultStats, String token) - throws CatalogException, IOException { - String userId = userManager.getUserId(token); - // We need to add variableSets and groups to avoid additional queries as it will be used in the catalogSolrManager - Study study = catalogManager.getStudyManager().resolveId(studyId, userId, new QueryOptions(QueryOptions.INCLUDE, - Arrays.asList(StudyDBAdaptor.QueryParams.VARIABLE_SET.key(), StudyDBAdaptor.QueryParams.GROUPS.key()))); - - ParamUtils.defaultObject(query, Query::new); - ParamUtils.defaultObject(options, QueryOptions::new); - - ObjectMap auditParams = new ObjectMap() - .append("studyId", studyId) - .append("query", new Query(query)) - .append("options", options) - .append("defaultStats", defaultStats) - .append("token", token); - - try { - if (defaultStats || StringUtils.isEmpty(options.getString(QueryOptions.FACET))) { - String facet = options.getString(QueryOptions.FACET); - options.put(QueryOptions.FACET, StringUtils.isNotEmpty(facet) ? defaultFacet + ";" + facet : defaultFacet); - } - AnnotationUtils.fixQueryAnnotationSearch(study, userId, query, authorizationManager); - - try (CatalogSolrManager catalogSolrManager = new CatalogSolrManager(catalogManager)) { - DataResult result = catalogSolrManager.facetedQuery(study, CatalogSolrManager.JOB_SOLR_COLLECTION, query, - options, userId); - - auditManager.auditFacet(userId, Enums.Resource.JOB, study.getId(), study.getUuid(), auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); - return result; - } - } catch (CatalogException e) { - auditManager.auditFacet(userId, Enums.Resource.JOB, study.getId(), study.getUuid(), auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(0, "", e.getMessage()))); - throw e; + auditManager.finishAuditBatch(organizationId, operationId); } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/NoteManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/NoteManager.java new file mode 100644 index 00000000000..f32c8422dec --- /dev/null +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/NoteManager.java @@ -0,0 +1,407 @@ +package org.opencb.opencga.catalog.managers; + +import com.fasterxml.jackson.core.JsonProcessingException; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.commons.datastore.core.result.Error; +import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager; +import org.opencb.opencga.catalog.db.DBAdaptorFactory; +import org.opencb.opencga.catalog.db.api.NoteDBAdaptor; +import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.exceptions.CatalogParameterException; +import org.opencb.opencga.catalog.utils.CatalogFqn; +import org.opencb.opencga.catalog.utils.ParamUtils; +import org.opencb.opencga.catalog.utils.UuidUtils; +import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.config.Configuration; +import org.opencb.opencga.core.models.JwtPayload; +import org.opencb.opencga.core.models.audit.AuditRecord; +import org.opencb.opencga.core.models.common.Enums; +import org.opencb.opencga.core.models.notes.Note; +import org.opencb.opencga.core.models.notes.NoteCreateParams; +import org.opencb.opencga.core.models.notes.NoteUpdateParams; +import org.opencb.opencga.core.models.study.Study; +import org.opencb.opencga.core.response.OpenCGAResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; + +public class NoteManager extends AbstractManager { + + private final Logger logger; + + NoteManager(AuthorizationManager authorizationManager, AuditManager auditManager, CatalogManager catalogManager, + DBAdaptorFactory catalogDBAdaptorFactory, Configuration configuration) { + super(authorizationManager, auditManager, catalogManager, catalogDBAdaptorFactory, configuration); + this.logger = LoggerFactory.getLogger(NoteManager.class); + } + + private OpenCGAResult internalGet(String organizationId, long studyUid, String noteId) throws CatalogException { + Query query = new Query(); + if (studyUid > 0) { + query.put(NoteDBAdaptor.QueryParams.STUDY_UID.key(), studyUid); + } else { + query.put(NoteDBAdaptor.QueryParams.STUDY_UID.key(), -1L); + } + if (UuidUtils.isOpenCgaUuid(noteId)) { + query.put(NoteDBAdaptor.QueryParams.UUID.key(), noteId); + } else { + query.put(NoteDBAdaptor.QueryParams.ID.key(), noteId); + } + OpenCGAResult result = getCatalogDBAdaptorFactory().getCatalogNoteDBAdaptor(organizationId).get(query, QueryOptions.empty()); + if (result.getNumResults() == 0) { + throw CatalogException.notFound("note", Collections.singletonList(noteId)); + } + return result; + } + + public OpenCGAResult searchOrganizationNote(Query query, QueryOptions options, String token) throws CatalogException { + ObjectMap auditParams = new ObjectMap() + .append("query", query) + .append("options", options) + .append("token", token); + + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + String organizationId = tokenPayload.getOrganization(); + try { + Query queryCopy = ParamUtils.defaultObject(query, Query::new); + QueryOptions optionsCopy = ParamUtils.defaultObject(options, QueryOptions::new); + + if (!authorizationManager.isAtLeastOrganizationOwnerOrAdmin(organizationId, tokenPayload.getUserId())) { + String visibility = queryCopy.getString(NoteDBAdaptor.QueryParams.VISIBILITY.key()); + String scope = queryCopy.getString(NoteDBAdaptor.QueryParams.SCOPE.key()); + if (StringUtils.isNotEmpty(visibility) && !Note.Visibility.PUBLIC.name().equals(visibility)) { + throw new CatalogAuthorizationException("User '" + tokenPayload.getUserId() + "' is only authorised to see " + + Note.Visibility.PUBLIC + " organization notes."); + } + if (StringUtils.isNotEmpty(scope) && !Note.Scope.ORGANIZATION.name().equals(scope)) { + throw new CatalogAuthorizationException("User '" + tokenPayload.getUserId() + "' is only authorised to see " + + "organization notes of scope '" + Note.Scope.ORGANIZATION + "' from this method."); + } + queryCopy.put(NoteDBAdaptor.QueryParams.SCOPE.key(), Note.Scope.ORGANIZATION); + queryCopy.put(NoteDBAdaptor.QueryParams.VISIBILITY.key(), Note.Visibility.PUBLIC); + } + + OpenCGAResult result = getCatalogDBAdaptorFactory().getCatalogNoteDBAdaptor(organizationId).get(queryCopy, optionsCopy); + auditManager.auditSearch(organizationId, tokenPayload.getUserId(), Enums.Resource.NOTE, "", "", auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + return result; + } catch (Exception e) { + auditManager.auditSearch(organizationId, tokenPayload.getUserId(), Enums.Resource.NOTE, "", "", auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(0, "Note search", e.getMessage()))); + throw e; + } + } + + public OpenCGAResult searchStudyNote(String studyStr, Query query, QueryOptions options, String token) throws CatalogException { + ObjectMap auditParams = new ObjectMap() + .append("study", studyStr) + .append("query", query) + .append("options", options) + .append("token", token); + + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + String organizationId = tokenPayload.getOrganization(); + String studyId = ""; + String studyUuid = ""; + try { + Query queryCopy = ParamUtils.defaultObject(query, Query::new); + QueryOptions optionsCopy = ParamUtils.defaultObject(options, QueryOptions::new); + + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + organizationId = studyFqn.getOrganizationId(); + Study study = catalogManager.getStudyManager().resolveId(studyFqn, null, tokenPayload); + authorizationManager.checkCanViewStudy(organizationId, study.getUid(), tokenPayload.getUserId()); + studyId = study.getId(); + studyUuid = study.getUuid(); + queryCopy.put(NoteDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); + + if (!authorizationManager.isAtLeastStudyAdministrator(organizationId, study.getUid(), tokenPayload.getUserId())) { + String visibility = queryCopy.getString(NoteDBAdaptor.QueryParams.VISIBILITY.key()); + if (StringUtils.isNotEmpty(visibility) && !Note.Visibility.PUBLIC.name().equals(visibility)) { + throw new CatalogAuthorizationException("User '" + tokenPayload.getUserId() + "' is only authorised to see " + + Note.Visibility.PUBLIC + " study notes."); + } + queryCopy.put(NoteDBAdaptor.QueryParams.VISIBILITY.key(), Note.Visibility.PUBLIC); + } + + OpenCGAResult result = getCatalogDBAdaptorFactory().getCatalogNoteDBAdaptor(organizationId).get(queryCopy, optionsCopy); + auditManager.auditSearch(organizationId, tokenPayload.getUserId(), Enums.Resource.NOTE, studyId, studyUuid, auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + return result; + } catch (Exception e) { + auditManager.auditSearch(organizationId, tokenPayload.getUserId(), Enums.Resource.NOTE, studyId, studyUuid, auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(0, "Note search", e.getMessage()))); + throw e; + } + } + + public OpenCGAResult createOrganizationNote(NoteCreateParams noteCreateParams, QueryOptions options, String token) + throws CatalogException { + ObjectMap auditParams = new ObjectMap() + .append("noteCreateParams", noteCreateParams) + .append("options", options) + .append("token", token); + + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + String organizationId = tokenPayload.getOrganization(); + try { + QueryOptions optionsCopy = ParamUtils.defaultObject(options, QueryOptions::new); + Note note = noteCreateParams.toNote(Note.Scope.ORGANIZATION, tokenPayload.getUserId()); + + authorizationManager.checkIsAtLeastOrganizationOwnerOrAdmin(organizationId, tokenPayload.getUserId()); + + OpenCGAResult insert = create(note, optionsCopy, tokenPayload); + auditManager.auditCreate(organizationId, tokenPayload.getUserId(), Enums.Resource.NOTE, note.getId(), note.getUuid(), "", "", + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + return insert; + } catch (Exception e) { + auditManager.auditCreate(organizationId, tokenPayload.getUserId(), Enums.Resource.NOTE, noteCreateParams.getId(), "", "", "", + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(0, "Note create", e.getMessage()))); + throw e; + } + } + + public OpenCGAResult createStudyNote(String studyStr, NoteCreateParams noteCreateParams, QueryOptions options, String token) + throws CatalogException { + ObjectMap auditParams = new ObjectMap() + .append("study", studyStr) + .append("noteCreateParams", noteCreateParams) + .append("options", options) + .append("token", token); + + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + String organizationId = tokenPayload.getOrganization(); + String studyId = studyStr; + String studyUuid = ""; + try { + ParamUtils.checkParameter(studyStr, "study"); + QueryOptions optionsCopy = ParamUtils.defaultObject(options, QueryOptions::new); + Note note = noteCreateParams.toNote(Note.Scope.STUDY, tokenPayload.getUserId()); + + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + Study study = catalogManager.getStudyManager().resolveId(studyFqn, null, tokenPayload); + // Set study fqn and uid + note.setStudy(study.getFqn()); + note.setStudyUid(study.getUid()); + studyId = study.getFqn(); + studyUuid = study.getUuid(); + authorizationManager.checkIsAtLeastStudyAdministrator(organizationId, study.getUid(), tokenPayload.getUserId()); + + OpenCGAResult insert = create(note, optionsCopy, tokenPayload); + auditManager.auditCreate(organizationId, tokenPayload.getUserId(), Enums.Resource.NOTE, note.getId(), note.getUuid(), studyId, + studyUuid, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + return insert; + } catch (Exception e) { + auditManager.auditCreate(organizationId, tokenPayload.getUserId(), Enums.Resource.NOTE, noteCreateParams.getId(), "", studyId, + studyUuid, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(0, "Note create", + e.getMessage()))); + throw e; + } + } + + private OpenCGAResult create(Note note, QueryOptions options, JwtPayload tokenPayload) throws CatalogException { + String organizationId = tokenPayload.getOrganization(); + validateNewNote(note, tokenPayload.getUserId()); + OpenCGAResult insert = getCatalogDBAdaptorFactory().getCatalogNoteDBAdaptor(organizationId).insert(note); + if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { + // Fetch created note + Query query = new Query(NoteDBAdaptor.QueryParams.UID.key(), note.getUid()); + OpenCGAResult result = getCatalogDBAdaptorFactory().getCatalogNoteDBAdaptor(organizationId).get(query, options); + insert.setResults(result.getResults()); + } + return insert; + } + + public OpenCGAResult updateOrganizationNote(String noteStr, NoteUpdateParams noteUpdateParams, QueryOptions options, + String token) throws CatalogException { + ObjectMap auditParams = new ObjectMap() + .append("noteId", noteStr) + .append("update", noteUpdateParams) + .append("options", options) + .append("token", token); + + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + String organizationId = tokenPayload.getOrganization(); + String noteId = noteStr; + String noteUuid = ""; + try { + QueryOptions optionsCopy = ParamUtils.defaultObject(options, QueryOptions::new); + authorizationManager.checkIsAtLeastOrganizationOwnerOrAdmin(organizationId, tokenPayload.getUserId()); + Note note = internalGet(organizationId, -1L, noteStr).first(); + noteId = note.getId(); + noteUuid = note.getUuid(); + + OpenCGAResult update = update(note.getUid(), noteUpdateParams, optionsCopy, tokenPayload); + auditManager.auditUpdate(organizationId, tokenPayload.getUserId(), Enums.Resource.NOTE, noteId, noteUuid, "", "", auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + return update; + } catch (Exception e) { + auditManager.auditCreate(organizationId, tokenPayload.getUserId(), Enums.Resource.NOTE, noteId, noteUuid, "", "", auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(0, "Note update", e.getMessage()))); + throw e; + } + } + + public OpenCGAResult updateStudyNote(String studyStr, String noteStr, NoteUpdateParams noteUpdateParams, QueryOptions options, + String token) throws CatalogException { + ObjectMap auditParams = new ObjectMap() + .append("studyStr", studyStr) + .append("noteId", noteStr) + .append("update", noteUpdateParams) + .append("options", options) + .append("token", token); + + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + String organizationId = tokenPayload.getOrganization(); + String noteId = noteStr; + String noteUuid = ""; + String studyId = ""; + String studyUuid = ""; + try { + ParamUtils.checkParameter(studyStr, "study"); + QueryOptions optionsCopy = ParamUtils.defaultObject(options, QueryOptions::new); + + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + Study study = catalogManager.getStudyManager().resolveId(studyFqn, null, tokenPayload); + studyId = study.getFqn(); + studyUuid = study.getUuid(); + authorizationManager.checkIsAtLeastStudyAdministrator(organizationId, study.getUid(), tokenPayload.getUserId()); + + Note note = internalGet(organizationId, study.getUid(), noteStr).first(); + noteId = note.getId(); + noteUuid = note.getUuid(); + + OpenCGAResult update = update(note.getUid(), noteUpdateParams, optionsCopy, tokenPayload); + auditManager.auditUpdate(organizationId, tokenPayload.getUserId(), Enums.Resource.NOTE, noteId, noteUuid, studyId, studyUuid, + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + return update; + } catch (Exception e) { + auditManager.auditCreate(organizationId, tokenPayload.getUserId(), Enums.Resource.NOTE, noteId, noteUuid, studyId, studyUuid, + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(0, "Note update", e.getMessage()))); + throw e; + } + } + + private OpenCGAResult update(long noteUid, NoteUpdateParams noteUpdateParams, QueryOptions options, JwtPayload tokenPayload) + throws CatalogException { + ObjectMap updateMap; + try { + updateMap = noteUpdateParams != null ? noteUpdateParams.getUpdateMap() : null; + } catch (JsonProcessingException e) { + throw new CatalogException("Could not parse NoteUpdateParams object: " + e.getMessage(), e); + } + String organizationId = tokenPayload.getOrganization(); + + // Write who's performing the update + updateMap.put(NoteDBAdaptor.QueryParams.USER_ID.key(), tokenPayload.getUserId()); + + OpenCGAResult update = getCatalogDBAdaptorFactory().getCatalogNoteDBAdaptor(organizationId).update(noteUid, updateMap, + options); + if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { + // Fetch updated note + OpenCGAResult result = getCatalogDBAdaptorFactory().getCatalogNoteDBAdaptor(organizationId).get(noteUid, options); + update.setResults(result.getResults()); + } + return update; + } + + public OpenCGAResult deleteOrganizationNote(String noteId, QueryOptions options, String token) throws CatalogException { + ObjectMap auditParams = new ObjectMap() + .append("noteId", noteId) + .append("options", options) + .append("token", token); + + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + String organizationId = tokenPayload.getOrganization(); + try { + authorizationManager.checkIsAtLeastOrganizationOwnerOrAdmin(organizationId, tokenPayload.getUserId()); + + ParamUtils.checkParameter(noteId, "note id"); + QueryOptions optionsCopy = ParamUtils.defaultObject(options, QueryOptions::new); + + Note note = internalGet(organizationId, -1L, noteId).first(); + + OpenCGAResult delete = delete(note, optionsCopy, tokenPayload); + auditManager.auditDelete(organizationId, tokenPayload.getUserId(), Enums.Resource.NOTE, note.getId(), note.getUuid(), "", "", + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + return delete; + } catch (Exception e) { + auditManager.auditDelete(organizationId, tokenPayload.getUserId(), Enums.Resource.NOTE, noteId, "", "", "", auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(0, "Note delete", e.getMessage()))); + throw e; + } + } + + public OpenCGAResult deleteStudyNote(String studyStr, String noteId, QueryOptions options, String token) throws CatalogException { + ObjectMap auditParams = new ObjectMap() + .append("studyStr", studyStr) + .append("noteId", noteId) + .append("options", options) + .append("token", token); + + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + String organizationId = tokenPayload.getOrganization(); + String studyId = studyStr; + String studyUuid = ""; + try { + QueryOptions optionsCopy = ParamUtils.defaultObject(options, QueryOptions::new); + + ParamUtils.checkParameter(studyStr, "study"); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + organizationId = studyFqn.getOrganizationId(); + + Study study = catalogManager.getStudyManager().resolveId(studyFqn, null, tokenPayload); + // Set study fqn and uid + studyId = study.getFqn(); + studyUuid = study.getUuid(); + authorizationManager.checkIsAtLeastStudyAdministrator(organizationId, study.getUid(), tokenPayload.getUserId()); + + ParamUtils.checkParameter(noteId, "note id"); + + Note note = internalGet(organizationId, study.getUid(), noteId).first(); + + OpenCGAResult delete = delete(note, optionsCopy, tokenPayload); + auditManager.auditDelete(organizationId, tokenPayload.getUserId(), Enums.Resource.NOTE, note.getId(), note.getUuid(), + studyId, studyUuid, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + return delete; + } catch (Exception e) { + auditManager.auditDelete(organizationId, tokenPayload.getUserId(), Enums.Resource.NOTE, noteId, "", studyId, studyUuid, + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(0, "Note delete", e.getMessage()))); + throw e; + } + } + + private OpenCGAResult delete(Note note, QueryOptions options, JwtPayload jwtPayload) throws CatalogException { + OpenCGAResult delete = getCatalogDBAdaptorFactory().getCatalogNoteDBAdaptor(jwtPayload.getOrganization()).delete(note); + if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { + delete.setResults(Collections.singletonList(note)); + } + return delete; + } + + public static void validateNewNote(Note note, String userId) throws CatalogParameterException { + ParamUtils.checkIdentifier(note.getId(), NoteDBAdaptor.QueryParams.ID.key()); + ParamUtils.checkObj(note.getScope(), NoteDBAdaptor.QueryParams.SCOPE.key()); + if (note.getScope().equals(Note.Scope.STUDY)) { + ParamUtils.checkParameter(note.getStudy(), NoteDBAdaptor.QueryParams.STUDY.key()); + } + ParamUtils.checkObj(note.getVisibility(), NoteDBAdaptor.QueryParams.VISIBILITY.key()); + ParamUtils.checkObj(note.getValueType(), NoteDBAdaptor.QueryParams.VALUE_TYPE.key()); + ParamUtils.checkObj(note.getValue(), NoteDBAdaptor.QueryParams.VALUE.key()); + + note.setTags(CollectionUtils.isNotEmpty(note.getTags()) ? note.getTags() : Collections.emptyList()); + note.setUserId(userId); + + note.setUuid(UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.NOTES)); + note.setVersion(1); + note.setCreationDate(TimeUtils.getTime()); + note.setModificationDate(TimeUtils.getTime()); + } +} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/OrganizationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/OrganizationManager.java new file mode 100644 index 00000000000..20ec12a7f2b --- /dev/null +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/OrganizationManager.java @@ -0,0 +1,615 @@ +package org.opencb.opencga.catalog.managers; + +import com.fasterxml.jackson.core.JsonProcessingException; +import io.jsonwebtoken.security.InvalidKeyException; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.opencb.commons.datastore.core.Event; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.commons.datastore.core.result.Error; +import org.opencb.opencga.catalog.auth.authentication.CatalogAuthenticationManager; +import org.opencb.opencga.catalog.auth.authentication.azure.AuthenticationFactory; +import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager; +import org.opencb.opencga.catalog.db.DBAdaptorFactory; +import org.opencb.opencga.catalog.db.api.OrganizationDBAdaptor; +import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.exceptions.CatalogIOException; +import org.opencb.opencga.catalog.exceptions.CatalogParameterException; +import org.opencb.opencga.catalog.io.CatalogIOManager; +import org.opencb.opencga.catalog.utils.Constants; +import org.opencb.opencga.catalog.utils.JwtUtils; +import org.opencb.opencga.catalog.utils.ParamUtils; +import org.opencb.opencga.catalog.utils.UuidUtils; +import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.common.GitRepositoryState; +import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.config.AuthenticationOrigin; +import org.opencb.opencga.core.config.Configuration; +import org.opencb.opencga.core.config.Optimizations; +import org.opencb.opencga.core.models.JwtPayload; +import org.opencb.opencga.core.models.audit.AuditRecord; +import org.opencb.opencga.core.models.common.Enums; +import org.opencb.opencga.core.models.common.InternalStatus; +import org.opencb.opencga.core.models.organizations.*; +import org.opencb.opencga.core.models.user.OrganizationUserUpdateParams; +import org.opencb.opencga.core.models.user.User; +import org.opencb.opencga.core.response.OpenCGAResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; +import java.util.*; +import java.util.stream.Collectors; + +import static org.opencb.opencga.core.common.JacksonUtils.getUpdateObjectMapper; + +public class OrganizationManager extends AbstractManager { + + public static final QueryOptions INCLUDE_ORGANIZATION_IDS = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList( + OrganizationDBAdaptor.QueryParams.ID.key(), OrganizationDBAdaptor.QueryParams.UID.key(), + OrganizationDBAdaptor.QueryParams.UUID.key())); + public static final QueryOptions INCLUDE_ORGANIZATION_ADMINS = keepFieldsInQueryOptions(INCLUDE_ORGANIZATION_IDS, + Arrays.asList(OrganizationDBAdaptor.QueryParams.OWNER.key(), OrganizationDBAdaptor.QueryParams.ADMINS.key())); + public static final QueryOptions INCLUDE_ORGANIZATION_CONFIGURATION = keepFieldsInQueryOptions(INCLUDE_ORGANIZATION_ADMINS, + Collections.singletonList(OrganizationDBAdaptor.QueryParams.CONFIGURATION.key())); + protected static Logger logger = LoggerFactory.getLogger(OrganizationManager.class); + private final CatalogIOManager catalogIOManager; + private final AuthenticationFactory authenticationFactory; + + OrganizationManager(AuthorizationManager authorizationManager, AuditManager auditManager, CatalogManager catalogManager, + DBAdaptorFactory catalogDBAdaptorFactory, CatalogIOManager catalogIOManager, + AuthenticationFactory authenticationFactory, Configuration configuration) { + super(authorizationManager, auditManager, catalogManager, catalogDBAdaptorFactory, configuration); + this.catalogIOManager = catalogIOManager; + this.authenticationFactory = authenticationFactory; + } + +// OpenCGAResult internalGet(String organizationId, QueryOptions options, String user) throws CatalogException { +// return internalGet(Collections.singletonList(organizationId), null, options, user, false); +// } +// +// OpenCGAResult internalGet(List organizationList, @Nullable Query query, QueryOptions options, String user, +// boolean ignoreException) throws CatalogException { +// if (CollectionUtils.isEmpty(organizationList)) { +// throw new CatalogException("Missing organization entries."); +// } +// List uniqueList = ListUtils.unique(organizationList); +// +// QueryOptions queryOptions = new QueryOptions(ParamUtils.defaultObject(options, QueryOptions::new)); +// +// Query queryCopy = query == null ? new Query() : new Query(query); +// +// OrganizationDBAdaptor.QueryParams idQueryParam = getFieldFilter(uniqueList); +// queryCopy.put(idQueryParam.key(), uniqueList); +// +// if (!authorizationManager.isInstallationAdministrator(user)) { +// // Only admins and owner are allowed to see the organizations +// queryCopy.put(OrganizationDBAdaptor.QueryParams.ADMINS.key(), user); +// } +// +// // Ensure the field by which we are querying for will be kept in the results +// queryOptions = keepFieldInQueryOptions(queryOptions, idQueryParam.key()); +// +// OpenCGAResult organizationDataResult = getOrganizationDBAdaptor(organization).get(queryCopy, queryOptions); +// +// Function organizationStringFunction = Organization::getId; +// if (idQueryParam.equals(OrganizationDBAdaptor.QueryParams.UUID)) { +// organizationStringFunction = Organization::getUuid; +// } +// +// if (ignoreException || organizationDataResult.getNumResults() == uniqueList.size()) { +// return organizationDataResult; +// } +// +// List missingOrganizations = new ArrayList<>(organizationList.size()); +// for (Organization organization : organizationDataResult.getResults()) { +// if (!uniqueList.contains(organizationStringFunction.apply(organization))) { +// missingOrganizations.add(organizationStringFunction.apply(organization)); +// } +// } +// +// throw CatalogException.notFound("organizations", missingOrganizations); +// } + +// OrganizationDBAdaptor.QueryParams getFieldFilter(List idList) throws CatalogException { +// OrganizationDBAdaptor.QueryParams idQueryParam = null; +// for (String entry : idList) { +// OrganizationDBAdaptor.QueryParams param = OrganizationDBAdaptor.QueryParams.ID; +// if (UuidUtils.isOpenCgaUuid(entry)) { +// param = OrganizationDBAdaptor.QueryParams.UUID; +// } +// if (idQueryParam == null) { +// idQueryParam = param; +// } +// if (idQueryParam != param) { +// throw new CatalogException("Found uuids and ids in the same query. Please, choose one or do two different queries."); +// } +// } +// return idQueryParam; +// } + + public OpenCGAResult get(String organizationId, QueryOptions options, String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + String userId = tokenPayload.getUserId(organizationId); + + ObjectMap auditParams = new ObjectMap() + .append("organizationId", organizationId) + .append("options", options) + .append("token", token); + + OpenCGAResult queryResult; + try { + authorizationManager.checkCanViewOrganization(organizationId, userId); + ParamUtils.checkParameter(organizationId, "organization id"); + QueryOptions queryOptions = ParamUtils.defaultObject(options, QueryOptions::new); + boolean isOrgAdmin = authorizationManager.isAtLeastOrganizationOwnerOrAdmin(organizationId, userId); + queryOptions.put(OrganizationDBAdaptor.IS_ORGANIZATION_ADMIN_OPTION, isOrgAdmin); + queryResult = getOrganizationDBAdaptor(organizationId).get(userId, queryOptions); + privatizeResults(queryResult); + } catch (CatalogException e) { + auditManager.auditInfo(organizationId, userId, Enums.Resource.ORGANIZATION, organizationId, "", "", "", auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + throw e; + } + auditManager.auditInfo(organizationId, userId, Enums.Resource.ORGANIZATION, organizationId, "", "", "", auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + return queryResult; + } + + public OpenCGAResult create(OrganizationCreateParams organizationCreateParams, QueryOptions options, String token) + throws CatalogException { + ObjectMap auditParams = new ObjectMap() + .append("organizationCreateParams", organizationCreateParams) + .append("options", options) + .append("token", token); + + options = ParamUtils.defaultObject(options, QueryOptions::new); + OpenCGAResult queryResult; + Organization organization; + String userId = null; + try { + // The first time we create the ADMIN_ORGANIZATION as there are no users yet, we should not check anything + if (!ParamConstants.ADMIN_ORGANIZATION.equals(organizationCreateParams.getId())) { + JwtPayload jwtPayload = this.catalogManager.getUserManager().validateToken(token); + userId = jwtPayload.getUserId(organizationCreateParams.getId()); + + if (!ParamConstants.ADMIN_ORGANIZATION.equals(jwtPayload.getOrganization())) { + throw CatalogAuthorizationException.opencgaAdminOnlySupportedOperation(); + } + } + + ParamUtils.checkObj(organizationCreateParams, "organizationCreateParams"); + + organization = organizationCreateParams.toOrganization(); + validateOrganizationForCreation(organization, userId); + + queryResult = getCatalogDBAdaptorFactory().createOrganization(organization, options, userId); + if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { + OpenCGAResult result = getOrganizationDBAdaptor(organization.getId()).get(options); + organization = result.first(); + // Fetch created organization + queryResult.setResults(result.getResults()); + } + // Add required authentication manager for the new organization + authenticationFactory.configureOrganizationAuthenticationManager(organization); + + privatizeResults(queryResult); + } catch (CatalogException e) { + if (!ParamConstants.ADMIN_ORGANIZATION.equals(organizationCreateParams.getId())) { + auditManager.auditCreate(ParamConstants.ADMIN_ORGANIZATION, userId, Enums.Resource.ORGANIZATION, + organizationCreateParams.getId(), "", "", "", auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + } + throw e; + } + + try { + catalogIOManager.createOrganization(organization.getId()); + } catch (CatalogIOException e) { + auditManager.auditCreate(ParamConstants.ADMIN_ORGANIZATION, userId, Enums.Resource.ORGANIZATION, organization.getId(), "", "", + "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + try { + getCatalogDBAdaptorFactory().deleteOrganization(organization); + } catch (Exception e1) { + logger.error("Error deleting organization from catalog after failing creating the folder in the filesystem", e1); + throw e; + } + throw e; + } + + if (!ParamConstants.ADMIN_ORGANIZATION.equals(organizationCreateParams.getId())) { + // Skip old available migrations + catalogManager.getMigrationManager().skipPendingMigrations(organizationCreateParams.getId(), token); + } + + auditManager.auditCreate(ParamConstants.ADMIN_ORGANIZATION, userId, Enums.Resource.ORGANIZATION, organization.getId(), + organization.getUuid(), "", "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + + return queryResult; + } + + public OpenCGAResult update(String organizationId, OrganizationUpdateParams updateParams, QueryOptions options, + String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + String userId = tokenPayload.getUserId(organizationId); + + ObjectMap updateMap; + try { + updateMap = updateParams != null ? updateParams.getUpdateMap() : null; + } catch (JsonProcessingException e) { + throw new CatalogException("Could not parse OrganizationUpdateParams object: " + e.getMessage(), e); + } + + ObjectMap auditParams = new ObjectMap() + .append("organizationId", organizationId) + .append("updateParams", updateMap) + .append("options", options) + .append("token", token); + + options = ParamUtils.defaultObject(options, QueryOptions::new); + OpenCGAResult result = OpenCGAResult.empty(Organization.class); + try { + ParamUtils.checkObj(updateParams, "OrganizationUpdateParams"); + if (StringUtils.isNotEmpty(updateParams.getOwner()) || CollectionUtils.isNotEmpty(updateParams.getAdmins())) { + authorizationManager.checkIsAtLeastOrganizationOwner(organizationId, userId); + } else { + authorizationManager.checkIsAtLeastOrganizationOwnerOrAdmin(organizationId, userId); + } + + OpenCGAResult internalResult = get(organizationId, INCLUDE_ORGANIZATION_ADMINS, token); + if (internalResult.getNumResults() == 0) { + throw new CatalogException("Organization '" + organizationId + "' not found"); + } + Organization organization = internalResult.first(); + + // We set the proper values for the audit + organizationId = organization.getId(); + + result = getOrganizationDBAdaptor(organizationId).update(organizationId, updateMap, options); + + auditManager.auditUpdate(organizationId, userId, Enums.Resource.ORGANIZATION, organization.getId(), organization.getUuid(), "", + "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + + if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { + // Fetch updated organization + OpenCGAResult queryResult = getOrganizationDBAdaptor(organizationId).get(options); + result.setResults(queryResult.getResults()); + } + privatizeResults(result); + } catch (Exception e) { + Event event = new Event(Event.Type.ERROR, organizationId, e.getMessage()); + result.getEvents().add(event); + result.setNumErrors(result.getNumErrors() + 1); + + logger.error("Cannot update organization {}: {}", organizationId, e.getMessage()); + auditManager.auditUpdate(organizationId, userId, Enums.Resource.ORGANIZATION, organizationId, organizationId, "", "", + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(0, "Update organization", + e.getMessage()))); + throw e; + } + return result; + } + + public OpenCGAResult updateUser(@Nullable String organizationId, String userId, OrganizationUserUpdateParams updateParams, + QueryOptions options, String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + + ObjectMap auditParams = new ObjectMap() + .append("organizationId", organizationId) + .append("userId", userId) + .append("updateParams", updateParams) + .append("options", options) + .append("token", token); + + options = ParamUtils.defaultObject(options, QueryOptions::new); + String myOrganizationId = StringUtils.isNotEmpty(organizationId) ? organizationId : tokenPayload.getOrganization(); + try { + authorizationManager.checkIsAtLeastOrganizationOwnerOrAdmin(myOrganizationId, tokenPayload.getUserId(myOrganizationId)); + ParamUtils.checkObj(updateParams, "OrganizationUserUpdateParams"); + getUserDBAdaptor(myOrganizationId).checkId(userId); + + if (StringUtils.isNotEmpty(updateParams.getEmail())) { + ParamUtils.checkEmail(updateParams.getEmail()); + } + if (updateParams.getQuota() != null) { + if (updateParams.getQuota().getDiskUsage() < 0) { + throw new CatalogException("Disk usage cannot be negative"); + } + if (updateParams.getQuota().getCpuUsage() < 0) { + throw new CatalogException("CPU usage cannot be negative"); + } + if (updateParams.getQuota().getMaxDisk() < 0) { + throw new CatalogException("Max disk cannot be negative"); + } + if (updateParams.getQuota().getMaxCpu() < 0) { + throw new CatalogException("Max CPU cannot be negative"); + } + } + + ObjectMap updateMap; + try { + updateMap = updateParams.getUpdateMap(); + } catch (JsonProcessingException e) { + throw new CatalogException("Could not parse OrganizationUserUpdateParams object: " + e.getMessage(), e); + } + + if (updateParams.getInternal() != null && updateParams.getInternal().getAccount() != null + && StringUtils.isNotEmpty(updateParams.getInternal().getAccount().getExpirationDate())) { + ParamUtils.checkDateIsNotExpired(updateParams.getInternal().getAccount().getExpirationDate(), "expirationDate"); + } + + OpenCGAResult updateResult = getUserDBAdaptor(myOrganizationId).update(userId, updateMap); + auditManager.auditUpdate(myOrganizationId, tokenPayload.getUserId(myOrganizationId), Enums.Resource.USER, userId, "", "", "", + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + + if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { + // Fetch updated user + OpenCGAResult result = getUserDBAdaptor(myOrganizationId).get(userId, options); + updateResult.setResults(result.getResults()); + } + + return updateResult; + } catch (CatalogException e) { + auditManager.auditUpdate(myOrganizationId, tokenPayload.getUserId(myOrganizationId), Enums.Resource.USER, userId, "", "", "", + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + throw e; + } + } + + public OpenCGAResult updateConfiguration(String organizationId, OrganizationConfiguration updateParams, + QueryOptions options, String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + String userId = tokenPayload.getUserId(organizationId); + + ObjectMap auditParams = new ObjectMap() + .append("organizationId", organizationId) + .append("updateParams", updateParams) + .append("options", options) + .append("token", token); + + QueryOptions queryOptions = options != null ? new QueryOptions(options) : new QueryOptions(); + OpenCGAResult result = OpenCGAResult.empty(OrganizationConfiguration.class); + try { + authorizationManager.checkIsAtLeastOrganizationOwnerOrAdmin(organizationId, userId); + + ParamUtils.checkObj(updateParams, "OrganizationConfiguration"); + ObjectMap updateConfigurationMap; + try { + updateConfigurationMap = new ObjectMap(getUpdateObjectMapper().writeValueAsString(updateParams)); + } catch (JsonProcessingException e) { + throw new CatalogException("Could not parse OrganizationConfiguration object: " + e.getMessage(), e); + } + + OpenCGAResult internalResult = get(organizationId, INCLUDE_ORGANIZATION_CONFIGURATION, token); + if (internalResult.getNumResults() == 0) { + throw new CatalogException("Organization '" + organizationId + "' not found"); + } + Organization organization = internalResult.first(); + + // We set the proper values for the audit + organizationId = organization.getId(); + + if (CollectionUtils.isNotEmpty(updateParams.getAuthenticationOrigins())) { + // Check action + ParamUtils.UpdateAction authOriginsAction = null; + Map map = queryOptions.getMap(Constants.ACTIONS); + if (map == null || map.get(OrganizationDBAdaptor.AUTH_ORIGINS_FIELD) == null) { + // Write default option + authOriginsAction = ParamUtils.UpdateAction.ADD; + Map actionMap = new HashMap<>(); + actionMap.put(OrganizationDBAdaptor.AUTH_ORIGINS_FIELD, authOriginsAction); + queryOptions.put(Constants.ACTIONS, actionMap); + } else { + authOriginsAction = ParamUtils.UpdateAction.valueOf(map.get(OrganizationDBAdaptor.AUTH_ORIGINS_FIELD).toString()); + } + + Set currentAuthOriginIds = organization.getConfiguration().getAuthenticationOrigins() + .stream() + .map(AuthenticationOrigin::getId) + .collect(Collectors.toSet()); + + Set updateAuthOriginIds = new HashSet<>(); + StringBuilder authOriginUpdateBuilder = new StringBuilder(); + List authenticationOrigins = updateParams.getAuthenticationOrigins(); + for (int i = 0; i < authenticationOrigins.size(); i++) { + AuthenticationOrigin authenticationOrigin = authenticationOrigins.get(i); + ParamUtils.checkParameter(authenticationOrigin.getId(), "AuthenticationOrigin id"); + ParamUtils.checkObj(authenticationOrigin.getType(), "AuthenticationOrigin type"); + if (updateAuthOriginIds.contains(authenticationOrigin.getId())) { + throw new CatalogParameterException("Found duplicated authentication origin id '" + authenticationOrigin.getId() + + "'."); + } + // Check authOrigin OPENCGA-OPENCGA + if ((authenticationOrigin.getType().equals(AuthenticationOrigin.AuthenticationType.OPENCGA) + && !CatalogAuthenticationManager.OPENCGA.equals(authenticationOrigin.getId())) + || (!authenticationOrigin.getType().equals(AuthenticationOrigin.AuthenticationType.OPENCGA) + && CatalogAuthenticationManager.OPENCGA.equals(authenticationOrigin.getId()))) { + throw new CatalogParameterException("AuthenticationOrigin type '" + AuthenticationOrigin.AuthenticationType.OPENCGA + + "' must go together with id '" + CatalogAuthenticationManager.OPENCGA + "'."); + } + updateAuthOriginIds.add(authenticationOrigin.getId()); + if (i > 0) { + authOriginUpdateBuilder.append(", "); + } + authOriginUpdateBuilder.append(authenticationOrigin.getType()).append(": ").append(authenticationOrigin.getId()); + + if (authOriginsAction != ParamUtils.UpdateAction.REMOVE) { + // Validate configuration is correct and can be used + authenticationFactory.validateAuthenticationOrigin(authenticationOrigin); + } + } + + switch (authOriginsAction) { + case ADD: + for (AuthenticationOrigin authenticationOrigin : updateParams.getAuthenticationOrigins()) { + if (currentAuthOriginIds.contains(authenticationOrigin.getId())) { + throw new CatalogException("Authentication origin '" + authenticationOrigin.getId() + "' already exists. " + + "Please, set the authOriginsAction to 'REPLACE' to replace the current configuration."); + } + } + logger.debug("Adding new list of Authentication Origins: {}.", authOriginUpdateBuilder); + break; + case SET: + boolean userAuthOriginFound = false; + for (AuthenticationOrigin authenticationOrigin : updateParams.getAuthenticationOrigins()) { + if (tokenPayload.getAuthOrigin().equals(authenticationOrigin.getType())) { + userAuthOriginFound = true; + } + } + if (!userAuthOriginFound) { + throw new CatalogException("User authentication origin not found in the list of authentication origins. " + + "Please, add an AuthenticationOrigin of type '" + tokenPayload.getAuthOrigin() + "' to the list."); + } + logger.debug("Set new list of Authentication Origins: {}.", authOriginUpdateBuilder); + break; + case REMOVE: + for (AuthenticationOrigin authenticationOrigin : updateParams.getAuthenticationOrigins()) { + if (!currentAuthOriginIds.contains(authenticationOrigin.getId())) { + throw new CatalogException("Authentication origin '" + authenticationOrigin.getId() + "' does not exist. " + + "The current available authentication origin ids are: '" + + StringUtils.join(currentAuthOriginIds, ", ") + "'."); + } + if (tokenPayload.getAuthOrigin().equals(authenticationOrigin.getType())) { + throw new CatalogException("Removing the authentication origin '" + tokenPayload.getAuthOrigin() + "' " + + "not allowed. Your user account uses that AuthenticationOrigin."); + } + } + logger.debug("Removing list of Authentication Origins: {}.", authOriginUpdateBuilder); + break; + case REPLACE: + for (AuthenticationOrigin authenticationOrigin : updateParams.getAuthenticationOrigins()) { + if (!currentAuthOriginIds.contains(authenticationOrigin.getId())) { + throw new CatalogException("Authentication origin '" + authenticationOrigin.getId() + "' not found." + + "Please, set the authOriginsAction to 'ADD' to add the new AuthenticationOrigin."); + } + } + logger.debug("Replace list of Authentication Origins: {}.", authOriginUpdateBuilder); + break; + default: + throw new CatalogParameterException("Unknown authentication origins action " + authOriginsAction); + } + } + + if (updateParams.getToken() != null) { + try { + JwtUtils.validateJWTKey(updateParams.getToken().getAlgorithm(), updateParams.getToken().getSecretKey()); + } catch (InvalidKeyException e) { + throw new CatalogParameterException("Invalid secret key - algorithm for JWT token: " + e.getMessage(), e); + } + if (updateParams.getToken().getExpiration() <= 0) { + throw new CatalogParameterException("Invalid expiration for JWT token. It must be a positive number."); + } + } + + ObjectMap updateMap = new ObjectMap(OrganizationDBAdaptor.QueryParams.CONFIGURATION.key(), updateConfigurationMap); + OpenCGAResult update = getOrganizationDBAdaptor(organizationId).update(organizationId, updateMap, queryOptions); + result.append(update); + auditManager.auditUpdate(organizationId, userId, Enums.Resource.ORGANIZATION, organization.getId(), organization.getUuid(), "", + "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + + organization = null; + if (queryOptions.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { + // Fetch updated organization + organization = getOrganizationDBAdaptor(organizationId).get(INCLUDE_ORGANIZATION_CONFIGURATION).first(); + result.setResults(Collections.singletonList(organization.getConfiguration())); + } + + if (CollectionUtils.isNotEmpty(updateParams.getAuthenticationOrigins()) || updateParams.getToken() != null) { + if (organization == null) { + organization = getOrganizationDBAdaptor(organizationId).get(INCLUDE_ORGANIZATION_CONFIGURATION).first(); + } + authenticationFactory.configureOrganizationAuthenticationManager(organization); + } + + } catch (Exception e) { + auditManager.auditUpdate(organizationId, userId, Enums.Resource.ORGANIZATION, organizationId, organizationId, "", "", + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(0, "Update organization", + e.getMessage()))); + throw e; + } + return result; + } + + private void validateOrganizationForCreation(Organization organization, String userId) throws CatalogParameterException { + ParamUtils.checkParameter(organization.getId(), OrganizationDBAdaptor.QueryParams.ID.key()); + + organization.setUuid(UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.ORGANIZATION)); + organization.setName(ParamUtils.defaultString(organization.getName(), organization.getId())); + organization.setCreationDate(ParamUtils.checkDateOrGetCurrentDate(organization.getCreationDate(), + OrganizationDBAdaptor.QueryParams.CREATION_DATE.key())); + organization.setModificationDate(ParamUtils.checkDateOrGetCurrentDate(organization.getModificationDate(), + OrganizationDBAdaptor.QueryParams.MODIFICATION_DATE.key())); + organization.setInternal(new OrganizationInternal(new InternalStatus(), TimeUtils.getTime(), TimeUtils.getTime(), + GitRepositoryState.getInstance().getBuildVersion(), Collections.emptyList())); + organization.setOwner(userId); + organization.setAdmins(Collections.emptyList()); + organization.setProjects(Collections.emptyList()); + + if (organization.getConfiguration() == null) { + organization.setConfiguration(new OrganizationConfiguration()); + } + validateOrganizationConfiguration(organization); + + organization.setAttributes(ParamUtils.defaultObject(organization.getAttributes(), HashMap::new)); + } + + private void validateOrganizationConfiguration(Organization organization) throws CatalogParameterException { + if (CollectionUtils.isNotEmpty(organization.getConfiguration().getAuthenticationOrigins())) { + for (AuthenticationOrigin authenticationOrigin : organization.getConfiguration().getAuthenticationOrigins()) { + ParamUtils.checkParameter(authenticationOrigin.getId(), "AuthenticationOrigin id"); + } + } else { + organization.getConfiguration() + .setAuthenticationOrigins(Collections.singletonList( + CatalogAuthenticationManager.createOpencgaAuthenticationOrigin())); + } + if (organization.getConfiguration().getToken() == null + || StringUtils.isEmpty(organization.getConfiguration().getToken().getSecretKey())) { + organization.getConfiguration().setToken(TokenConfiguration.init()); + } + organization.getConfiguration().setDefaultUserExpirationDate(ParamUtils.defaultString( + organization.getConfiguration().getDefaultUserExpirationDate(), Constants.DEFAULT_USER_EXPIRATION_DATE)); + if (organization.getConfiguration().getOptimizations() == null) { + organization.getConfiguration().setOptimizations(new Optimizations(false)); + } + } + + Set getOrganizationOwnerAndAdmins(String organizationId) throws CatalogException { + OpenCGAResult result = getOrganizationDBAdaptor(organizationId).get(INCLUDE_ORGANIZATION_ADMINS); + if (result.getNumResults() == 0) { + throw new CatalogException("Could not get owner and admins of organization '" + organizationId + "'. Organization not found."); + } + Organization organization = result.first(); + Set users = new HashSet<>(); + if (StringUtils.isNotEmpty(organization.getOwner())) { + users.add(organization.getOwner()); + } + if (CollectionUtils.isNotEmpty(organization.getAdmins())) { + users.addAll(organization.getAdmins()); + } + return users; + } + + public List getOrganizationIds(String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + authorizationManager.checkIsOpencgaAdministrator(tokenPayload, "get all organization ids"); + return getCatalogDBAdaptorFactory().getOrganizationIds(); + } + + private void privatizeResults(OpenCGAResult result) { + if (CollectionUtils.isNotEmpty(result.getResults())) { + for (Organization organization : result.getResults()) { + if (organization.getConfiguration() != null) { + organization.getConfiguration().setToken(null); + if (CollectionUtils.isNotEmpty(organization.getConfiguration().getAuthenticationOrigins())) { + for (AuthenticationOrigin authenticationOrigin : organization.getConfiguration().getAuthenticationOrigins()) { + authenticationOrigin.setOptions(null); + } + } + } + } + } + } +} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/PanelManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/PanelManager.java index e0f6b2284fe..fddf86b32cb 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/PanelManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/PanelManager.java @@ -36,6 +36,7 @@ import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.models.InternalGetDataResult; +import org.opencb.opencga.catalog.utils.CatalogFqn; import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.catalog.utils.UuidUtils; @@ -45,6 +46,7 @@ import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.models.AclEntryList; import org.opencb.opencga.core.models.AclParams; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.audit.AuditRecord; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.panel.Panel; @@ -92,8 +94,8 @@ Enums.Resource getEntity() { } @Override - InternalGetDataResult internalGet(long studyUid, List entryList, @Nullable Query query, QueryOptions options, - String user, boolean ignoreException) throws CatalogException { + InternalGetDataResult internalGet(String organizationId, long studyUid, List entryList, @Nullable Query query, + QueryOptions options, String user, boolean ignoreException) throws CatalogException { if (ListUtils.isEmpty(entryList)) { throw new CatalogException("Missing panel entries."); } @@ -115,7 +117,7 @@ InternalGetDataResult internalGet(long studyUid, List entryList, // Ensure the field by which we are querying for will be kept in the results queryOptions = keepFieldInQueryOptions(queryOptions, idQueryParam.key()); - OpenCGAResult panelDataResult = panelDBAdaptor.get(studyUid, queryCopy, queryOptions, user); + OpenCGAResult panelDataResult = getPanelDBAdaptor(organizationId).get(studyUid, queryCopy, queryOptions, user); Function panelStringFunction = Panel::getId; if (idQueryParam.equals(PanelDBAdaptor.QueryParams.UUID)) { @@ -126,7 +128,7 @@ InternalGetDataResult internalGet(long studyUid, List entryList, return keepOriginalOrder(uniqueList, panelStringFunction, panelDataResult, ignoreException, versioned); } // Query without adding the user check - OpenCGAResult resultsNoCheck = panelDBAdaptor.get(queryCopy, queryOptions); + OpenCGAResult resultsNoCheck = getPanelDBAdaptor(organizationId).get(queryCopy, queryOptions); if (resultsNoCheck.getNumResults() == panelDataResult.getNumResults()) { throw CatalogException.notFound("panels", getMissingFields(uniqueList, panelDataResult.getResults(), panelStringFunction)); @@ -152,17 +154,21 @@ PanelDBAdaptor.QueryParams getFieldFilter(List idList) throws CatalogExc return idQueryParam; } - private OpenCGAResult getPanel(long studyUid, String panelUuid, QueryOptions options) throws CatalogException { + private OpenCGAResult getPanel(String organizationId, long studyUid, String panelUuid, QueryOptions options) + throws CatalogException { Query query = new Query() .append(PanelDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(PanelDBAdaptor.QueryParams.UUID.key(), panelUuid); - return panelDBAdaptor.get(query, options); + return getPanelDBAdaptor(organizationId).get(query, options); } @Override public OpenCGAResult create(String studyStr, Panel panel, QueryOptions options, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = catalogManager.getStudyManager().resolveId(studyStr, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyStr, userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("study", studyStr) @@ -171,31 +177,34 @@ public OpenCGAResult create(String studyStr, Panel panel, QueryOptions op .append("token", token); try { // 1. We check everything can be done - authorizationManager.checkStudyPermission(study.getUid(), userId, StudyPermissions.Permissions.WRITE_PANELS); + authorizationManager.checkStudyPermission(organizationId, study.getUid(), userId, StudyPermissions.Permissions.WRITE_PANELS); - autoCompletePanel(study, panel); + autoCompletePanel(organizationId, study, panel); options = ParamUtils.defaultObject(options, QueryOptions::new); - OpenCGAResult insert = panelDBAdaptor.insert(study.getUid(), panel, options); + OpenCGAResult insert = getPanelDBAdaptor(organizationId).insert(study.getUid(), panel, options); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { // Fetch created panel - OpenCGAResult result = getPanel(study.getUid(), panel.getUuid(), options); + OpenCGAResult result = getPanel(organizationId, study.getUid(), panel.getUuid(), options); insert.setResults(result.getResults()); } - auditManager.auditCreate(userId, Enums.Resource.DISEASE_PANEL, panel.getId(), panel.getUuid(), study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.auditCreate(organizationId, userId, Enums.Resource.DISEASE_PANEL, panel.getId(), panel.getUuid(), study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return insert; } catch (CatalogException e) { - auditManager.auditCreate(userId, Enums.Resource.DISEASE_PANEL, panel.getId(), "", study.getId(), study.getUuid(), auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditCreate(organizationId, userId, Enums.Resource.DISEASE_PANEL, panel.getId(), "", study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } public OpenCGAResult importFromSource(String studyId, String source, String panelIds, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = catalogManager.getStudyManager().resolveId(studyId, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyId, userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) @@ -206,7 +215,7 @@ public OpenCGAResult importFromSource(String studyId, String source, Stri try { // 1. We check everything can be done - authorizationManager.checkStudyPermission(study.getUid(), userId, StudyPermissions.Permissions.WRITE_PANELS); + authorizationManager.checkStudyPermission(organizationId, study.getUid(), userId, StudyPermissions.Permissions.WRITE_PANELS); ParamUtils.checkParameter(source, "source"); List sources = Arrays.asList(source.split(",")); @@ -280,40 +289,40 @@ public OpenCGAResult importFromSource(String studyId, String source, Stri logger.info("Downloading panel '{}' from '{}'", panelId, url); try (InputStream inputStream = url.openStream()) { Panel panel = JacksonUtils.getDefaultObjectMapper().readValue(inputStream, Panel.class); - autoCompletePanel(study, panel); + autoCompletePanel(organizationId, study, panel); panelList.add(panel); } } logger.info("Inserting panels in database"); - result.append(panelDBAdaptor.insert(study.getUid(), panelList)); + result.append(getPanelDBAdaptor(organizationId).insert(study.getUid(), panelList)); } result.setResults(importedPanels); auditManager.initAuditBatch(operationId); // Audit creation for (Panel importedPanel : importedPanels) { - auditManager.audit(operationId, userId, Enums.Action.IMPORT, Enums.Resource.DISEASE_PANEL, importedPanel.getId(), - importedPanel.getUuid(), study.getId(), study.getUuid(), auditParams, + auditManager.audit(organizationId, operationId, userId, Enums.Action.IMPORT, Enums.Resource.DISEASE_PANEL, + importedPanel.getId(), importedPanel.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS), new ObjectMap()); } - auditManager.finishAuditBatch(operationId); + auditManager.finishAuditBatch(organizationId, operationId); return result; } catch (CatalogException e) { - auditManager.audit(operationId, userId, Enums.Action.IMPORT, Enums.Resource.DISEASE_PANEL, "", "", + auditManager.audit(organizationId, operationId, userId, Enums.Action.IMPORT, Enums.Resource.DISEASE_PANEL, "", "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError()), new ObjectMap()); throw e; } catch (IOException e) { CatalogException exception = new CatalogException("Error parsing panels: " + e.getMessage(), e); - auditManager.audit(operationId, userId, Enums.Action.IMPORT, Enums.Resource.DISEASE_PANEL, "", "", + auditManager.audit(organizationId, operationId, userId, Enums.Action.IMPORT, Enums.Resource.DISEASE_PANEL, "", "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, exception.getError()), new ObjectMap()); throw exception; } } - private void autoCompletePanel(Study study, Panel panel) throws CatalogException { + private void autoCompletePanel(String organizationId, Study study, Panel panel) throws CatalogException { // Check all the panel fields ParamUtils.checkIdentifier(panel.getId(), "id"); panel.setName(ParamUtils.defaultString(panel.getName(), panel.getId())); @@ -344,8 +353,11 @@ public OpenCGAResult update(String studyId, Query query, PanelUpdateParam QueryOptions options, String token) throws CatalogException { Query finalQuery = new Query(ParamUtils.defaultObject(query, Query::new)); - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyId, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyId, userId, organizationId); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -368,10 +380,10 @@ public OpenCGAResult update(String studyId, Query query, PanelUpdateParam DBIterator iterator; try { finalQuery.append(PanelDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - iterator = panelDBAdaptor.iterator(study.getUid(), finalQuery, INCLUDE_PANEL_IDS, userId); + iterator = getPanelDBAdaptor(organizationId).iterator(study.getUid(), finalQuery, INCLUDE_PANEL_IDS, userId); } catch (CatalogException e) { - auditManager.auditUpdate(operationId, userId, Enums.Resource.DISEASE_PANEL, "", "", study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.DISEASE_PANEL, "", "", study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -380,30 +392,33 @@ public OpenCGAResult update(String studyId, Query query, PanelUpdateParam while (iterator.hasNext()) { Panel panel = iterator.next(); try { - OpenCGAResult updateResult = update(study, panel, updateParams, options, userId); + OpenCGAResult updateResult = update(organizationId, study, panel, updateParams, options, userId); result.append(updateResult); - auditManager.auditUpdate(operationId, userId, Enums.Resource.DISEASE_PANEL, panel.getId(), panel.getUuid(), study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.DISEASE_PANEL, panel.getId(), panel.getUuid(), + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { Event event = new Event(Event.Type.ERROR, panel.getId(), e.getMessage()); result.getEvents().add(event); result.setNumErrors(result.getNumErrors() + 1); logger.error("Could not update panel {}: {}", panel.getId(), e.getMessage(), e); - auditManager.auditUpdate(operationId, userId, Enums.Resource.DISEASE_PANEL, panel.getId(), panel.getUuid(), study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.DISEASE_PANEL, panel.getId(), panel.getUuid(), + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); } } - auditManager.finishAuditBatch(operationId); + auditManager.finishAuditBatch(organizationId, operationId); return endResult(result, ignoreException); } public OpenCGAResult update(String studyStr, String panelId, PanelUpdateParams updateParams, QueryOptions options, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, userId, organizationId); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -424,7 +439,7 @@ public OpenCGAResult update(String studyStr, String panelId, PanelUpdateP OpenCGAResult result = OpenCGAResult.empty(); String panelUuid = ""; try { - OpenCGAResult internalResult = internalGet(study.getUid(), panelId, QueryOptions.empty(), userId); + OpenCGAResult internalResult = internalGet(organizationId, study.getUid(), panelId, QueryOptions.empty(), userId); if (internalResult.getNumResults() == 0) { throw new CatalogException("Panel '" + panelId + "' not found"); } @@ -434,18 +449,18 @@ public OpenCGAResult update(String studyStr, String panelId, PanelUpdateP panelId = panel.getId(); panelUuid = panel.getUuid(); - OpenCGAResult updateResult = update(study, panel, updateParams, options, userId); + OpenCGAResult updateResult = update(organizationId, study, panel, updateParams, options, userId); result.append(updateResult); - auditManager.auditUpdate(userId, Enums.Resource.DISEASE_PANEL, panel.getId(), panel.getUuid(), study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.auditUpdate(organizationId, userId, Enums.Resource.DISEASE_PANEL, panel.getId(), panel.getUuid(), study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { Event event = new Event(Event.Type.ERROR, panelId, e.getMessage()); result.getEvents().add(event); result.setNumErrors(result.getNumErrors() + 1); logger.error("Could not update panel {}: {}", panelId, e.getMessage(), e); - auditManager.auditUpdate(operationId, userId, Enums.Resource.DISEASE_PANEL, panelId, panelUuid, study.getId(), + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.DISEASE_PANEL, panelId, panelUuid, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -456,7 +471,8 @@ public OpenCGAResult update(String studyStr, String panelId, PanelUpdateP /** * Update Panel from catalog. * - * @param studyStr Study id in string format. Could be one of [id|user@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy]. + * @param studyStr Study id in string format. Could be one of + * [id|organization@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy] * @param panelIds List of Panel ids. Could be either the id or uuid. * @param updateParams Data model filled only with the parameters to be updated. * @param options QueryOptions object. @@ -472,8 +488,11 @@ public OpenCGAResult update(String studyStr, List panelIds, Panel public OpenCGAResult update(String studyStr, List panelIds, PanelUpdateParams updateParams, boolean ignoreException, QueryOptions options, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, userId, organizationId); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -499,7 +518,7 @@ public OpenCGAResult update(String studyStr, List panelIds, Panel String panelUuid = ""; try { - OpenCGAResult internalResult = internalGet(study.getUid(), panelId, QueryOptions.empty(), userId); + OpenCGAResult internalResult = internalGet(organizationId, study.getUid(), panelId, QueryOptions.empty(), userId); if (internalResult.getNumResults() == 0) { throw new CatalogException("Panel '" + id + "' not found"); } @@ -509,28 +528,28 @@ public OpenCGAResult update(String studyStr, List panelIds, Panel panelId = panel.getId(); panelUuid = panel.getUuid(); - OpenCGAResult updateResult = update(study, panel, updateParams, options, userId); + OpenCGAResult updateResult = update(organizationId, study, panel, updateParams, options, userId); result.append(updateResult); - auditManager.auditUpdate(userId, Enums.Resource.DISEASE_PANEL, panel.getId(), panel.getUuid(), study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.auditUpdate(organizationId, userId, Enums.Resource.DISEASE_PANEL, panel.getId(), panel.getUuid(), + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { Event event = new Event(Event.Type.ERROR, panelId, e.getMessage()); result.getEvents().add(event); result.setNumErrors(result.getNumErrors() + 1); logger.error("Could not update panel {}: {}", panelId, e.getMessage(), e); - auditManager.auditUpdate(operationId, userId, Enums.Resource.DISEASE_PANEL, panelId, panelUuid, study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.DISEASE_PANEL, panelId, panelUuid, + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); } } - auditManager.finishAuditBatch(operationId); + auditManager.finishAuditBatch(organizationId, operationId); return endResult(result, ignoreException); } - private OpenCGAResult update(Study study, Panel panel, PanelUpdateParams updateParams, QueryOptions options, String userId) - throws CatalogException { + private OpenCGAResult update(String organizationId, Study study, Panel panel, PanelUpdateParams updateParams, QueryOptions options, + String userId) throws CatalogException { ObjectMap parameters = new ObjectMap(); if (updateParams != null) { try { @@ -545,17 +564,17 @@ private OpenCGAResult update(Study study, Panel panel, PanelUpdateParams updateP ParamUtils.checkUpdateParametersMap(parameters); // Check update permissions - authorizationManager.checkPanelPermission(study.getUid(), panel.getUid(), userId, PanelPermissions.WRITE); + authorizationManager.checkPanelPermission(organizationId, study.getUid(), panel.getUid(), userId, PanelPermissions.WRITE); if (parameters.containsKey(PanelDBAdaptor.QueryParams.ID.key())) { ParamUtils.checkIdentifier(parameters.getString(PanelDBAdaptor.QueryParams.ID.key()), PanelDBAdaptor.QueryParams.ID.key()); } - OpenCGAResult update = panelDBAdaptor.update(panel.getUid(), parameters, options); + OpenCGAResult update = getPanelDBAdaptor(organizationId).update(panel.getUid(), parameters, options); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { // Fetch updated panel - OpenCGAResult result = panelDBAdaptor.get(study.getUid(), + OpenCGAResult result = getPanelDBAdaptor(organizationId).get(study.getUid(), new Query(PanelDBAdaptor.QueryParams.UID.key(), panel.getUid()), options, userId); update.setResults(result.getResults()); } @@ -567,12 +586,15 @@ public DBIterator iterator(String studyStr, Query query, QueryOptions opt query = ParamUtils.defaultObject(query, Query::new); options = ParamUtils.defaultObject(options, QueryOptions::new); - String userId = userManager.getUserId(sessionId); - Study study = catalogManager.getStudyManager().resolveId(studyStr, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(sessionId); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyStr, userId, organizationId); fixQueryObject(query); query.append(PanelDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - return panelDBAdaptor.iterator(study.getUid(), query, options, userId); + return getPanelDBAdaptor(organizationId).iterator(study.getUid(), query, options, userId); } @Override @@ -580,9 +602,12 @@ public OpenCGAResult search(String studyId, Query query, QueryOptions opt query = ParamUtils.defaultObject(query, Query::new); options = ParamUtils.defaultObject(options, QueryOptions::new); - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyId, userId, new QueryOptions(QueryOptions.INCLUDE, - StudyDBAdaptor.QueryParams.VARIABLE_SET.key())); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyId, new QueryOptions(QueryOptions.INCLUDE, + StudyDBAdaptor.QueryParams.VARIABLE_SET.key()), userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) @@ -593,13 +618,13 @@ public OpenCGAResult search(String studyId, Query query, QueryOptions opt try { fixQueryObject(query); query.append(PanelDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - OpenCGAResult result = panelDBAdaptor.get(study.getUid(), query, options, userId); + OpenCGAResult result = getPanelDBAdaptor(organizationId).get(study.getUid(), query, options, userId); - auditManager.auditSearch(userId, Enums.Resource.DISEASE_PANEL, study.getId(), study.getUuid(), auditParams, + auditManager.auditSearch(organizationId, userId, Enums.Resource.DISEASE_PANEL, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return result; } catch (CatalogException e) { - auditManager.auditSearch(userId, Enums.Resource.DISEASE_PANEL, study.getId(), study.getUuid(), auditParams, + auditManager.auditSearch(organizationId, userId, Enums.Resource.DISEASE_PANEL, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -609,8 +634,11 @@ public OpenCGAResult search(String studyId, Query query, QueryOptions opt public OpenCGAResult distinct(String studyId, List fields, Query query, String token) throws CatalogException { query = ParamUtils.defaultObject(query, Query::new); - String userId = userManager.getUserId(token); - Study study = catalogManager.getStudyManager().resolveId(studyId, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyId, userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) @@ -622,14 +650,14 @@ public OpenCGAResult distinct(String studyId, List fields, Query quer fixQueryObject(query); query.append(PanelDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - OpenCGAResult result = panelDBAdaptor.distinct(study.getUid(), fields, query, userId); + OpenCGAResult result = getPanelDBAdaptor(organizationId).distinct(study.getUid(), fields, query, userId); - auditManager.auditDistinct(userId, Enums.Resource.DISEASE_PANEL, study.getId(), study.getUuid(), auditParams, + auditManager.auditDistinct(organizationId, userId, Enums.Resource.DISEASE_PANEL, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return result; } catch (CatalogException e) { - auditManager.auditDistinct(userId, Enums.Resource.DISEASE_PANEL, study.getId(), study.getUuid(), auditParams, + auditManager.auditDistinct(organizationId, userId, Enums.Resource.DISEASE_PANEL, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -639,9 +667,12 @@ public OpenCGAResult distinct(String studyId, List fields, Query quer public OpenCGAResult count(String studyId, Query query, String token) throws CatalogException { query = ParamUtils.defaultObject(query, Query::new); - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyId, userId, new QueryOptions(QueryOptions.INCLUDE, - StudyDBAdaptor.QueryParams.VARIABLE_SET.key())); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyId, new QueryOptions(QueryOptions.INCLUDE, + StudyDBAdaptor.QueryParams.VARIABLE_SET.key()), userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) @@ -652,15 +683,15 @@ public OpenCGAResult count(String studyId, Query query, String token) thr query.append(PanelDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); // Here view permissions will be checked - OpenCGAResult queryResultAux = panelDBAdaptor.count(query, userId); + OpenCGAResult queryResultAux = getPanelDBAdaptor(organizationId).count(query, userId); - auditManager.auditCount(userId, Enums.Resource.DISEASE_PANEL, study.getId(), study.getUuid(), auditParams, + auditManager.auditCount(organizationId, userId, Enums.Resource.DISEASE_PANEL, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return new OpenCGAResult<>(queryResultAux.getTime(), queryResultAux.getEvents(), 0, Collections.emptyList(), queryResultAux.getNumMatches()); } catch (CatalogException e) { - auditManager.auditCount(userId, Enums.Resource.DISEASE_PANEL, study.getId(), study.getUuid(), auditParams, + auditManager.auditCount(organizationId, userId, Enums.Resource.DISEASE_PANEL, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -677,8 +708,11 @@ public OpenCGAResult delete(String studyStr, List panelIds, ObjectMap pa throw new CatalogException("Missing list of panel ids"); } - String userId = catalogManager.getUserManager().getUserId(token); - Study study = studyManager.resolveId(studyStr, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, userId, organizationId); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -692,10 +726,11 @@ public OpenCGAResult delete(String studyStr, List panelIds, ObjectMap pa boolean checkPermissions; try { // If the user is the owner or the admin, we won't check if he has permissions for every single entry - checkPermissions = !authorizationManager.isOwnerOrAdmin(study.getUid(), userId); + long studyId = study.getUid(); + checkPermissions = !authorizationManager.isAtLeastStudyAdministrator(organizationId, studyId, userId); } catch (CatalogException e) { - auditManager.auditDelete(operationId, userId, Enums.Resource.DISEASE_PANEL, "", "", study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditDelete(organizationId, operationId, userId, Enums.Resource.DISEASE_PANEL, "", "", study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -706,7 +741,7 @@ public OpenCGAResult delete(String studyStr, List panelIds, ObjectMap pa String panelId = id; String panelUuid = ""; try { - OpenCGAResult internalResult = internalGet(study.getUid(), id, INCLUDE_PANEL_IDS, userId); + OpenCGAResult internalResult = internalGet(organizationId, study.getUid(), id, INCLUDE_PANEL_IDS, userId); if (internalResult.getNumResults() == 0) { throw new CatalogException("Panel '" + id + "' not found"); } @@ -717,7 +752,7 @@ public OpenCGAResult delete(String studyStr, List panelIds, ObjectMap pa panelUuid = panel.getUuid(); if (checkPermissions) { - authorizationManager.checkPanelPermission(study.getUid(), panel.getUid(), userId, + authorizationManager.checkPanelPermission(organizationId, study.getUid(), panel.getUid(), userId, PanelPermissions.DELETE); } @@ -725,21 +760,21 @@ public OpenCGAResult delete(String studyStr, List panelIds, ObjectMap pa // TODO: Check if the panel is used in an interpretation. At this point, it can be deleted no matter what. // Delete the panel - result.append(panelDBAdaptor.delete(panel)); + result.append(getPanelDBAdaptor(organizationId).delete(panel)); - auditManager.auditDelete(operationId, userId, Enums.Resource.DISEASE_PANEL, panel.getId(), panel.getUuid(), study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.auditDelete(organizationId, operationId, userId, Enums.Resource.DISEASE_PANEL, panel.getId(), panel.getUuid(), + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { Event event = new Event(Event.Type.ERROR, id, e.getMessage()); result.getEvents().add(event); result.setNumErrors(result.getNumErrors() + 1); logger.error("Cannot delete panel {}: {}", panelId, e.getMessage()); - auditManager.auditDelete(operationId, userId, Enums.Resource.DISEASE_PANEL, panelId, panelUuid, study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditDelete(organizationId, operationId, userId, Enums.Resource.DISEASE_PANEL, panelId, panelUuid, + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); } } - auditManager.finishAuditBatch(operationId); + auditManager.finishAuditBatch(organizationId, operationId); return endResult(result, ignoreException); } @@ -754,8 +789,11 @@ public OpenCGAResult delete(String studyStr, Query query, ObjectMap params, bool Query finalQuery = new Query(ParamUtils.defaultObject(query, Query::new)); OpenCGAResult result = OpenCGAResult.empty(); - String userId = catalogManager.getUserManager().getUserId(token); - Study study = studyManager.resolveId(studyStr, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, userId, organizationId); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -775,13 +813,14 @@ public OpenCGAResult delete(String studyStr, Query query, ObjectMap params, bool try { finalQuery.append(FamilyDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - iterator = panelDBAdaptor.iterator(study.getUid(), finalQuery, INCLUDE_PANEL_IDS, userId); + iterator = getPanelDBAdaptor(organizationId).iterator(study.getUid(), finalQuery, INCLUDE_PANEL_IDS, userId); // If the user is the owner or the admin, we won't check if he has permissions for every single entry - checkPermissions = !authorizationManager.isOwnerOrAdmin(study.getUid(), userId); + long studyId = study.getUid(); + checkPermissions = !authorizationManager.isAtLeastStudyAdministrator(organizationId, studyId, userId); } catch (CatalogException e) { - auditManager.auditDelete(operationId, userId, Enums.Resource.DISEASE_PANEL, "", "", study.getId(), study.getUuid(), auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditDelete(organizationId, operationId, userId, Enums.Resource.DISEASE_PANEL, "", "", study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -791,7 +830,7 @@ public OpenCGAResult delete(String studyStr, Query query, ObjectMap params, bool try { if (checkPermissions) { - authorizationManager.checkPanelPermission(study.getUid(), panel.getUid(), userId, + authorizationManager.checkPanelPermission(organizationId, study.getUid(), panel.getUid(), userId, PanelPermissions.DELETE); } @@ -799,10 +838,10 @@ public OpenCGAResult delete(String studyStr, Query query, ObjectMap params, bool // TODO: Check if the panel is used in an interpretation. At this point, it can be deleted no matter what. // Delete the panel - result.append(panelDBAdaptor.delete(panel)); + result.append(getPanelDBAdaptor(organizationId).delete(panel)); - auditManager.auditDelete(operationId, userId, Enums.Resource.DISEASE_PANEL, panel.getId(), panel.getUuid(), study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.auditDelete(organizationId, operationId, userId, Enums.Resource.DISEASE_PANEL, panel.getId(), panel.getUuid(), + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { String errorMsg = "Cannot delete panel " + panel.getId() + ": " + e.getMessage(); @@ -811,26 +850,29 @@ public OpenCGAResult delete(String studyStr, Query query, ObjectMap params, bool result.setNumErrors(result.getNumErrors() + 1); logger.error(errorMsg); - auditManager.auditDelete(operationId, userId, Enums.Resource.DISEASE_PANEL, panel.getId(), panel.getUuid(), study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditDelete(organizationId, operationId, userId, Enums.Resource.DISEASE_PANEL, panel.getId(), panel.getUuid(), + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); } } - auditManager.finishAuditBatch(operationId); + auditManager.finishAuditBatch(organizationId, operationId); return endResult(result, ignoreException); } @Override - public OpenCGAResult rank(String studyStr, Query query, String field, int numResults, boolean asc, String sessionId) + public OpenCGAResult rank(String studyStr, Query query, String field, int numResults, boolean asc, String token) throws CatalogException { query = ParamUtils.defaultObject(query, Query::new); ParamUtils.checkObj(field, "field"); - ParamUtils.checkObj(sessionId, "sessionId"); + ParamUtils.checkObj(token, "token"); - String userId = userManager.getUserId(sessionId); - Study study = catalogManager.getStudyManager().resolveId(studyStr, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyStr, userId, organizationId); - authorizationManager.checkStudyPermission(study.getUid(), userId, StudyPermissions.Permissions.VIEW_PANELS); + authorizationManager.checkStudyPermission(organizationId, study.getUid(), userId, StudyPermissions.Permissions.VIEW_PANELS); fixQueryObject(query); @@ -840,7 +882,7 @@ public OpenCGAResult rank(String studyStr, Query query, String field, int numRes OpenCGAResult queryResult = null; if (count) { // We do not need to check for permissions when we show the count of files - queryResult = panelDBAdaptor.rank(query, field, numResults, asc); + queryResult = getPanelDBAdaptor(organizationId).rank(query, field, numResults, asc); } return ParamUtils.defaultObject(queryResult, OpenCGAResult::new); @@ -852,31 +894,37 @@ public OpenCGAResult groupBy(@Nullable String studyStr, Query query, List> getAcls(String studyId, List panelList, String member, boolean ignoreException, String token) throws CatalogException { - return getAcls(studyId, panelList, StringUtils.isNotEmpty(member) ? Collections.singletonList(member) : Collections.emptyList(), - ignoreException, token); + return getAcls(studyId, panelList, + StringUtils.isNotEmpty(member) ? Collections.singletonList(member) : Collections.emptyList(), ignoreException, token); } public OpenCGAResult> getAcls(String studyId, List panelList, List members, boolean ignoreException, String token) throws CatalogException { - String user = userManager.getUserId(token); - Study study = studyManager.resolveId(studyId, user); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyId, userId, organizationId); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); ObjectMap auditParams = new ObjectMap() @@ -890,7 +938,8 @@ public OpenCGAResult> getAcls(String studyId, Lis Map missingMap = new HashMap<>(); try { auditManager.initAuditBatch(operationId); - InternalGetDataResult queryResult = internalGet(study.getUid(), panelList, INCLUDE_PANEL_IDS, user, ignoreException); + InternalGetDataResult queryResult = internalGet(organizationId, study.getUid(), panelList, INCLUDE_PANEL_IDS, userId, + ignoreException); if (queryResult.getMissing() != null) { missingMap = queryResult.getMissing().stream() @@ -899,11 +948,11 @@ public OpenCGAResult> getAcls(String studyId, Lis List panelUids = queryResult.getResults().stream().map(Panel::getUid).collect(Collectors.toList()); if (CollectionUtils.isNotEmpty(members)) { - panelAcls = authorizationManager.getAcl(user, study.getUid(), panelUids, members, Enums.Resource.DISEASE_PANEL, - PanelPermissions.class); + panelAcls = authorizationManager.getAcl(organizationId, study.getUid(), panelUids, members, Enums.Resource.DISEASE_PANEL, + PanelPermissions.class, userId); } else { - panelAcls = authorizationManager.getAcl(user, study.getUid(), panelUids, Enums.Resource.DISEASE_PANEL, - PanelPermissions.class); + panelAcls = authorizationManager.getAcl(organizationId, study.getUid(), panelUids, Enums.Resource.DISEASE_PANEL, + PanelPermissions.class, userId); } // Include non-existing panels to the result list @@ -914,15 +963,15 @@ public OpenCGAResult> getAcls(String studyId, Lis if (!missingMap.containsKey(panelId)) { Panel panel = queryResult.getResults().get(counter); resultList.add(panelAcls.getResults().get(counter)); - auditManager.audit(operationId, user, Enums.Action.FETCH_ACLS, Enums.Resource.DISEASE_PANEL, panel.getId(), - panel.getUuid(), study.getId(), study.getUuid(), auditParams, + auditManager.audit(organizationId, operationId, userId, Enums.Action.FETCH_ACLS, Enums.Resource.DISEASE_PANEL, + panel.getId(), panel.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS), new ObjectMap()); counter++; } else { resultList.add(new AclEntryList<>()); eventList.add(new Event(Event.Type.ERROR, panelId, missingMap.get(panelId).getErrorMsg())); - auditManager.audit(operationId, user, Enums.Action.FETCH_ACLS, Enums.Resource.DISEASE_PANEL, panelId, "", - study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, + auditManager.audit(organizationId, operationId, userId, Enums.Action.FETCH_ACLS, Enums.Resource.DISEASE_PANEL, panelId, + "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(0, "", missingMap.get(panelId).getErrorMsg())), new ObjectMap()); } } @@ -933,7 +982,7 @@ public OpenCGAResult> getAcls(String studyId, Lis panelAcls.setEvents(eventList); } catch (CatalogException e) { for (String panelId : panelList) { - auditManager.audit(operationId, user, Enums.Action.FETCH_ACLS, Enums.Resource.DISEASE_PANEL, panelId, "", + auditManager.audit(organizationId, operationId, userId, Enums.Action.FETCH_ACLS, Enums.Resource.DISEASE_PANEL, panelId, "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError()), new ObjectMap()); } @@ -946,18 +995,20 @@ public OpenCGAResult> getAcls(String studyId, Lis } } } finally { - auditManager.finishAuditBatch(operationId); + auditManager.finishAuditBatch(organizationId, operationId); } return panelAcls; } - public OpenCGAResult> updateAcl(String studyId, List panelStrList, - String memberList, AclParams aclParams, - ParamUtils.AclAction action, String token) + public OpenCGAResult> updateAcl(String studyId, List panelStrList, String memberList, + AclParams aclParams, ParamUtils.AclAction action, String token) throws CatalogException { - String user = userManager.getUserId(token); - Study study = studyManager.resolveId(studyId, user); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyId, userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) @@ -985,8 +1036,9 @@ public OpenCGAResult> updateAcl(String studyId, L checkPermissions(permissions, PanelPermissions::valueOf); } - OpenCGAResult panelDataResult = internalGet(study.getUid(), panelStrList, INCLUDE_PANEL_IDS, user, false); - authorizationManager.checkCanAssignOrSeePermissions(study.getUid(), user); + OpenCGAResult panelDataResult = internalGet(organizationId, study.getUid(), panelStrList, INCLUDE_PANEL_IDS, userId, + false); + authorizationManager.checkCanAssignOrSeePermissions(organizationId, study.getUid(), userId); // Validate that the members are actually valid members List members; @@ -996,7 +1048,7 @@ public OpenCGAResult> updateAcl(String studyId, L members = Collections.emptyList(); } authorizationManager.checkNotAssigningPermissionsToAdminsGroup(members); - checkMembers(study.getUid(), members); + checkMembers(organizationId, study.getUid(), members); List panelUids = panelDataResult.getResults().stream().map(Panel::getUid).collect(Collectors.toList()); AuthorizationManager.CatalogAclParams catalogAclParams = new AuthorizationManager.CatalogAclParams(panelUids, permissions, @@ -1004,43 +1056,43 @@ public OpenCGAResult> updateAcl(String studyId, L switch (action) { case SET: - authorizationManager.setAcls(study.getUid(), members, catalogAclParams); + authorizationManager.setAcls(organizationId, study.getUid(), members, catalogAclParams); break; case ADD: - authorizationManager.addAcls(study.getUid(), members, catalogAclParams); + authorizationManager.addAcls(organizationId, study.getUid(), members, catalogAclParams); break; case REMOVE: - authorizationManager.removeAcls(members, catalogAclParams); + authorizationManager.removeAcls(organizationId, members, catalogAclParams); break; case RESET: catalogAclParams.setPermissions(null); - authorizationManager.removeAcls(members, catalogAclParams); + authorizationManager.removeAcls(organizationId, members, catalogAclParams); break; default: throw new CatalogException("Unexpected error occurred. No valid action found."); } - OpenCGAResult> queryResultList = authorizationManager.getAcls(study.getUid(), + OpenCGAResult> queryResultList = authorizationManager.getAcls(organizationId, study.getUid(), panelUids, members, Enums.Resource.DISEASE_PANEL, PanelPermissions.class); for (int i = 0; i < queryResultList.getResults().size(); i++) { queryResultList.getResults().get(i).setId(panelDataResult.getResults().get(i).getId()); } for (Panel panel : panelDataResult.getResults()) { - auditManager.audit(operationId, user, Enums.Action.UPDATE_ACLS, Enums.Resource.DISEASE_PANEL, panel.getId(), - panel.getUuid(), study.getId(), study.getUuid(), auditParams, + auditManager.audit(organizationId, operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.DISEASE_PANEL, + panel.getId(), panel.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS), new ObjectMap()); } return queryResultList; } catch (CatalogException e) { if (panelStrList != null) { for (String panelId : panelStrList) { - auditManager.audit(operationId, user, Enums.Action.UPDATE_ACLS, Enums.Resource.DISEASE_PANEL, panelId, "", - study.getId(), study.getUuid(), auditParams, + auditManager.audit(organizationId, operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.DISEASE_PANEL, panelId, + "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError()), new ObjectMap()); } } throw e; } finally { - auditManager.finishAuditBatch(operationId); + auditManager.finishAuditBatch(organizationId, operationId); } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ProjectManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ProjectManager.java index ae99b2e10ee..92591852817 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ProjectManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ProjectManager.java @@ -22,20 +22,18 @@ import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.commons.datastore.core.result.Error; import org.opencb.commons.utils.ListUtils; import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager; import org.opencb.opencga.catalog.db.DBAdaptorFactory; import org.opencb.opencga.catalog.db.api.*; import org.opencb.opencga.catalog.exceptions.*; import org.opencb.opencga.catalog.io.CatalogIOManager; -import org.opencb.opencga.catalog.utils.Constants; -import org.opencb.opencga.catalog.utils.ParamUtils; -import org.opencb.opencga.catalog.utils.UuidUtils; +import org.opencb.opencga.catalog.utils.*; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.cellbase.CellBaseValidator; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.config.storage.CellBaseConfiguration; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.audit.AuditRecord; import org.opencb.opencga.core.models.cohort.Cohort; import org.opencb.opencga.core.models.common.Enums; @@ -43,7 +41,6 @@ import org.opencb.opencga.core.models.project.*; import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.models.user.Account; import org.opencb.opencga.core.models.user.User; import org.opencb.opencga.core.response.OpenCGAResult; @@ -78,6 +75,9 @@ public class ProjectManager extends AbstractManager { ProjectDBAdaptor.QueryParams.INTERNAL_DATASTORES_VARIANT.key(), ProjectDBAdaptor.QueryParams.CELLBASE.key() )); + public static final QueryOptions INCLUDE_PROJECT_IDS = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList( + ProjectDBAdaptor.QueryParams.UID.key(), ProjectDBAdaptor.QueryParams.ID.key(), ProjectDBAdaptor.QueryParams.UUID.key(), + ProjectDBAdaptor.QueryParams.FQN.key())); ProjectManager(AuthorizationManager authorizationManager, AuditManager auditManager, CatalogManager catalogManager, DBAdaptorFactory catalogDBAdaptorFactory, CatalogIOManager catalogIOManager, Configuration configuration) { @@ -85,131 +85,61 @@ public class ProjectManager extends AbstractManager { this.catalogIOManager = catalogIOManager; } - public String getOwner(long projectId) throws CatalogException { - return projectDBAdaptor.getOwnerId(projectId); - } - /** * Fetch the project qualifying the projectStr structure as long as userId has permissions to see it. * - * @param projectStr string that can contain the full qualified name (owner@projectId) or just the projectId. - * @param userId user asking for the project information. - * @return a OpenCGAResult containing the project. + * @param catalogFqn CatalogFqn object containing the full qualified name (org@projectId) or just the projectId. + * @param options QueryOptions object containing the fields to fetch. + * @param payload Payload containing the userId and organizationId. + * @return the requested project object. * @throws CatalogException if multiple projects are found. */ - Project resolveId(String projectStr, String userId) throws CatalogException { - if (StringUtils.isEmpty(userId)) { - throw new CatalogException("Missing mandatory parameter userId"); - } + OpenCGAResult resolveId(CatalogFqn catalogFqn, QueryOptions options, JwtPayload payload) throws CatalogException { + String userId = payload.getUserId(catalogFqn.getOrganizationId()); - String auxProject = ""; - String auxOwner = ""; - boolean isUuid = false; - - if (StringUtils.isNotEmpty(projectStr)) { - if (UuidUtils.isOpenCgaUuid(projectStr)) { - isUuid = true; - } else { - String[] split = projectStr.split("@"); - if (split.length == 1) { - auxProject = projectStr; - } else if (split.length == 2) { - auxOwner = split[0]; - auxProject = split[1]; - } else { - throw new CatalogException(projectStr + " does not follow the expected pattern [ownerId@projectId]"); - } - } - } + ParamUtils.checkParameter(userId, "userId"); + ParamUtils.checkParameter(catalogFqn.getOrganizationId(), "organizationId"); - QueryOptions projectOptions = new QueryOptions() - .append(QueryOptions.INCLUDE, Arrays.asList( - ProjectDBAdaptor.QueryParams.UID.key(), ProjectDBAdaptor.QueryParams.UUID.key(), - ProjectDBAdaptor.QueryParams.ID.key(), ProjectDBAdaptor.QueryParams.FQN.key(), - ProjectDBAdaptor.QueryParams.CURRENT_RELEASE.key())); - - if (StringUtils.isEmpty(auxOwner) || auxOwner.equals(userId)) { - // We look for own projects - Query query = new Query(ProjectDBAdaptor.QueryParams.USER_ID.key(), userId); - if (isUuid) { - query.putIfNotEmpty(ProjectDBAdaptor.QueryParams.UUID.key(), projectStr); - } else { - query.putIfNotEmpty(ProjectDBAdaptor.QueryParams.ID.key(), auxProject); - } - - OpenCGAResult projectDataResult = projectDBAdaptor.get(query, projectOptions); - if (projectDataResult.getNumResults() > 1) { - throw new CatalogException("Please be more concrete with the project. More than one project found for " + userId + " user"); - } else if (projectDataResult.getNumResults() == 1) { - return projectDataResult.first(); - } + QueryOptions queryOptions; + if (options != null) { + queryOptions = new QueryOptions(options); + } else { + queryOptions = new QueryOptions(); } + queryOptions = keepFieldsInQueryOptions(queryOptions, Arrays.asList( + ProjectDBAdaptor.QueryParams.UID.key(), ProjectDBAdaptor.QueryParams.UUID.key(), + ProjectDBAdaptor.QueryParams.ID.key(), ProjectDBAdaptor.QueryParams.FQN.key(), + ProjectDBAdaptor.QueryParams.CURRENT_RELEASE.key())); - // Look for shared projects. First, we will check all the studies in which the user is a member Query query = new Query(); - query.putIfNotEmpty(StudyDBAdaptor.QueryParams.OWNER.key(), auxOwner); - if (isUuid) { - query.putIfNotEmpty(StudyDBAdaptor.QueryParams.PROJECT_UUID.key(), projectStr); + if (StringUtils.isNotEmpty(catalogFqn.getProjectUuid())) { + query.putIfNotEmpty(ProjectDBAdaptor.QueryParams.UUID.key(), catalogFqn.getProjectUuid()); + } else if (StringUtils.isNotEmpty(catalogFqn.getProjectId())) { + query.putIfNotEmpty(ProjectDBAdaptor.QueryParams.ID.key(), catalogFqn.getProjectId()); } else { - query.putIfNotEmpty(StudyDBAdaptor.QueryParams.PROJECT_ID.key(), auxProject); + throw new CatalogException("Internal error. Missing project id or uuid."); } - QueryOptions queryOptions = new QueryOptions(QueryOptions.INCLUDE, StudyDBAdaptor.QueryParams.FQN.key()); - OpenCGAResult studyDataResult = studyDBAdaptor.get(query, queryOptions, userId); - - if (studyDataResult.getNumResults() > 0) { - Set projectFqnSet = new HashSet<>(); - for (Study study : studyDataResult.getResults()) { - projectFqnSet.add(StringUtils.split(study.getFqn(), ":")[0]); - } - - if (projectFqnSet.size() == 1) { - query = new Query(ProjectDBAdaptor.QueryParams.FQN.key(), projectFqnSet); - return projectDBAdaptor.get(query, projectOptions).first(); - } else { - throw new CatalogException("More than one project shared with user " + userId + ". Please, be more specific. " - + "The accepted pattern is [ownerId@projectId]"); - } + OpenCGAResult projectDataResult = getProjectDBAdaptor(catalogFqn.getOrganizationId()).get(query, queryOptions, userId); + if (projectDataResult.getNumResults() > 1) { + throw new CatalogException("Please be more concrete with the project. More than one project found for " + userId + " user"); + } else if (projectDataResult.getNumResults() == 1) { + return projectDataResult; } else { - if (StringUtils.isNotEmpty(projectStr)) { - // Check if it is a matter of permissions - if (studyDBAdaptor.count(query).getNumMatches() == 0) { - throw new CatalogException("Project " + projectStr + " not found"); - } else { - throw CatalogAuthorizationException.deny(userId, "view", "project", projectStr, null); - } + projectDataResult = getProjectDBAdaptor(catalogFqn.getOrganizationId()).get(query, queryOptions); + if (projectDataResult.getNumResults() == 0) { + throw new CatalogException("No project found given '" + catalogFqn.getProvidedId() + "'."); } else { - throw new CatalogException("No projects shared or owned by user " + userId); + throw CatalogAuthorizationException.denyAny(userId, "view", "project"); } } } - private OpenCGAResult getProject(String userId, String projectUuid, QueryOptions options) throws CatalogDBException { + private OpenCGAResult getProject(String organizationId, String projectUuid, QueryOptions options) throws CatalogDBException { Query query = new Query() - .append(ProjectDBAdaptor.QueryParams.USER_ID.key(), userId) .append(ProjectDBAdaptor.QueryParams.UUID.key(), projectUuid); options = ParamUtils.defaultObject(options, QueryOptions::new); - return projectDBAdaptor.get(query, options); - } - - /** - * Obtain the list of projects and studies that are shared with the user. - * - * @param userId user whose projects and studies are being shared with. - * @param queryOptions QueryOptions object. - * @param sessionId Session id which should correspond to userId. - * @return A OpenCGAResult object containing the list of projects and studies that are shared with the user. - * @throws CatalogException CatalogException - */ - public OpenCGAResult getSharedProjects(String userId, QueryOptions queryOptions, String sessionId) throws CatalogException { - OpenCGAResult result = search(new Query(ProjectDBAdaptor.QueryParams.USER_ID.key(), "!=" + userId), queryOptions, - sessionId); - for (Event event : result.getEvents()) { - if (event.getType() == Event.Type.ERROR) { - throw new CatalogAuthorizationException(event.getMessage()); - } - } - return result; + return getProjectDBAdaptor(organizationId).get(query, options); } @Deprecated @@ -222,11 +152,9 @@ public OpenCGAResult create(String id, String name, String description, public OpenCGAResult create(ProjectCreateParams projectCreateParams, QueryOptions options, String token) throws CatalogException { - //Only the user can create a project - String userId = this.catalogManager.getUserManager().getUserId(token); - if (userId.isEmpty()) { - throw new CatalogException("The token introduced does not correspond to any registered user."); - } + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + String organizationId = tokenPayload.getOrganization(); + String userId = tokenPayload.getUserId(); ObjectMap auditParams = new ObjectMap() .append("project", projectCreateParams) @@ -237,69 +165,44 @@ public OpenCGAResult create(ProjectCreateParams projectCreateParams, Qu OpenCGAResult queryResult; Project project; try { + authorizationManager.checkIsAtLeastOrganizationOwnerOrAdmin(organizationId, userId); ParamUtils.checkObj(projectCreateParams, "ProjectCreateParams"); - - // Check that the account type is not guest - OpenCGAResult user = userDBAdaptor.get(userId, QueryOptions.empty()); - if (user.getNumResults() == 0) { - throw new CatalogException("Internal error happened. Could not find user " + userId); - } - - if (Account.AccountType.FULL != user.first().getAccount().getType()) { - if (user.first().getAccount().getType() == Account.AccountType.ADMINISTRATOR) { - // Check it is the first project - if (user.first().getProjects() != null && !user.first().getProjects().isEmpty()) { - String errorMsg = "Cannot create more projects for ADMINISTRATOR user '" + user.first().getId() + "'."; - auditManager.auditCreate(userId, Enums.Resource.PROJECT, projectCreateParams.getId(), "", "", "", auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(0, "", errorMsg))); - throw new CatalogException(errorMsg); - } - } else { - - String errorMsg = "User " + userId + " is not authorized to create new projects. Only users with " - + Account.AccountType.FULL + " accounts are allowed to do so."; - auditManager.auditCreate(userId, Enums.Resource.PROJECT, projectCreateParams.getId(), "", "", "", auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(0, "", errorMsg))); - throw new CatalogException(errorMsg); - } - } - project = projectCreateParams.toProject(); - validateProjectForCreation(project, user.first()); + validateProjectForCreation(organizationId, project); - queryResult = projectDBAdaptor.insert(project, userId, options); - OpenCGAResult result = getProject(userId, project.getUuid(), options); + queryResult = getProjectDBAdaptor(organizationId).insert(project, options); + OpenCGAResult result = getProject(organizationId, project.getUuid(), options); project = result.first(); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { // Fetch created project queryResult.setResults(result.getResults()); } } catch (CatalogException e) { - auditManager.auditCreate(userId, Enums.Resource.PROJECT, projectCreateParams.getId(), "", "", "", auditParams, + auditManager.auditCreate(organizationId, userId, Enums.Resource.PROJECT, projectCreateParams.getId(), "", "", "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } try { - catalogIOManager.createProject(userId, Long.toString(project.getUid())); + catalogIOManager.createProject(organizationId, Long.toString(project.getUid())); } catch (CatalogIOException e) { - auditManager.auditCreate(userId, Enums.Resource.PROJECT, project.getId(), "", "", "", auditParams, + auditManager.auditCreate(organizationId, userId, Enums.Resource.PROJECT, project.getId(), "", "", "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); try { - projectDBAdaptor.delete(project); + getProjectDBAdaptor(organizationId).delete(project); } catch (Exception e1) { logger.error("Error deleting project from catalog after failing creating the folder in the filesystem", e1); throw e; } throw e; } - auditManager.auditCreate(userId, Enums.Resource.PROJECT, project.getId(), project.getUuid(), "", "", auditParams, + auditManager.auditCreate(organizationId, userId, Enums.Resource.PROJECT, project.getId(), project.getUuid(), "", "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return queryResult; } - private void validateProjectForCreation(Project project, User user) throws CatalogParameterException { + private void validateProjectForCreation(String organizationId, Project project) throws CatalogParameterException { ParamUtils.checkParameter(project.getId(), ProjectDBAdaptor.QueryParams.ID.key()); project.setName(ParamUtils.defaultString(project.getName(), project.getId())); project.setDescription(ParamUtils.defaultString(project.getDescription(), "")); @@ -310,34 +213,28 @@ private void validateProjectForCreation(Project project, User user) throws Catal project.setCurrentRelease(1); project.setInternal(ProjectInternal.init()); project.setAttributes(ParamUtils.defaultObject(project.getAttributes(), HashMap::new)); + project.setFqn(FqnUtils.buildFqn(organizationId, project.getId())); - if (user.getAccount().getType() == Account.AccountType.ADMINISTRATOR) { - // Do not validate organism nor CellBase for admin account - } else { - if (project.getOrganism() == null || StringUtils.isEmpty(project.getOrganism().getAssembly()) - || StringUtils.isEmpty(project.getOrganism().getScientificName())) { - throw new CatalogParameterException("Missing mandatory organism information"); - } - try { - //TODO: Should the datarelease be undefined? When undefined, it'd be read from cellbase meta endpoints. - String defaultDataRelease = project.getOrganism().getAssembly().equalsIgnoreCase("grch38") - ? ParamConstants.CELLBASE_DATA_RELEASE_GRCH38 - : null; - CellBaseConfiguration cellBaseConfiguration = ParamUtils.defaultObject(project.getCellbase(), - new CellBaseConfiguration(ParamConstants.CELLBASE_URL, ParamConstants.CELLBASE_VERSION, - defaultDataRelease, ParamConstants.CELLBASE_APIKEY)); - cellBaseConfiguration = CellBaseValidator.validate(cellBaseConfiguration, - project.getOrganism().getScientificName(), - project.getOrganism().getAssembly(), true); - project.setCellbase(cellBaseConfiguration); - } catch (IOException e) { - throw new CatalogParameterException(e); - } + if (project.getOrganism() == null || StringUtils.isEmpty(project.getOrganism().getAssembly()) + || StringUtils.isEmpty(project.getOrganism().getScientificName())) { + throw new CatalogParameterException("Missing mandatory organism information"); + } + try { + //TODO: Should the datarelease be undefined? When undefined, it'd be read from cellbase meta endpoints. + String defaultDataRelease = project.getOrganism().getAssembly().equalsIgnoreCase("grch38") + ? ParamConstants.CELLBASE_DATA_RELEASE_GRCH38 + : null; + CellBaseConfiguration cellBaseConfiguration = ParamUtils.defaultObject(project.getCellbase(), + new CellBaseConfiguration(ParamConstants.CELLBASE_URL, ParamConstants.CELLBASE_VERSION, + defaultDataRelease, ParamConstants.CELLBASE_APIKEY)); + cellBaseConfiguration = CellBaseValidator.validate(cellBaseConfiguration, + project.getOrganism().getScientificName(), + project.getOrganism().getAssembly(), true); + project.setCellbase(cellBaseConfiguration); + } catch (IOException e) { + throw new CatalogParameterException(e); } - - project.setUuid(UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.PROJECT)); - if (project.getStudies() != null && !project.getStudies().isEmpty()) { throw new CatalogParameterException("Creating project and studies in a single transaction is forbidden"); } @@ -353,33 +250,36 @@ private void validateProjectForCreation(Project project, User user) throws Catal * @throws CatalogException CatalogException */ public OpenCGAResult get(String projectId, QueryOptions options, String token) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(token); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn projectFqn = CatalogFqn.extractFqnFromProject(projectId, tokenPayload); + String organizationId = projectFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); ObjectMap auditParams = new ObjectMap() + .append("organizationId", organizationId) .append("projectId", projectId) .append("options", options) .append("token", token); try { - Project project = resolveId(projectId, userId); - OpenCGAResult queryResult = projectDBAdaptor.get(project.getUid(), options); - auditManager.auditInfo(userId, Enums.Resource.PROJECT, project.getId(), project.getUuid(), "", "", auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + OpenCGAResult queryResult = resolveId(projectFqn, options, tokenPayload); + auditManager.auditInfo(organizationId, userId, Enums.Resource.PROJECT, queryResult.first().getId(), + queryResult.first().getUuid(), "", "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return queryResult; } catch (CatalogException e) { - auditManager.auditInfo(userId, Enums.Resource.PROJECT, projectId, "", "", "", auditParams, + auditManager.auditInfo(organizationId, userId, Enums.Resource.PROJECT, projectId, "", "", "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } - public OpenCGAResult get(List projectList, QueryOptions options, boolean ignoreException, String sessionId) + public OpenCGAResult get(List projectList, QueryOptions options, boolean ignoreException, String token) throws CatalogException { OpenCGAResult result = OpenCGAResult.empty(); for (int i = 0; i < projectList.size(); i++) { String project = projectList.get(i); try { - OpenCGAResult projectResult = get(project, options, sessionId); + OpenCGAResult projectResult = get(project, options, token); result.append(projectResult); } catch (CatalogException e) { Event event = new Event(Event.Type.ERROR, projectList.get(i), e.getMessage()); @@ -399,40 +299,52 @@ public OpenCGAResult get(List projectList, QueryOptions options /** * Fetch all the project objects matching the query. * - * @param query Query to catalog. - * @param options Query options, like "include", "exclude", "limit" and "skip" - * @param token sessionId + * @param organizationId Organization id. + * @param query Query to catalog. + * @param options Query options, like "include", "exclude", "limit" and "skip" + * @param token sessionId * @return All matching elements. * @throws CatalogException CatalogException */ - public OpenCGAResult search(Query query, QueryOptions options, String token) throws CatalogException { + public OpenCGAResult search(String organizationId, Query query, QueryOptions options, String token) throws CatalogException { query = ParamUtils.defaultObject(query, Query::new); options = ParamUtils.defaultObject(options, QueryOptions::new); - String userId = catalogManager.getUserManager().getUserId(token); - query = new Query(query); - ObjectMap auditParams = new ObjectMap() + .append("organizationId", organizationId) .append("query", query) .append("options", options) .append("token", token); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + if (StringUtils.isEmpty(organizationId)) { + organizationId = tokenPayload.getOrganization(); + } + String userId = tokenPayload.getUserId(organizationId); + query = new Query(query); + + try { fixQueryObject(query); // If study is provided, we need to check if it will be study alias or id if (StringUtils.isNotEmpty(query.getString(ProjectDBAdaptor.QueryParams.STUDY.key()))) { List studies = catalogManager.getStudyManager() - .resolveIds(query.getAsStringList(ProjectDBAdaptor.QueryParams.STUDY.key()), userId); + .resolveIds(query.getAsStringList(ProjectDBAdaptor.QueryParams.STUDY.key()), userId, organizationId); query.remove(ProjectDBAdaptor.QueryParams.STUDY.key()); query.put(ProjectDBAdaptor.QueryParams.STUDY_UID.key(), studies.stream().map(Study::getUid).collect(Collectors.toList())); } - OpenCGAResult queryResult = projectDBAdaptor.get(query, options, userId); - auditManager.auditSearch(userId, Enums.Resource.PROJECT, "", "", auditParams, + OpenCGAResult queryResult = getProjectDBAdaptor(organizationId).get(query, options, userId); + auditManager.auditSearch(organizationId, userId, Enums.Resource.PROJECT, "", "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return queryResult; } catch (CatalogException e) { - auditManager.auditSearch(userId, Enums.Resource.PROJECT, "", "", auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + try { + auditManager.auditSearch(organizationId, userId, Enums.Resource.PROJECT, "", "", auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + } catch (RuntimeException ex) { + logger.error("Could not generate audit log", ex); + e.addSuppressed(ex); + } throw e; } @@ -465,11 +377,14 @@ public OpenCGAResult update(String projectId, ObjectMap parameters, Que * @throws CatalogException CatalogException */ private OpenCGAResult update(String projectId, ObjectMap parameters, QueryOptions options, boolean allowProtectedUpdates, - String token) - throws CatalogException { - String userId = this.catalogManager.getUserManager().getUserId(token); + String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn catalogFqn = CatalogFqn.extractFqnFromProject(projectId, tokenPayload); + String organizationId = catalogFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); ObjectMap auditParams = new ObjectMap() + .append("organizationId", organizationId) .append("project", projectId) .append("updateParams", parameters) .append("options", options) @@ -477,9 +392,9 @@ private OpenCGAResult update(String projectId, ObjectMap parameters, Qu Project project; try { - project = resolveId(projectId, userId); + project = resolveId(catalogFqn, options, tokenPayload).first(); } catch (CatalogException e) { - auditManager.auditUpdate(userId, Enums.Resource.PROJECT, projectId, "", "", "", auditParams, + auditManager.auditUpdate(organizationId, userId, Enums.Resource.PROJECT, projectId, "", "", "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -490,7 +405,7 @@ private OpenCGAResult update(String projectId, ObjectMap parameters, Qu options = ParamUtils.defaultObject(options, QueryOptions::new); long projectUid = project.getUid(); - authorizationManager.checkCanEditProject(projectUid, userId); + authorizationManager.checkCanEditProject(organizationId, projectUid, userId); for (String s : parameters.keySet()) { if (UPDATABLE_FIELDS.contains(s)) { @@ -517,7 +432,7 @@ private OpenCGAResult update(String projectId, ObjectMap parameters, Qu if (parameters.containsKey(ProjectDBAdaptor.QueryParams.ORGANISM_SCIENTIFIC_NAME.key()) || parameters.containsKey(ProjectDBAdaptor.QueryParams.ORGANISM_COMMON_NAME.key()) || parameters.containsKey(ProjectDBAdaptor.QueryParams.ORGANISM_ASSEMBLY.key())) { - OpenCGAResult projectQR = projectDBAdaptor + OpenCGAResult projectQR = getProjectDBAdaptor(organizationId) .get(projectUid, new QueryOptions(QueryOptions.INCLUDE, ProjectDBAdaptor.QueryParams.ORGANISM.key())); if (projectQR.getNumResults() == 0) { throw new CatalogException("Project " + projectUid + " not found"); @@ -544,21 +459,21 @@ private OpenCGAResult update(String projectId, ObjectMap parameters, Qu ParamUtils.checkIdentifier(parameters.getString(ProjectDBAdaptor.QueryParams.ID.key()), "id"); } - OpenCGAResult update = projectDBAdaptor.update(projectUid, parameters, QueryOptions.empty()); - auditManager.auditUpdate(userId, Enums.Resource.PROJECT, project.getId(), project.getUuid(), "", "", auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + OpenCGAResult update = getProjectDBAdaptor(organizationId).update(projectUid, parameters, QueryOptions.empty()); + auditManager.auditUpdate(organizationId, userId, Enums.Resource.PROJECT, project.getId(), project.getUuid(), "", "", + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { // Fetch updated project - OpenCGAResult result = projectDBAdaptor.get(new Query(ProjectDBAdaptor.QueryParams.UID.key(), projectUid), options, - userId); + OpenCGAResult result = getProjectDBAdaptor(organizationId) + .get(new Query(ProjectDBAdaptor.QueryParams.UID.key(), projectUid), options, userId); update.setResults(result.getResults()); } return update; } catch (CatalogException e) { - auditManager.auditUpdate(userId, Enums.Resource.PROJECT, project.getId(), project.getUuid(), "", "", auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditUpdate(organizationId, userId, Enums.Resource.PROJECT, project.getId(), project.getUuid(), "", "", + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } @@ -569,8 +484,7 @@ public OpenCGAResult setDatastoreVariant(String projectStr, DataStore d } public OpenCGAResult setCellbaseConfiguration(String projectStr, CellBaseConfiguration configuration, boolean validate, - String token) - throws CatalogException { + String token) throws CatalogException { if (validate) { try { ProjectOrganism organism = get(projectStr, @@ -584,62 +498,47 @@ public OpenCGAResult setCellbaseConfiguration(String projectStr, CellBa new ObjectMap(ProjectDBAdaptor.QueryParams.CELLBASE.key(), configuration), new QueryOptions(), true, token); } - public Map facet(String projectStr, String fileFields, String sampleFields, String individualFields, - String cohortFields, String familyFields, String jobFields, boolean defaultStats, String sessionId) - throws CatalogException, IOException { - String userId = catalogManager.getUserManager().getUserId(sessionId); - Project project = resolveId(projectStr, userId); - Query query = new Query(StudyDBAdaptor.QueryParams.PROJECT_UID.key(), project.getUid()); - OpenCGAResult studyDataResult = catalogManager.getStudyManager().search(query, new QueryOptions(QueryOptions.INCLUDE, - Arrays.asList(StudyDBAdaptor.QueryParams.FQN.key(), StudyDBAdaptor.QueryParams.ID.key())), sessionId); - - Map result = new HashMap<>(); - for (Study study : studyDataResult.getResults()) { - result.put(study.getId(), catalogManager.getStudyManager().facet(study.getFqn(), fileFields, sampleFields, individualFields, - cohortFields, familyFields, jobFields, defaultStats, sessionId)); - } - - return result; - } - - public OpenCGAResult incrementRelease(String projectStr, String sessionId) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(sessionId); + public OpenCGAResult incrementRelease(String projectStr, String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn catalogFqn = CatalogFqn.extractFqnFromProject(projectStr, tokenPayload); + String organizationId = catalogFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); try { - Project project = resolveId(projectStr, userId); + Project project = resolveId(catalogFqn, null, tokenPayload).first(); long projectUid = project.getUid(); - authorizationManager.checkCanEditProject(projectUid, userId); + authorizationManager.checkCanEditProject(organizationId, projectUid, userId); // Obtain the current release number int currentRelease = project.getCurrentRelease(); // Check current release has been used at least in one study or file or cohort or individual... QueryOptions studyOptions = keepFieldInQueryOptions(StudyManager.INCLUDE_STUDY_IDS, StudyDBAdaptor.QueryParams.RELEASE.key()); - OpenCGAResult studyResult = studyDBAdaptor.getAllStudiesInProject(projectUid, studyOptions); + OpenCGAResult studyResult = getStudyDBAdaptor(organizationId).getAllStudiesInProject(projectUid, studyOptions); List allStudiesInProject = studyResult.getResults(); if (allStudiesInProject.isEmpty()) { throw new CatalogException("Cannot increment current release number. No studies found for release " + currentRelease); } - if (checkCurrentReleaseInUse(allStudiesInProject, currentRelease)) { + if (checkCurrentReleaseInUse(organizationId, allStudiesInProject, currentRelease)) { // Increment current project release - OpenCGAResult writeResult = projectDBAdaptor.incrementCurrentRelease(projectUid); - OpenCGAResult projectDataResult = projectDBAdaptor.get(projectUid, + OpenCGAResult writeResult = getProjectDBAdaptor(organizationId).incrementCurrentRelease(projectUid); + OpenCGAResult projectDataResult = getProjectDBAdaptor(organizationId).get(projectUid, new QueryOptions(QueryOptions.INCLUDE, ProjectDBAdaptor.QueryParams.CURRENT_RELEASE.key())); OpenCGAResult queryResult = new OpenCGAResult<>(projectDataResult.getTime() + writeResult.getTime(), Collections.emptyList(), 1, Collections.singletonList(projectDataResult.first().getCurrentRelease()), 1); // Upgrade release in every versioned data model for (Study study : allStudiesInProject) { - sampleDBAdaptor.updateProjectRelease(study.getUid(), queryResult.first()); - individualDBAdaptor.updateProjectRelease(study.getUid(), queryResult.first()); - familyDBAdaptor.updateProjectRelease(study.getUid(), queryResult.first()); - panelDBAdaptor.updateProjectRelease(study.getUid(), queryResult.first()); - interpretationDBAdaptor.updateProjectRelease(study.getUid(), queryResult.first()); + getSampleDBAdaptor(organizationId).updateProjectRelease(study.getUid(), queryResult.first()); + getIndividualDBAdaptor(organizationId).updateProjectRelease(study.getUid(), queryResult.first()); + getFamilyDBAdaptor(organizationId).updateProjectRelease(study.getUid(), queryResult.first()); + getPanelDBAdaptor(organizationId).updateProjectRelease(study.getUid(), queryResult.first()); + getInterpretationDBAdaptor(organizationId).updateProjectRelease(study.getUid(), queryResult.first()); } - auditManager.audit(userId, Enums.Action.INCREMENT_PROJECT_RELEASE, Enums.Resource.PROJECT, project.getId(), + auditManager.audit(organizationId, userId, Enums.Action.INCREMENT_PROJECT_RELEASE, Enums.Resource.PROJECT, project.getId(), project.getUuid(), "", "", new ObjectMap(), new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return queryResult; @@ -648,20 +547,22 @@ public OpenCGAResult incrementRelease(String projectStr, String session + " has not yet been used in any entry"); } } catch (CatalogException e) { - auditManager.audit(userId, Enums.Action.INCREMENT_PROJECT_RELEASE, Enums.Resource.PROJECT, projectStr, "", "", "", - new ObjectMap(), new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.audit(organizationId, userId, Enums.Action.INCREMENT_PROJECT_RELEASE, Enums.Resource.PROJECT, projectStr, "", "", + "", new ObjectMap(), new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } - public void importReleases(String owner, String inputDirStr, String sessionId) throws CatalogException, IOException { - String userId = catalogManager.getUserManager().getUserId(sessionId); - if (!authorizationManager.isInstallationAdministrator(userId)) { - throw new CatalogAuthorizationException("Only admin of OpenCGA is authorised to import data"); + public void importReleases(String organizationId, String owner, String inputDirStr, String token) + throws CatalogException, IOException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + String userId = tokenPayload.getUserId(organizationId); + if (!authorizationManager.isAtLeastOrganizationOwnerOrAdmin(organizationId, userId)) { + throw CatalogAuthorizationException.notOrganizationOwnerOrAdmin("import data"); } - OpenCGAResult userDataResult = userDBAdaptor.get(owner, new QueryOptions(QueryOptions.INCLUDE, - Arrays.asList(UserDBAdaptor.QueryParams.ACCOUNT.key(), UserDBAdaptor.QueryParams.PROJECTS.key()))); + OpenCGAResult userDataResult = getUserDBAdaptor(organizationId).get(owner, new QueryOptions(QueryOptions.INCLUDE, + Collections.singletonList(UserDBAdaptor.QueryParams.INTERNAL_ACCOUNT.key()))); if (userDataResult.getNumResults() == 0) { throw new CatalogException("User " + owner + " not found"); } @@ -690,19 +591,19 @@ public void importReleases(String owner, String inputDirStr, String sessionId) t project.put(ProjectDBAdaptor.QueryParams.UID.key(), ParamUtils.getAsLong(project.get(ProjectDBAdaptor.QueryParams.UID.key()))); // Check the projectId - if (projectDBAdaptor.exists((Long) project.get(ProjectDBAdaptor.QueryParams.UID.key()))) { + if (getProjectDBAdaptor(organizationId).exists((Long) project.get(ProjectDBAdaptor.QueryParams.UID.key()))) { throw new CatalogException("The database is not empty. Project " + project.get(ProjectDBAdaptor.QueryParams.NAME.key()) + " already exists"); } logger.info("Importing projects..."); - projectDBAdaptor.nativeInsert(project, owner); + getProjectDBAdaptor(organizationId).nativeInsert(project, owner); // Reading studies try (BufferedReader br = new BufferedReader(new FileReader(inputDir.resolve("studies.json").toFile()))) { logger.info("Importing studies..."); for (String line = br.readLine(); line != null; line = br.readLine()) { Map study = objectMapper.readValue(line, Map.class); - studyDBAdaptor.nativeInsert(study, owner); + getStudyDBAdaptor(organizationId).nativeInsert(study); } } @@ -711,7 +612,7 @@ public void importReleases(String owner, String inputDirStr, String sessionId) t logger.info("Importing files..."); for (String line = br.readLine(); line != null; line = br.readLine()) { Map file = objectMapper.readValue(line, Map.class); - fileDBAdaptor.nativeInsert(file, owner); + getFileDBAdaptor(organizationId).nativeInsert(file, owner); } } @@ -720,7 +621,7 @@ public void importReleases(String owner, String inputDirStr, String sessionId) t logger.info("Importing samples..."); for (String line = br.readLine(); line != null; line = br.readLine()) { Map sample = objectMapper.readValue(line, Map.class); - sampleDBAdaptor.nativeInsert(sample, owner); + getSampleDBAdaptor(organizationId).nativeInsert(sample, owner); } } @@ -729,7 +630,7 @@ public void importReleases(String owner, String inputDirStr, String sessionId) t logger.info("Importing individuals..."); for (String line = br.readLine(); line != null; line = br.readLine()) { Map file = objectMapper.readValue(line, Map.class); - individualDBAdaptor.nativeInsert(file, owner); + getIndividualDBAdaptor(organizationId).nativeInsert(file, owner); } } @@ -738,7 +639,7 @@ public void importReleases(String owner, String inputDirStr, String sessionId) t logger.info("Importing families..."); for (String line = br.readLine(); line != null; line = br.readLine()) { Map file = objectMapper.readValue(line, Map.class); - familyDBAdaptor.nativeInsert(file, owner); + getFamilyDBAdaptor(organizationId).nativeInsert(file, owner); } } @@ -747,7 +648,7 @@ public void importReleases(String owner, String inputDirStr, String sessionId) t logger.info("Importing clinical analysis..."); for (String line = br.readLine(); line != null; line = br.readLine()) { Map file = objectMapper.readValue(line, Map.class); - clinicalDBAdaptor.nativeInsert(file, owner); + getClinicalAnalysisDBAdaptor(organizationId).nativeInsert(file, owner); } } @@ -756,7 +657,7 @@ public void importReleases(String owner, String inputDirStr, String sessionId) t logger.info("Importing cohorts..."); for (String line = br.readLine(); line != null; line = br.readLine()) { Map file = objectMapper.readValue(line, Map.class); - cohortDBAdaptor.nativeInsert(file, owner); + getCohortDBAdaptor(organizationId).nativeInsert(file, owner); } } @@ -765,15 +666,18 @@ public void importReleases(String owner, String inputDirStr, String sessionId) t logger.info("Importing jobs..."); for (String line = br.readLine(); line != null; line = br.readLine()) { Map file = objectMapper.readValue(line, Map.class); - jobDBAdaptor.nativeInsert(file, owner); + getJobDBAdaptor(organizationId).nativeInsert(file, owner); } } } public void exportByFileNames(String studyStr, File outputDir, File filePath, String token) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(token); - if (!authorizationManager.isInstallationAdministrator(userId)) { - throw new CatalogAuthorizationException("Only admin of OpenCGA is authorised to export data"); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + if (!authorizationManager.isAtLeastOrganizationOwnerOrAdmin(organizationId, userId)) { + throw CatalogAuthorizationException.notOrganizationOwnerOrAdmin("export data"); } if (!filePath.exists() || !filePath.isFile()) { @@ -801,7 +705,7 @@ public void exportByFileNames(String studyStr, File outputDir, File filePath, St String owner = studyDataResult.first().getFqn().split("@")[0]; - String ownerToken = catalogManager.getUserManager().getNonExpiringToken(owner, Collections.emptyMap(), token); + String ownerToken = catalogManager.getUserManager().getNonExpiringToken(organizationId, owner, Collections.emptyMap(), token); try (BufferedReader buf = new BufferedReader(new FileReader(filePath))) { @@ -954,10 +858,13 @@ private void exportToFile(List dataList, File file, ObjectMapper objectM } } - public void exportReleases(String projectStr, int release, String outputDirStr, String sessionId) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(sessionId); - if (!authorizationManager.isInstallationAdministrator(userId)) { - throw new CatalogAuthorizationException("Only admin of OpenCGA is authorised to export data"); + public void exportReleases(String projectStr, int release, String outputDirStr, String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn catalogFqn = CatalogFqn.extractFqnFromProject(projectStr, tokenPayload); + String organizationId = catalogFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + if (!authorizationManager.isAtLeastOrganizationOwnerOrAdmin(organizationId, userId)) { + throw CatalogAuthorizationException.notOrganizationOwnerOrAdmin("export data"); } Path outputDir = Paths.get(outputDirStr); @@ -981,9 +888,8 @@ public void exportReleases(String projectStr, int release, String outputDirStr, .append(QueryOptions.INCLUDE, Arrays.asList( ProjectDBAdaptor.QueryParams.UID.key(), ProjectDBAdaptor.QueryParams.CURRENT_RELEASE.key() - )) - .append(QueryOptions.EXCLUDE, "studies"); - Project project = resolveId(projectStr, userId); + )); + Project project = resolveId(catalogFqn, options, tokenPayload).first(); long projectId = project.getUid(); int currentRelease = project.getCurrentRelease(); @@ -993,20 +899,20 @@ public void exportReleases(String projectStr, int release, String outputDirStr, ObjectMapper objectMapper = getDefaultObjectMapper(); Query query = new Query(ProjectDBAdaptor.QueryParams.UID.key(), projectId); - DBIterator dbIterator = projectDBAdaptor.nativeIterator(query, QueryOptions.empty()); + DBIterator dbIterator = getProjectDBAdaptor(organizationId).nativeIterator(query, QueryOptions.empty()); exportToFile(dbIterator, outputDir.resolve("projects.json").toFile(), objectMapper, "project"); // Get all the studies contained in the project query = new Query() .append(StudyDBAdaptor.QueryParams.PROJECT_ID.key(), projectId) .append(StudyDBAdaptor.QueryParams.RELEASE.key(), "<=" + release); - OpenCGAResult studyDataResult = studyDBAdaptor.get(query, + OpenCGAResult studyDataResult = getStudyDBAdaptor(organizationId).get(query, new QueryOptions(QueryOptions.INCLUDE, StudyDBAdaptor.QueryParams.UID.key())); if (studyDataResult.getNumResults() == 0) { logger.info("The project does not contain any study under the specified release"); return; } - dbIterator = studyDBAdaptor.nativeIterator(query, QueryOptions.empty()); + dbIterator = getStudyDBAdaptor(organizationId).nativeIterator(query, QueryOptions.empty()); exportToFile(dbIterator, outputDir.resolve("studies.json").toFile(), objectMapper, "studies"); List studyIds = studyDataResult.getResults().stream().map(Study::getUid).collect(Collectors.toList()); @@ -1015,45 +921,45 @@ public void exportReleases(String projectStr, int release, String outputDirStr, .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), studyIds) .append(SampleDBAdaptor.QueryParams.SNAPSHOT.key(), "<=" + release) .append(Constants.ALL_VERSIONS, true); - dbIterator = sampleDBAdaptor.nativeIterator(query, QueryOptions.empty()); + dbIterator = getSampleDBAdaptor(organizationId).nativeIterator(query, QueryOptions.empty()); exportToFile(dbIterator, outputDir.resolve("samples.json").toFile(), objectMapper, "samples"); query = new Query() .append(IndividualDBAdaptor.QueryParams.STUDY_UID.key(), studyIds) .append(IndividualDBAdaptor.QueryParams.SNAPSHOT.key(), "<=" + release) .append(Constants.ALL_VERSIONS, true); - dbIterator = individualDBAdaptor.nativeIterator(query, QueryOptions.empty()); + dbIterator = getIndividualDBAdaptor(organizationId).nativeIterator(query, QueryOptions.empty()); exportToFile(dbIterator, outputDir.resolve("individuals.json").toFile(), objectMapper, "individuals"); query = new Query() .append(FamilyDBAdaptor.QueryParams.STUDY_UID.key(), studyIds) .append(FamilyDBAdaptor.QueryParams.SNAPSHOT.key(), "<=" + release) .append(Constants.ALL_VERSIONS, true); - dbIterator = familyDBAdaptor.nativeIterator(query, QueryOptions.empty()); + dbIterator = getFamilyDBAdaptor(organizationId).nativeIterator(query, QueryOptions.empty()); exportToFile(dbIterator, outputDir.resolve("families.json").toFile(), objectMapper, "families"); query = new Query() .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), studyIds) .append(FileDBAdaptor.QueryParams.RELEASE.key(), "<=" + release); - dbIterator = fileDBAdaptor.nativeIterator(query, QueryOptions.empty()); + dbIterator = getFileDBAdaptor(organizationId).nativeIterator(query, QueryOptions.empty()); exportToFile(dbIterator, outputDir.resolve("files.json").toFile(), objectMapper, "files"); query = new Query() .append(ClinicalAnalysisDBAdaptor.QueryParams.STUDY_UID.key(), studyIds) .append(ClinicalAnalysisDBAdaptor.QueryParams.RELEASE.key(), "<=" + release); - dbIterator = clinicalDBAdaptor.nativeIterator(query, QueryOptions.empty()); + dbIterator = getClinicalAnalysisDBAdaptor(organizationId).nativeIterator(query, QueryOptions.empty()); exportToFile(dbIterator, outputDir.resolve("clinical_analysis.json").toFile(), objectMapper, "clinical analysis"); query = new Query() .append(CohortDBAdaptor.QueryParams.STUDY_UID.key(), studyIds) .append(CohortDBAdaptor.QueryParams.RELEASE.key(), "<=" + release); - dbIterator = cohortDBAdaptor.nativeIterator(query, QueryOptions.empty()); + dbIterator = getCohortDBAdaptor(organizationId).nativeIterator(query, QueryOptions.empty()); exportToFile(dbIterator, outputDir.resolve("cohorts.json").toFile(), objectMapper, "cohorts"); query = new Query() .append(JobDBAdaptor.QueryParams.STUDY_UID.key(), studyIds) .append(JobDBAdaptor.QueryParams.RELEASE.key(), "<=" + release); - dbIterator = jobDBAdaptor.nativeIterator(query, QueryOptions.empty()); + dbIterator = getJobDBAdaptor(organizationId).nativeIterator(query, QueryOptions.empty()); exportToFile(dbIterator, outputDir.resolve("jobs.json").toFile(), objectMapper, "jobs"); } @@ -1090,14 +996,15 @@ private void exportToFile(DBIterator dbIterator, File file, ObjectMapper objectM } } - public OpenCGAResult rank(String userId, Query query, String field, int numResults, boolean asc, String sessionId) + public OpenCGAResult rank(String organizationId, String userId, Query query, String field, int numResults, boolean asc, String token) throws CatalogException { query = ParamUtils.defaultObject(query, Query::new); ParamUtils.checkObj(field, "field"); ParamUtils.checkObj(userId, "userId"); - ParamUtils.checkObj(sessionId, "sessionId"); + ParamUtils.checkObj(token, "sessionId"); - String userOfQuery = this.catalogManager.getUserManager().getUserId(sessionId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + String userOfQuery = tokenPayload.getUserId(organizationId); if (!userOfQuery.equals(userId)) { // The user cannot read projects of other users. throw CatalogAuthorizationException.cantRead(userOfQuery, "Project", null, userId); @@ -1109,20 +1016,22 @@ public OpenCGAResult rank(String userId, Query query, String field, int numResul OpenCGAResult queryResult = null; if (count) { // We do not need to check for permissions when we show the count of files - queryResult = projectDBAdaptor.rank(query, field, numResults, asc); + queryResult = getProjectDBAdaptor(organizationId).rank(query, field, numResults, asc); } return ParamUtils.defaultObject(queryResult, OpenCGAResult::new); } - public OpenCGAResult groupBy(String userId, Query query, String field, QueryOptions options, String sessionId) throws CatalogException { + public OpenCGAResult groupBy(String organizationId, String userId, Query query, String field, QueryOptions options, String token) + throws CatalogException { query = ParamUtils.defaultObject(query, Query::new); options = ParamUtils.defaultObject(options, QueryOptions::new); ParamUtils.checkObj(field, "field"); ParamUtils.checkObj(userId, "userId"); - ParamUtils.checkObj(sessionId, "sessionId"); + ParamUtils.checkObj(token, "sessionId"); - String userOfQuery = this.catalogManager.getUserManager().getUserId(sessionId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + String userOfQuery = tokenPayload.getUserId(organizationId); if (!userOfQuery.equals(userId)) { // The user cannot read projects of other users. throw CatalogAuthorizationException.cantRead(userOfQuery, "Project", null, userId); @@ -1133,21 +1042,22 @@ public OpenCGAResult groupBy(String userId, Query query, String field, QueryOpti OpenCGAResult queryResult = null; if (count) { // We do not need to check for permissions when we show the count of files - queryResult = projectDBAdaptor.groupBy(query, field, options); + queryResult = getProjectDBAdaptor(organizationId).groupBy(query, field, options); } return ParamUtils.defaultObject(queryResult, OpenCGAResult::new); } - public OpenCGAResult groupBy(String userId, Query query, List fields, QueryOptions options, String sessionId) + public OpenCGAResult groupBy(String organizationId, String userId, Query query, List fields, QueryOptions options, String token) throws CatalogException { query = ParamUtils.defaultObject(query, Query::new); options = ParamUtils.defaultObject(options, QueryOptions::new); ParamUtils.checkObj(fields, "fields"); ParamUtils.checkObj(userId, "userId"); - ParamUtils.checkObj(sessionId, "sessionId"); + ParamUtils.checkObj(token, "sessionId"); - String userOfQuery = this.catalogManager.getUserManager().getUserId(sessionId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + String userOfQuery = tokenPayload.getUserId(organizationId); if (!userOfQuery.equals(userId)) { // The user cannot read projects of other users. throw CatalogAuthorizationException.cantRead(userOfQuery, "Project", null, userId); @@ -1158,14 +1068,15 @@ public OpenCGAResult groupBy(String userId, Query query, List fields, Qu OpenCGAResult queryResult = null; if (count) { // We do not need to check for permissions when we show the count of files - queryResult = projectDBAdaptor.groupBy(query, fields, options); + queryResult = getProjectDBAdaptor(organizationId).groupBy(query, fields, options); } return ParamUtils.defaultObject(queryResult, OpenCGAResult::new); } // Return true if currentRelease is found in any entry - private boolean checkCurrentReleaseInUse(List allStudiesInProject, int currentRelease) throws CatalogException { + private boolean checkCurrentReleaseInUse(String organizationId, List allStudiesInProject, int currentRelease) + throws CatalogException { for (Study study : allStudiesInProject) { if (study.getRelease() == currentRelease) { return true; @@ -1175,28 +1086,28 @@ private boolean checkCurrentReleaseInUse(List allStudiesInProject, int cu Query query = new Query() .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), studyIds) .append(FileDBAdaptor.QueryParams.RELEASE.key(), currentRelease); - if (fileDBAdaptor.count(query).getNumMatches() > 0) { + if (getFileDBAdaptor(organizationId).count(query).getNumMatches() > 0) { return true; } - if (sampleDBAdaptor.count(query).getNumMatches() > 0) { + if (getSampleDBAdaptor(organizationId).count(query).getNumMatches() > 0) { return true; } - if (individualDBAdaptor.count(query).getNumMatches() > 0) { + if (getIndividualDBAdaptor(organizationId).count(query).getNumMatches() > 0) { return true; } - if (cohortDBAdaptor.count(query).getNumMatches() > 0) { + if (getCohortDBAdaptor(organizationId).count(query).getNumMatches() > 0) { return true; } - if (familyDBAdaptor.count(query).getNumMatches() > 0) { + if (getFamilyDBAdaptor(organizationId).count(query).getNumMatches() > 0) { return true; } - if (jobDBAdaptor.count(query).getNumMatches() > 0) { + if (getJobDBAdaptor(organizationId).count(query).getNumMatches() > 0) { return true; } // if (diseasePanelDBAdaptor.count(query).getNumMatches() > 0) { // return true; // } - if (clinicalDBAdaptor.count(query).getNumMatches() > 0) { + if (getClinicalAnalysisDBAdaptor(organizationId).count(query).getNumMatches() > 0) { return true; } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ResourceManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ResourceManager.java index a6e14d9942c..9584a4a7301 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ResourceManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ResourceManager.java @@ -27,10 +27,12 @@ import org.opencb.opencga.catalog.db.api.DBIterator; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.models.InternalGetDataResult; +import org.opencb.opencga.catalog.utils.CatalogFqn; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.catalog.utils.UuidUtils; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.models.IPrivateStudyUid; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.audit.AuditRecord; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.study.Study; @@ -53,59 +55,67 @@ public abstract class ResourceManager extends Abstra abstract Enums.Resource getEntity(); - OpenCGAResult internalGet(long studyUid, String entry, QueryOptions options, String user) throws CatalogException { - return internalGet(studyUid, entry, null, options, user); + OpenCGAResult internalGet(String organizationId, long studyUid, String entry, QueryOptions options, String user) + throws CatalogException { + return internalGet(organizationId, studyUid, entry, null, options, user); } - OpenCGAResult internalGet(long studyUid, String entry, @Nullable Query query, QueryOptions options, String user) - throws CatalogException { + OpenCGAResult internalGet(String organizationId, long studyUid, String entry, @Nullable Query query, QueryOptions options, + String user) throws CatalogException { ParamUtils.checkIsSingleID(entry); - return internalGet(studyUid, Collections.singletonList(entry), query, options, user, false); + return internalGet(organizationId, studyUid, Collections.singletonList(entry), query, options, user, false); } - InternalGetDataResult internalGet(long studyUid, List entryList, QueryOptions options, String user, boolean ignoreException) - throws CatalogException { - return internalGet(studyUid, entryList, null, options, user, ignoreException); + InternalGetDataResult internalGet(String organizationId, long studyUid, List entryList, QueryOptions options, String user, + boolean ignoreException) throws CatalogException { + return internalGet(organizationId, studyUid, entryList, null, options, user, ignoreException); } - abstract InternalGetDataResult internalGet(long studyUid, List entryList, @Nullable Query query, QueryOptions options, - String user, boolean ignoreException) throws CatalogException; + abstract InternalGetDataResult internalGet(String organizationId, long studyUid, List entryList, @Nullable Query query, + QueryOptions options, String user, boolean ignoreException) throws CatalogException; /** * Create an entry in catalog. * - * @param studyStr Study id in string format. Could be one of [id|user@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy]. - * @param entry entry that needs to be added in Catalog. - * @param options QueryOptions object. - * @param token Session id of the user logged in. + * @param studyStr Study id in string format. Could be one of + * [id|organization@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy] + * @param entry entry that needs to be added in Catalog. + * @param options QueryOptions object. + * @param token Session id of the user logged in. * @return A OpenCGAResult of the object created. * @throws CatalogException if any parameter from the entry is incorrect, the user does not have permissions... */ - public abstract OpenCGAResult create(String studyStr, R entry, QueryOptions options, String token) throws CatalogException; + public abstract OpenCGAResult create(String studyStr, R entry, QueryOptions options, String token) + throws CatalogException; /** * Fetch the R object. * - * @param studyStr Study id in string format. Could be one of [id|user@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy]. - * @param entryStr Entry id to be fetched. - * @param options QueryOptions object, like "include", "exclude", "limit" and "skip". - * @param token token + * @param studyStr Study id in string format. Could be one of + * [id|organization@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy] + * @param entryStr Entry id to be fetched. + * @param options QueryOptions object, like "include", "exclude", "limit" and "skip". + * @param token token * @return All matching elements. * @throws CatalogException CatalogException. */ public OpenCGAResult get(String studyStr, String entryStr, QueryOptions options, String token) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(token); - Study study = catalogManager.getStudyManager().resolveId(studyStr, userId); - return internalGet(study.getUid(), entryStr, options, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyStr, userId, organizationId); + return internalGet(organizationId, study.getUid(), entryStr, options, userId); } /** * Fetch all the R objects matching the query. * - * @param studyStr Study id in string format. Could be one of [id|user@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy]. + * @param studyStr Study id in string format. Could be one of + * [id|organization@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy] * @param entryList Comma separated list of entries to be fetched. * @param options QueryOptions object, like "include", "exclude", "limit" and "skip". - * @param token token + * @param token token * @return All matching elements. * @throws CatalogException CatalogException. */ @@ -120,8 +130,11 @@ public OpenCGAResult get(String studyStr, List entryList, QueryOption public OpenCGAResult get(String studyId, List entryList, Query query, QueryOptions options, boolean ignoreException, String token) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(token); - Study study = catalogManager.getStudyManager().resolveId(studyId, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyId, userId, organizationId); query = ParamUtils.defaultObject(query, Query::new); options = ParamUtils.defaultObject(options, QueryOptions::new); @@ -140,7 +153,8 @@ public OpenCGAResult get(String studyId, List entryList, Query query, OpenCGAResult result = OpenCGAResult.empty(); options.remove(QueryOptions.LIMIT); - InternalGetDataResult responseResult = internalGet(study.getUid(), entryList, query, options, userId, ignoreException); + InternalGetDataResult responseResult = internalGet(organizationId, study.getUid(), entryList, query, options, userId, + ignoreException); Map missingMap = new HashMap<>(); if (responseResult.getMissing() != null) { @@ -164,7 +178,7 @@ public OpenCGAResult get(String studyId, List entryList, Query query, // size)); R entry = versionedResults.get(i).get(0); - auditManager.auditInfo(operationUuid, userId, getEntity(), entry.getId(), entry.getUuid(), + auditManager.auditInfo(organizationId, operationUuid, userId, getEntity(), entry.getId(), entry.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } } @@ -172,118 +186,134 @@ public OpenCGAResult get(String studyId, List entryList, Query query, return result; } catch (CatalogException e) { for (String entryId : entryList) { - auditManager.auditInfo(operationUuid, userId, getEntity(), entryId, "", study.getId(), study.getUuid(), auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditInfo(organizationId, operationUuid, userId, getEntity(), entryId, "", study.getId(), study.getUuid(), + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); } throw e; } finally { - auditManager.finishAuditBatch(operationUuid); + auditManager.finishAuditBatch(organizationId, operationUuid); } } /** * Obtain an entry iterator to iterate over the matching entries. * - * @param studyStr study id in string format. Could be one of [id|user@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy]. - * @param query Query object. - * @param options QueryOptions object. - * @param token Session id of the user logged in. + * @param studyStr study id in string format. Could be one of + * [id|organization@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy] + * @param query Query object. + * @param options QueryOptions object. + * @param token Session id of the user logged in. * @return An iterator. * @throws CatalogException if there is any internal error. */ - public abstract DBIterator iterator(String studyStr, Query query, QueryOptions options, String token) throws CatalogException; + public abstract DBIterator iterator(String studyStr, Query query, QueryOptions options, String token) + throws CatalogException; /** * Search of entries in catalog. * - * @param studyId study id in string format. Could be one of [id|user@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy]. - * @param query Query object. - * @param options QueryOptions object. - * @param token Session id of the user logged in. + * @param studyId study id in string format. Could be one of + * [id|organization@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy] + * @param query Query object. + * @param options QueryOptions object. + * @param token Session id of the user logged in. * @return The list of entries matching the query. * @throws CatalogException catalogException. */ - public abstract OpenCGAResult search(String studyId, Query query, QueryOptions options, String token) throws CatalogException; + public abstract OpenCGAResult search(String studyId, Query query, QueryOptions options, String token) + throws CatalogException; /** * Fetch a list containing all the distinct values of the key {@code field}. * - * @param studyId study id in string format. Could be one of [id|user@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy]. - * @param field The field for which to return distinct values. - * @param query Query object. - * @param token Token of the user logged in. + * @param organizationId Organization id. + * @param studyId study id in string format. Could be one of + * [id|organization@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy] + * @param field The field for which to return distinct values. + * @param query Query object. + * @param token Token of the user logged in. * @return The list of distinct values. * @throws CatalogException CatalogException. */ - public OpenCGAResult distinct(String studyId, String field, Query query, String token) throws CatalogException { + public OpenCGAResult distinct(String organizationId, String studyId, String field, Query query, String token) + throws CatalogException { return distinct(studyId, Collections.singletonList(field), query, token); } /** * Fetch a list containing all the distinct values of the key {@code field}. * - * @param studyId study id in string format. Could be one of [id|user@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy]. - * @param fields Fields for which to return distinct values. - * @param query Query object. - * @param token Token of the user logged in. + * @param studyId study id in string format. Could be one of + * [id|organization@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy] + * @param fields Fields for which to return distinct values. + * @param query Query object. + * @param token Token of the user logged in. * @return The list of distinct values. * @throws CatalogException CatalogException. */ - public abstract OpenCGAResult distinct(String studyId, List fields, Query query, String token) throws CatalogException; + public abstract OpenCGAResult distinct(String studyId, List fields, Query query, String token) + throws CatalogException; /** * Count matching entries in catalog. * - * @param studyId study id in string format. Could be one of [id|user@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy]. - * @param query Query object. - * @param token Session id of the user logged in. + * @param studyId study id in string format. Could be one of + * [id|organization@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy] + * @param query Query object. + * @param token Session id of the user logged in. * @return A OpenCGAResult with the total number of entries matching the query. * @throws CatalogException catalogException. */ public abstract OpenCGAResult count(String studyId, Query query, String token) throws CatalogException; - public abstract OpenCGAResult delete(String studyStr, List ids, QueryOptions options, String token) throws CatalogException; + public abstract OpenCGAResult delete(String studyStr, List ids, QueryOptions options, String token) + throws CatalogException; /** * Delete all entries matching the query. * - * @param studyStr Study id in string format. Could be one of [id|user@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy]. - * @param query Query object. - * @param options Map containing additional parameters to be considered for the deletion. - * @param token Session id of the user logged in. - * @throws CatalogException if the study or the user do not exist. + * @param studyStr Study id in string format. Could be one of + * [id|organization@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy] + * @param query Query object. + * @param options Map containing additional parameters to be considered for the deletion. + * @param token Session id of the user logged in. * @return A OpenCGAResult object containing the number of matching elements, deleted and elements that could not be deleted. + * @throws CatalogException if the study or the user do not exist. */ - public abstract OpenCGAResult delete(String studyStr, Query query, QueryOptions options, String token) throws CatalogException; + public abstract OpenCGAResult delete(String studyStr, Query query, QueryOptions options, String token) + throws CatalogException; /** * Ranks the elements queried, groups them by the field(s) given and return it sorted. * - * @param studyStr study id in string format. Could be one of [id|user@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy]. + * @param studyStr study id in string format. Could be one of + * [id|organization@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy] * @param query Query object containing the query that will be executed. * @param field A field or a comma separated list of fields by which the results will be grouped in. * @param numResults Maximum number of results to be reported. * @param asc Order in which the results will be reported. - * @param token Session id of the user logged in. + * @param token Session id of the user logged in. * @return A OpenCGAResult object containing each of the fields in field and the count of them matching the query. * @throws CatalogException CatalogException */ - public abstract OpenCGAResult rank(String studyStr, Query query, String field, int numResults, boolean asc, String token) - throws CatalogException; + public abstract OpenCGAResult rank(String studyStr, Query query, String field, int numResults, boolean asc, + String token) throws CatalogException; /** * Groups the matching entries by some fields. * - * @param studyStr study id in string format. Could be one of [id|user@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy]. - * @param query Query object. - * @param fields A field or a comma separated list of fields by which the results will be grouped in. - * @param options QueryOptions object. - * @param token Session id of the user logged in. + * @param organizationId Organization id. + * @param studyStr study id in string format. Could be one of + * [id|organization@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy] + * @param query Query object. + * @param fields A field or a comma separated list of fields by which the results will be grouped in. + * @param options QueryOptions object. + * @param token Session id of the user logged in. * @return A OpenCGAResult object containing the results of the query grouped by the fields. * @throws CatalogException CatalogException */ - public OpenCGAResult groupBy(@Nullable String studyStr, Query query, String fields, QueryOptions options, String token) - throws CatalogException { + public OpenCGAResult groupBy(String organizationId, @Nullable String studyStr, Query query, String fields, QueryOptions options, + String token) throws CatalogException { if (StringUtils.isEmpty(fields)) { throw new CatalogException("Empty fields parameter."); } @@ -293,16 +323,17 @@ public OpenCGAResult groupBy(@Nullable String studyStr, Query query, String fiel /** * Groups the matching entries by some fields. * - * @param studyStr study id in string format. Could be one of [id|user@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy]. - * @param query Query object. - * @param options QueryOptions object. - * @param fields A field or a comma separated list of fields by which the results will be grouped in. - * @param token Session id of the user logged in. + * @param studyStr study id in string format. Could be one of + * [id|organization@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy] + * @param query Query object. + * @param fields A field or a comma separated list of fields by which the results will be grouped in. + * @param options QueryOptions object. + * @param token Session id of the user logged in. * @return A OpenCGAResult object containing the results of the query grouped by the fields. * @throws CatalogException CatalogException */ - public abstract OpenCGAResult groupBy(@Nullable String studyStr, Query query, List fields, QueryOptions options, String token) - throws CatalogException; + public abstract OpenCGAResult groupBy(@Nullable String studyStr, Query query, List fields, + QueryOptions options, String token) throws CatalogException; /** * Returns result if there are no ERROR events or ignoreException is true. Otherwise, raise an exception with all error messages. diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/SampleManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/SampleManager.java index 580bc1c2c2c..48976806622 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/SampleManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/SampleManager.java @@ -23,7 +23,10 @@ import org.opencb.biodata.models.common.Status; import org.opencb.biodata.models.variant.StudyEntry; import org.opencb.biodata.models.variant.metadata.SampleVariantStats; -import org.opencb.commons.datastore.core.*; +import org.opencb.commons.datastore.core.Event; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.result.Error; import org.opencb.commons.utils.ListUtils; import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager; @@ -32,16 +35,13 @@ import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.models.InternalGetDataResult; -import org.opencb.opencga.catalog.stats.solr.CatalogSolrManager; -import org.opencb.opencga.catalog.utils.AnnotationUtils; -import org.opencb.opencga.catalog.utils.Constants; -import org.opencb.opencga.catalog.utils.ParamUtils; -import org.opencb.opencga.catalog.utils.UuidUtils; +import org.opencb.opencga.catalog.utils.*; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.models.AclEntryList; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.audit.AuditRecord; import org.opencb.opencga.core.models.cohort.Cohort; import org.opencb.opencga.core.models.cohort.CohortStatus; @@ -81,14 +81,11 @@ public class SampleManager extends AnnotationSetManager { SampleDBAdaptor.QueryParams.VERSION.key(), SampleDBAdaptor.QueryParams.STUDY_UID.key())); protected static Logger logger = LoggerFactory.getLogger(SampleManager.class); private final String defaultFacet = "creationYear>>creationMonth;status;phenotypes;somatic"; - private UserManager userManager; private StudyManager studyManager; SampleManager(AuthorizationManager authorizationManager, AuditManager auditManager, CatalogManager catalogManager, DBAdaptorFactory catalogDBAdaptorFactory, Configuration configuration) { super(authorizationManager, auditManager, catalogManager, catalogDBAdaptorFactory, configuration); - - userManager = catalogManager.getUserManager(); studyManager = catalogManager.getStudyManager(); } @@ -98,8 +95,8 @@ Enums.Resource getEntity() { } @Override - InternalGetDataResult internalGet(long studyUid, List entryList, @Nullable Query query, QueryOptions options, - String user, boolean ignoreException) throws CatalogException { + InternalGetDataResult internalGet(String organizationId, long studyUid, List entryList, @Nullable Query query, + QueryOptions options, String user, boolean ignoreException) throws CatalogException { if (ListUtils.isEmpty(entryList)) { throw new CatalogException("Missing sample entries."); } @@ -122,7 +119,7 @@ InternalGetDataResult internalGet(long studyUid, List entryList, // Ensure the field by which we are querying for will be kept in the results queryOptions = keepFieldInQueryOptions(queryOptions, idQueryParam.key()); - OpenCGAResult sampleDataResult = sampleDBAdaptor.get(studyUid, queryCopy, queryOptions, user); + OpenCGAResult sampleDataResult = getSampleDBAdaptor(organizationId).get(studyUid, queryCopy, queryOptions, user); Function sampleStringFunction = Sample::getId; if (idQueryParam.equals(SampleDBAdaptor.QueryParams.UUID)) { @@ -133,7 +130,7 @@ InternalGetDataResult internalGet(long studyUid, List entryList, return keepOriginalOrder(uniqueList, sampleStringFunction, sampleDataResult, ignoreException, versioned); } // Query without adding the user check - OpenCGAResult resultsNoCheck = sampleDBAdaptor.get(queryCopy, queryOptions); + OpenCGAResult resultsNoCheck = getSampleDBAdaptor(organizationId).get(queryCopy, queryOptions); if (resultsNoCheck.getNumResults() == sampleDataResult.getNumResults()) { throw CatalogException.notFound("samples", getMissingFields(uniqueList, sampleDataResult.getResults(), sampleStringFunction)); @@ -159,28 +156,29 @@ SampleDBAdaptor.QueryParams getFieldFilter(List idList) throws CatalogEx return idQueryParam; } - private OpenCGAResult getSample(long studyUid, String sampleUuid, QueryOptions options) throws CatalogException { + private OpenCGAResult getSample(String organizationId, long studyUid, String sampleUuid, QueryOptions options) + throws CatalogException { Query query = new Query() .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(SampleDBAdaptor.QueryParams.UUID.key(), sampleUuid); - return sampleDBAdaptor.get(query, options); + return getSampleDBAdaptor(organizationId).get(query, options); } - void validateNewSample(Study study, Sample sample, String userId) throws CatalogException { + void validateNewSample(String organizationId, Study study, Sample sample, String userId) throws CatalogException { ParamUtils.checkIdentifier(sample.getId(), "id"); // Check the id is not in use Query query = new Query() .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()) .append(SampleDBAdaptor.QueryParams.ID.key(), sample.getId()); - if (sampleDBAdaptor.count(query).getNumMatches() > 0) { + if (getSampleDBAdaptor(organizationId).count(query).getNumMatches() > 0) { throw new CatalogException("Sample '" + sample.getId() + "' already exists."); } if (StringUtils.isNotEmpty(sample.getIndividualId())) { // Check individual exists - OpenCGAResult individualDataResult = catalogManager.getIndividualManager().internalGet(study.getUid(), - sample.getIndividualId(), IndividualManager.INCLUDE_INDIVIDUAL_IDS, userId); + OpenCGAResult individualDataResult = catalogManager.getIndividualManager().internalGet(organizationId, + study.getUid(), sample.getIndividualId(), IndividualManager.INCLUDE_INDIVIDUAL_IDS, userId); if (individualDataResult.getNumResults() == 0) { throw new CatalogException("Individual '" + sample.getIndividualId() + "' not found."); } @@ -226,117 +224,142 @@ void validateNewSample(Study study, Sample sample, String userId) throws Catalog public OpenCGAResult create(String studyStr, Sample sample, QueryOptions options, String token) throws CatalogException { options = ParamUtils.defaultObject(options, QueryOptions::new); - String userId = userManager.getUserId(token); - Study study = catalogManager.getStudyManager().resolveId(studyStr, userId, StudyManager.INCLUDE_VARIABLE_SET); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); ObjectMap auditParams = new ObjectMap() .append("study", studyStr) .append("sample", sample) .append("options", options) .append("token", token); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + String studyId = studyFqn.getStudyId(); + String studyUuid = studyFqn.getStudyUuid(); try { + Study study = catalogManager.getStudyManager().resolveId(studyFqn, StudyManager.INCLUDE_VARIABLE_SET, tokenPayload); + studyId = study.getId(); + studyUuid = study.getUuid(); + // 1. We check everything can be done - authorizationManager.checkStudyPermission(study.getUid(), userId, StudyPermissions.Permissions.WRITE_SAMPLES); + authorizationManager.checkStudyPermission(organizationId, study.getUid(), tokenPayload, + StudyPermissions.Permissions.WRITE_SAMPLES); - validateNewSample(study, sample, userId); + validateNewSample(organizationId, study, sample, userId); // We create the sample - OpenCGAResult insert = sampleDBAdaptor.insert(study.getUid(), sample, study.getVariableSets(), options); + OpenCGAResult insert = getSampleDBAdaptor(organizationId).insert(study.getUid(), sample, study.getVariableSets(), + options); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { // Fetch created sample - OpenCGAResult result = getSample(study.getUid(), sample.getUuid(), options); + OpenCGAResult result = getSample(organizationId, study.getUid(), sample.getUuid(), options); insert.setResults(result.getResults()); } - auditManager.auditCreate(userId, Enums.Resource.SAMPLE, sample.getId(), sample.getUuid(), study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.auditCreate(organizationId, userId, Enums.Resource.SAMPLE, sample.getId(), sample.getUuid(), study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return insert; } catch (CatalogException e) { - auditManager.auditCreate(userId, Enums.Resource.SAMPLE, sample.getId(), "", study.getId(), study.getUuid(), auditParams, + auditManager.auditCreate(organizationId, userId, Enums.Resource.SAMPLE, sample.getId(), "", studyId, studyUuid, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } @Override - public DBIterator iterator(String studyStr, Query query, QueryOptions options, String sessionId) throws CatalogException { + public DBIterator iterator(String studyStr, Query query, QueryOptions options, String token) throws CatalogException { query = ParamUtils.defaultObject(query, Query::new); options = ParamUtils.defaultObject(options, QueryOptions::new); - String userId = userManager.getUserId(sessionId); - Study study = catalogManager.getStudyManager().resolveId(studyStr, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + Study study = catalogManager.getStudyManager().resolveId(studyFqn, null, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); Query finalQuery = new Query(query); - fixQueryObject(study, finalQuery, userId); + fixQueryObject(organizationId, study, finalQuery, userId); AnnotationUtils.fixQueryOptionAnnotation(options); finalQuery.append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - return sampleDBAdaptor.iterator(study.getUid(), finalQuery, options, userId); + return getSampleDBAdaptor(organizationId).iterator(study.getUid(), finalQuery, options, userId); } @Override - public OpenCGAResult search(String studyId, Query query, QueryOptions options, String token) throws CatalogException { + public OpenCGAResult search(String studyStr, Query query, QueryOptions options, String token) + throws CatalogException { query = ParamUtils.defaultObject(query, Query::new); options = ParamUtils.defaultObject(options, QueryOptions::new); - String userId = userManager.getUserId(token); - Study study = catalogManager.getStudyManager().resolveId(studyId, userId, new QueryOptions(QueryOptions.INCLUDE, - StudyDBAdaptor.QueryParams.VARIABLE_SET.key())); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + String studyId = studyFqn.getStudyId(); + String studyUuid = studyFqn.getStudyUuid(); ObjectMap auditParams = new ObjectMap() - .append("studyId", studyId) + .append("studyStr", studyStr) .append("query", new Query(query)) .append("options", options) .append("token", token); try { - fixQueryObject(study, query, userId); + Study study = catalogManager.getStudyManager().resolveId(studyFqn, StudyManager.INCLUDE_VARIABLE_SET, tokenPayload); + studyId = study.getId(); + studyUuid = study.getUuid(); + + fixQueryObject(organizationId, study, query, userId); AnnotationUtils.fixQueryOptionAnnotation(options); query.append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - OpenCGAResult queryResult = sampleDBAdaptor.get(study.getUid(), query, options, userId); + OpenCGAResult queryResult = getSampleDBAdaptor(organizationId).get(study.getUid(), query, options, userId); - auditManager.auditSearch(userId, Enums.Resource.SAMPLE, study.getId(), study.getUuid(), auditParams, + auditManager.auditSearch(organizationId, userId, Enums.Resource.SAMPLE, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return queryResult; } catch (CatalogException e) { - auditManager.auditSearch(userId, Enums.Resource.SAMPLE, study.getId(), study.getUuid(), auditParams, + auditManager.auditSearch(organizationId, userId, Enums.Resource.SAMPLE, studyId, studyUuid, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } @Override - public OpenCGAResult distinct(String studyId, List fields, Query query, String token) throws CatalogException { + public OpenCGAResult distinct(String studyStr, List fields, Query query, String token) throws CatalogException { query = ParamUtils.defaultObject(query, Query::new); - String userId = userManager.getUserId(token); - Study study = catalogManager.getStudyManager().resolveId(studyId, userId, new QueryOptions(QueryOptions.INCLUDE, - StudyDBAdaptor.QueryParams.VARIABLE_SET.key())); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + + Study study = catalogManager.getStudyManager().resolveId(studyStr, new QueryOptions(QueryOptions.INCLUDE, + StudyDBAdaptor.QueryParams.VARIABLE_SET.key()), userId, organizationId); ObjectMap auditParams = new ObjectMap() - .append("studyId", studyId) + .append("studyId", studyStr) .append("fields", fields) .append("query", new Query(query)) .append("token", token); try { - fixQueryObject(study, query, userId); + fixQueryObject(organizationId, study, query, userId); query.append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - OpenCGAResult result = sampleDBAdaptor.distinct(study.getUid(), fields, query, userId); + OpenCGAResult result = getSampleDBAdaptor(organizationId).distinct(study.getUid(), fields, query, userId); - auditManager.auditDistinct(userId, Enums.Resource.SAMPLE, study.getId(), study.getUuid(), auditParams, + auditManager.auditDistinct(organizationId, userId, Enums.Resource.SAMPLE, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return result; } catch (CatalogException e) { - auditManager.auditDistinct(userId, Enums.Resource.SAMPLE, study.getId(), study.getUuid(), auditParams, + auditManager.auditDistinct(organizationId, userId, Enums.Resource.SAMPLE, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } - void fixQueryObject(Study study, Query query, String userId) throws CatalogException { + void fixQueryObject(String organizationId, Study study, Query query, String userId) throws CatalogException { changeQueryId(query, ParamConstants.SAMPLE_RGA_STATUS_PARAM, SampleDBAdaptor.QueryParams.INTERNAL_RGA_STATUS.key()); changeQueryId(query, ParamConstants.SAMPLE_PROCESSING_PREPARATION_METHOD_PARAM, SampleDBAdaptor.QueryParams.PROCESSING_PREPARATION_METHOD.key()); @@ -353,7 +376,7 @@ void fixQueryObject(Study study, Query query, String userId) throws CatalogExcep super.fixQueryObject(query); // Fix query if it contains any annotation - AnnotationUtils.fixQueryAnnotationSearch(study, query); + AnnotationUtils.fixQueryAnnotationSearch(organizationId, study, query); // The files introduced could be either ids, paths or uuids. As so, we should use the smart resolutor to do this. if (StringUtils.isNotEmpty(query.getString(SampleDBAdaptor.QueryParams.FILE_IDS.key()))) { @@ -370,7 +393,7 @@ void fixQueryObject(Study study, Query query, String userId) throws CatalogExcep if (queryFileManager) { // Obtain the corresponding fileIds - InternalGetDataResult result = catalogManager.getFileManager().internalGet(study.getUid(), + InternalGetDataResult result = catalogManager.getFileManager().internalGet(organizationId, study.getUid(), query.getAsStringList(SampleDBAdaptor.QueryParams.FILE_IDS.key()), FileManager.INCLUDE_FILE_IDS, userId, true); if (result.getMissing() == null || result.getMissing().isEmpty()) { @@ -398,7 +421,7 @@ void fixQueryObject(Study study, Query query, String userId) throws CatalogExcep } if (queryIndividualManager) { - InternalGetDataResult result = catalogManager.getIndividualManager().internalGet(study.getUid(), + InternalGetDataResult result = catalogManager.getIndividualManager().internalGet(organizationId, study.getUid(), query.getAsStringList(SampleDBAdaptor.QueryParams.INDIVIDUAL_ID.key()), IndividualManager.INCLUDE_INDIVIDUAL_IDS, userId, true); if (result.getMissing() == null || result.getMissing().isEmpty()) { @@ -416,37 +439,42 @@ void fixQueryObject(Study study, Query query, String userId) throws CatalogExcep } @Override - public OpenCGAResult count(String studyId, Query query, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = catalogManager.getStudyManager().resolveId(studyId, userId, new QueryOptions(QueryOptions.INCLUDE, - StudyDBAdaptor.QueryParams.VARIABLE_SET.key())); + public OpenCGAResult count(String studyStr, Query query, String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + + Study study = catalogManager.getStudyManager().resolveId(studyStr, new QueryOptions(QueryOptions.INCLUDE, + StudyDBAdaptor.QueryParams.VARIABLE_SET.key()), userId, organizationId); query = new Query(ParamUtils.defaultObject(query, Query::new)); ObjectMap auditParams = new ObjectMap() - .append("studyId", studyId) + .append("studyId", studyStr) .append("query", query) .append("token", token); try { - fixQueryObject(study, query, userId); + fixQueryObject(organizationId, study, query, userId); query.append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - OpenCGAResult queryResultAux = sampleDBAdaptor.count(query, userId); + OpenCGAResult queryResultAux = getSampleDBAdaptor(organizationId).count(query, userId); - auditManager.auditCount(userId, Enums.Resource.SAMPLE, study.getId(), study.getUuid(), auditParams, + auditManager.auditCount(organizationId, userId, Enums.Resource.SAMPLE, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return new OpenCGAResult<>(queryResultAux.getTime(), queryResultAux.getEvents(), 0, Collections.emptyList(), queryResultAux.getNumMatches()); } catch (CatalogException e) { - auditManager.auditCount(userId, Enums.Resource.SAMPLE, study.getId(), study.getUuid(), auditParams, + auditManager.auditCount(organizationId, userId, Enums.Resource.SAMPLE, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } @Override - public OpenCGAResult delete(String studyStr, List sampleIds, QueryOptions options, String token) throws CatalogException { + public OpenCGAResult delete(String studyStr, List sampleIds, QueryOptions options, String token) + throws CatalogException { return delete(studyStr, sampleIds, options, false, token); } @@ -456,9 +484,12 @@ public OpenCGAResult delete(String studyStr, List sampleIds, ObjectMap p throw new CatalogException("Missing list of sample ids"); } - String userId = catalogManager.getUserManager().getUserId(token); - Study study = studyManager.resolveId(studyStr, userId, new QueryOptions(QueryOptions.INCLUDE, - StudyDBAdaptor.QueryParams.VARIABLE_SET.key())); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, new QueryOptions(QueryOptions.INCLUDE, + StudyDBAdaptor.QueryParams.VARIABLE_SET.key()), userId, organizationId); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -472,9 +503,10 @@ public OpenCGAResult delete(String studyStr, List sampleIds, ObjectMap p boolean checkPermissions; try { // If the user is the owner or the admin, we won't check if he has permissions for every single entry - checkPermissions = !authorizationManager.isOwnerOrAdmin(study.getUid(), userId); + long studyId = study.getUid(); + checkPermissions = !authorizationManager.isAtLeastStudyAdministrator(organizationId, studyId, userId); } catch (CatalogException e) { - auditManager.auditDelete(operationId, userId, Enums.Resource.SAMPLE, "", "", study.getId(), study.getUuid(), + auditManager.auditDelete(organizationId, operationId, userId, Enums.Resource.SAMPLE, "", "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -485,7 +517,7 @@ public OpenCGAResult delete(String studyStr, List sampleIds, ObjectMap p String sampleId = id; String sampleUuid = ""; try { - OpenCGAResult internalResult = internalGet(study.getUid(), id, INCLUDE_SAMPLE_IDS, userId); + OpenCGAResult internalResult = internalGet(organizationId, study.getUid(), id, INCLUDE_SAMPLE_IDS, userId); if (internalResult.getNumResults() == 0) { throw new CatalogException("Sample '" + id + "' not found"); } @@ -496,16 +528,16 @@ public OpenCGAResult delete(String studyStr, List sampleIds, ObjectMap p sampleUuid = sample.getUuid(); if (checkPermissions) { - authorizationManager.checkSamplePermission(study.getUid(), sample.getUid(), userId, + authorizationManager.checkSamplePermission(organizationId, study.getUid(), sample.getUid(), userId, SamplePermissions.DELETE); } // Check if the sample can be deleted - checkSampleCanBeDeleted(study.getUid(), sample, params.getBoolean(Constants.FORCE, false)); + checkSampleCanBeDeleted(organizationId, study.getUid(), sample, params.getBoolean(Constants.FORCE, false)); - result.append(sampleDBAdaptor.delete(sample)); + result.append(getSampleDBAdaptor(organizationId).delete(sample)); - auditManager.auditDelete(operationId, userId, Enums.Resource.SAMPLE, sample.getId(), sample.getUuid(), + auditManager.auditDelete(organizationId, operationId, userId, Enums.Resource.SAMPLE, sample.getId(), sample.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { String errorMsg = "Cannot delete sample " + sampleId + ": " + e.getMessage(); @@ -515,11 +547,11 @@ public OpenCGAResult delete(String studyStr, List sampleIds, ObjectMap p result.setNumErrors(result.getNumErrors() + 1); logger.error(errorMsg); - auditManager.auditDelete(operationId, userId, Enums.Resource.SAMPLE, sampleId, sampleUuid, + auditManager.auditDelete(organizationId, operationId, userId, Enums.Resource.SAMPLE, sampleId, sampleUuid, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); } } - auditManager.finishAuditBatch(operationId); + auditManager.finishAuditBatch(organizationId, operationId); return endResult(result, ignoreException); } @@ -536,8 +568,11 @@ public OpenCGAResult delete(String studyStr, Query query, ObjectMap params, bool OpenCGAResult result = OpenCGAResult.empty(); - String userId = catalogManager.getUserManager().getUserId(token); - Study study = catalogManager.getStudyManager().resolveId(studyStr, userId, StudyManager.INCLUDE_VARIABLE_SET); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyStr, StudyManager.INCLUDE_VARIABLE_SET, userId, organizationId); String operationUuid = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -567,15 +602,16 @@ public OpenCGAResult delete(String studyStr, Query query, ObjectMap params, bool // params.put(Constants.EMPTY_FILES_ACTION, "NONE"); // } - fixQueryObject(study, finalQuery, userId); + fixQueryObject(organizationId, study, finalQuery, userId); finalQuery.append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - iterator = sampleDBAdaptor.iterator(study.getUid(), finalQuery, INCLUDE_SAMPLE_IDS, userId); + iterator = getSampleDBAdaptor(organizationId).iterator(study.getUid(), finalQuery, INCLUDE_SAMPLE_IDS, userId); // If the user is the owner or the admin, we won't check if he has permissions for every single entry - checkPermissions = !authorizationManager.isOwnerOrAdmin(study.getUid(), userId); + long studyId = study.getUid(); + checkPermissions = !authorizationManager.isAtLeastStudyAdministrator(organizationId, studyId, userId); } catch (CatalogException e) { - auditManager.auditDelete(operationUuid, userId, Enums.Resource.SAMPLE, "", "", study.getId(), study.getUuid(), + auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.SAMPLE, "", "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -586,16 +622,16 @@ public OpenCGAResult delete(String studyStr, Query query, ObjectMap params, bool try { if (checkPermissions) { - authorizationManager.checkSamplePermission(study.getUid(), sample.getUid(), userId, + authorizationManager.checkSamplePermission(organizationId, study.getUid(), sample.getUid(), userId, SamplePermissions.DELETE); } // Check if the sample can be deleted - checkSampleCanBeDeleted(study.getUid(), sample, params.getBoolean(Constants.FORCE, false)); + checkSampleCanBeDeleted(organizationId, study.getUid(), sample, params.getBoolean(Constants.FORCE, false)); - result.append(sampleDBAdaptor.delete(sample)); + result.append(getSampleDBAdaptor(organizationId).delete(sample)); - auditManager.auditDelete(operationUuid, userId, Enums.Resource.SAMPLE, sample.getId(), sample.getUuid(), + auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.SAMPLE, sample.getId(), sample.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { String errorMsg = "Cannot delete sample " + sample.getId() + ": " + e.getMessage(); @@ -605,19 +641,22 @@ public OpenCGAResult delete(String studyStr, Query query, ObjectMap params, bool result.setNumErrors(result.getNumErrors() + 1); logger.error(errorMsg); - auditManager.auditDelete(operationUuid, userId, Enums.Resource.SAMPLE, sample.getId(), sample.getUuid(), + auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.SAMPLE, sample.getId(), sample.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); } } - auditManager.finishAuditBatch(operationUuid); + auditManager.finishAuditBatch(organizationId, operationUuid); return endResult(result, ignoreException); } // TODO: This method should be private. This should only be accessible internally. public OpenCGAResult resetRgaIndexes(String studyStr, String token) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(token); - Study study = catalogManager.getStudyManager().resolveId(studyStr, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyStr, userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("study", studyStr) @@ -625,14 +664,16 @@ public OpenCGAResult resetRgaIndexes(String studyStr, String token) thro OpenCGAResult result; try { - authorizationManager.isOwnerOrAdmin(study.getUid(), userId); - result = sampleDBAdaptor.setRgaIndexes(study.getUid(), new RgaIndex(RgaIndex.Status.NOT_INDEXED, TimeUtils.getTime())); + long studyId = study.getUid(); + authorizationManager.checkIsAtLeastStudyAdministrator(organizationId, studyId, userId); + result = getSampleDBAdaptor(organizationId).setRgaIndexes(study.getUid(), new RgaIndex(RgaIndex.Status.NOT_INDEXED, + TimeUtils.getTime())); - auditManager.audit(userId, Enums.Action.RESET_RGA_INDEXES, Enums.Resource.SAMPLE, "ALL", "", study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.audit(organizationId, userId, Enums.Action.RESET_RGA_INDEXES, Enums.Resource.SAMPLE, "ALL", "", study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { - auditManager.audit(userId, Enums.Action.RESET_RGA_INDEXES, Enums.Resource.SAMPLE, "ALL", "", study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.audit(organizationId, userId, Enums.Action.RESET_RGA_INDEXES, Enums.Resource.SAMPLE, "ALL", "", study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw new CatalogException("Could not reset all sample RGA indexes", e); } @@ -642,8 +683,11 @@ public OpenCGAResult resetRgaIndexes(String studyStr, String token) thro // TODO: This method should be somehow private. This should only be accessible internally. public OpenCGAResult updateRgaIndexes(String studyStr, List samples, RgaIndex rgaIndex, String token) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(token); - Study study = catalogManager.getStudyManager().resolveId(studyStr, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyStr, userId, organizationId); ObjectMap auditParams = new ObjectMap() .append("study", studyStr) @@ -658,7 +702,8 @@ public OpenCGAResult updateRgaIndexes(String studyStr, List samp try { auditManager.initAuditBatch(operationUuid); - authorizationManager.isOwnerOrAdmin(study.getUid(), userId); + long studyId = study.getUid(); + authorizationManager.checkIsAtLeastStudyAdministrator(organizationId, studyId, userId); ParamUtils.checkNotEmptyArray(samples, "samples"); ParamUtils.checkObj(rgaIndex, "RgaIndex"); @@ -666,8 +711,8 @@ public OpenCGAResult updateRgaIndexes(String studyStr, List samp rgaIndex.setDate(TimeUtils.getTime()); - sampleResult = internalGet(study.getUid(), samples, INCLUDE_SAMPLE_IDS, userId, false); - result = sampleDBAdaptor.setRgaIndexes(study.getUid(), + sampleResult = internalGet(organizationId, study.getUid(), samples, INCLUDE_SAMPLE_IDS, userId, false); + result = getSampleDBAdaptor(organizationId).setRgaIndexes(study.getUid(), sampleResult.getResults().stream().map(Sample::getUid).collect(Collectors.toList()), rgaIndex); for (Sample sample : sampleResult.getResults()) { @@ -688,54 +733,61 @@ public OpenCGAResult updateRgaIndexes(String studyStr, List samp } throw new CatalogException("Could not reset all sample RGA indexes", e); } finally { - auditManager.finishAuditBatch(operationUuid); + auditManager.finishAuditBatch(organizationId, operationUuid); } return result; } - public OpenCGAResult updateSampleInternalVariantIndex(Sample sample, SampleInternalVariantIndex index, String token) - throws CatalogException { - return updateSampleInternalVariant(sample, index, SampleDBAdaptor.QueryParams.INTERNAL_VARIANT_INDEX.key(), token); + public OpenCGAResult updateSampleInternalVariantIndex(String studyFqn, Sample sample, SampleInternalVariantIndex index, + String token) throws CatalogException { + return updateSampleInternalVariant(studyFqn, sample, index, SampleDBAdaptor.QueryParams.INTERNAL_VARIANT_INDEX.key(), token); } - public OpenCGAResult updateSampleInternalVariantSecondarySampleIndex( - Sample sample, SampleInternalVariantSecondarySampleIndex index, String token) + public OpenCGAResult updateSampleInternalVariantSecondarySampleIndex(String studyFqn, Sample sample, + SampleInternalVariantSecondarySampleIndex index, String token) throws CatalogException { - return updateSampleInternalVariant(sample, index, Arrays.asList( + return updateSampleInternalVariant(studyFqn, sample, index, Arrays.asList( SampleDBAdaptor.QueryParams.INTERNAL_VARIANT_SECONDARY_SAMPLE_INDEX.key(), SampleDBAdaptor.QueryParams.INTERNAL_VARIANT_GENOTYPE_INDEX.key()), token); } public OpenCGAResult updateSampleInternalVariantAnnotationIndex( - Sample sample, SampleInternalVariantAnnotationIndex index, String token) throws CatalogException { - return updateSampleInternalVariant(sample, index, SampleDBAdaptor.QueryParams.INTERNAL_VARIANT_ANNOTATION_INDEX.key(), token); + String studyFqn, Sample sample, SampleInternalVariantAnnotationIndex index, String token) throws CatalogException { + return updateSampleInternalVariant(studyFqn, sample, index, + SampleDBAdaptor.QueryParams.INTERNAL_VARIANT_ANNOTATION_INDEX.key(), token); } - public OpenCGAResult updateSampleInternalVariantSecondaryAnnotationIndex( - Sample sample, SampleInternalVariantSecondaryAnnotationIndex index, String token) throws CatalogException { - return updateSampleInternalVariant(sample, index, + public OpenCGAResult updateSampleInternalVariantSecondaryAnnotationIndex(String studyFqn, Sample sample, + SampleInternalVariantSecondaryAnnotationIndex index, + String token) throws CatalogException { + return updateSampleInternalVariant(studyFqn, sample, index, SampleDBAdaptor.QueryParams.INTERNAL_VARIANT_SECONDARY_ANNOTATION_INDEX.key(), token); } - private OpenCGAResult updateSampleInternalVariant(Sample sample, Object value, String fieldKey, String token) + private OpenCGAResult updateSampleInternalVariant(String studyFqn, Sample sample, Object value, String fieldKey, String token) throws CatalogException { - return updateSampleInternalVariant(sample, value, Collections.singletonList(fieldKey), token); + return updateSampleInternalVariant(studyFqn, sample, value, Collections.singletonList(fieldKey), token); } - private OpenCGAResult updateSampleInternalVariant(Sample sample, Object value, List fieldKeys, String token) - throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyDBAdaptor.get(sample.getStudyUid(), StudyManager.INCLUDE_STUDY_IDS).first(); + private OpenCGAResult updateSampleInternalVariant(String studyFqn, Sample sample, Object value, List fieldKeys, + String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn catalogFqn = CatalogFqn.extractFqnFromStudy(studyFqn, tokenPayload); + String organizationId = catalogFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = getStudyDBAdaptor(organizationId).get(sample.getStudyUid(), StudyManager.INCLUDE_STUDY_IDS).first(); ObjectMap auditParams = new ObjectMap() + .append("studyFqn", studyFqn) .append("sample", sample.getId()) .append("token", token); for (String fieldKey : fieldKeys) { auditParams.append(fieldKey, value); } - authorizationManager.isOwnerOrAdmin(study.getUid(), userId); + long studyId = study.getUid(); + authorizationManager.checkIsAtLeastStudyAdministrator(organizationId, studyId, userId); ObjectMap params; try { @@ -747,9 +799,9 @@ private OpenCGAResult updateSampleInternalVariant(Sample sample, Object value } catch (JsonProcessingException e) { throw new CatalogException("Cannot parse SampleInternalVariant object: " + e.getMessage(), e); } - OpenCGAResult update = sampleDBAdaptor.update(sample.getUid(), params, QueryOptions.empty()); - auditManager.audit(userId, Enums.Action.UPDATE_INTERNAL, Enums.Resource.SAMPLE, sample.getId(), sample.getUuid(), study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + OpenCGAResult update = getSampleDBAdaptor(organizationId).update(sample.getUid(), params, QueryOptions.empty()); + auditManager.audit(organizationId, userId, Enums.Action.UPDATE_INTERNAL, Enums.Resource.SAMPLE, sample.getId(), sample.getUuid(), + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return new OpenCGAResult<>(update.getTime(), update.getEvents(), 1, Collections.emptyList(), 1); } @@ -764,14 +816,15 @@ public OpenCGAResult updateAnnotationSet(String studyStr, String sampleS return update(studyStr, sampleStr, sampleUpdateParams, options, token); } - public OpenCGAResult addAnnotationSet(String studyStr, String sampleStr, AnnotationSet annotationSet, QueryOptions options, - String token) throws CatalogException { + public OpenCGAResult addAnnotationSet(String studyStr, String sampleStr, AnnotationSet annotationSet, + QueryOptions options, String token) throws CatalogException { return addAnnotationSets(studyStr, sampleStr, Collections.singletonList(annotationSet), options, token); } public OpenCGAResult addAnnotationSets(String studyStr, String sampleStr, List annotationSetList, QueryOptions options, String token) throws CatalogException { - return updateAnnotationSet(studyStr, sampleStr, annotationSetList, ParamUtils.BasicUpdateAction.ADD, options, token); + return updateAnnotationSet(studyStr, sampleStr, annotationSetList, ParamUtils.BasicUpdateAction.ADD, options, + token); } public OpenCGAResult removeAnnotationSet(String studyStr, String sampleStr, String annotationSetId, QueryOptions options, @@ -785,7 +838,14 @@ public OpenCGAResult removeAnnotationSets(String studyStr, String sample .stream() .map(id -> new AnnotationSet().setId(id)) .collect(Collectors.toList()); - return updateAnnotationSet(studyStr, sampleStr, annotationSetList, ParamUtils.BasicUpdateAction.REMOVE, options, token); + return updateAnnotationSet(studyStr, sampleStr, annotationSetList, ParamUtils.BasicUpdateAction.REMOVE, options, + token); + } + + public OpenCGAResult removeAnnotations(String studyStr, String sampleStr, String annotationSetId, List annotations, + QueryOptions options, String token) throws CatalogException { + return updateAnnotations(studyStr, sampleStr, annotationSetId, + new ObjectMap("remove", StringUtils.join(annotations, ",")), ParamUtils.CompleteUpdateAction.REMOVE, options, token); } public OpenCGAResult updateAnnotations(String studyStr, String sampleStr, String annotationSetId, @@ -802,25 +862,19 @@ public OpenCGAResult updateAnnotations(String studyStr, String sampleStr return update(studyStr, sampleStr, sampleUpdateParams, options, token); } - public OpenCGAResult removeAnnotations(String studyStr, String sampleStr, String annotationSetId, List annotations, - QueryOptions options, String token) throws CatalogException { - return updateAnnotations(studyStr, sampleStr, annotationSetId, new ObjectMap("remove", StringUtils.join(annotations, ",")), - ParamUtils.CompleteUpdateAction.REMOVE, options, token); - } - public OpenCGAResult resetAnnotations(String studyStr, String sampleStr, String annotationSetId, List annotations, QueryOptions options, String token) throws CatalogException { - return updateAnnotations(studyStr, sampleStr, annotationSetId, new ObjectMap("reset", StringUtils.join(annotations, ",")), - ParamUtils.CompleteUpdateAction.RESET, options, token); + return updateAnnotations(studyStr, sampleStr, annotationSetId, + new ObjectMap("reset", StringUtils.join(annotations, ",")), ParamUtils.CompleteUpdateAction.RESET, options, token); } - private void checkSampleCanBeDeleted(long studyId, Sample sample, boolean force) throws CatalogException { + private void checkSampleCanBeDeleted(String organizationId, long studyId, Sample sample, boolean force) throws CatalogException { String msg = "Could not delete sample '" + sample.getId() + "'. "; // Look for files related with the sample Query query = new Query() .append(FileDBAdaptor.QueryParams.SAMPLE_IDS.key(), sample.getId()) .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), studyId); - DBIterator fileIterator = fileDBAdaptor.iterator(query, QueryOptions.empty()); + DBIterator fileIterator = getFileDBAdaptor(organizationId).iterator(query, QueryOptions.empty()); List errorFiles = new ArrayList<>(); while (fileIterator.hasNext()) { File file = fileIterator.next(); @@ -845,7 +899,7 @@ private void checkSampleCanBeDeleted(long studyId, Sample sample, boolean force) query = new Query() .append(CohortDBAdaptor.QueryParams.SAMPLE_UIDS.key(), sample.getUid()) .append(CohortDBAdaptor.QueryParams.STUDY_UID.key(), studyId); - DBIterator cohortIterator = cohortDBAdaptor.iterator(query, QueryOptions.empty()); + DBIterator cohortIterator = getCohortDBAdaptor(organizationId).iterator(query, QueryOptions.empty()); List errorCohorts = new ArrayList<>(); boolean associatedToDefaultCohort = false; while (cohortIterator.hasNext()) { @@ -882,8 +936,9 @@ private void checkSampleCanBeDeleted(long studyId, Sample sample, boolean force) query = new Query() .append(IndividualDBAdaptor.QueryParams.SAMPLE_UIDS.key(), sample.getUid()) .append(IndividualDBAdaptor.QueryParams.STUDY_UID.key(), studyId); - OpenCGAResult individualDataResult = individualDBAdaptor.get(query, new QueryOptions(QueryOptions.INCLUDE, - Arrays.asList(IndividualDBAdaptor.QueryParams.UID.key(), IndividualDBAdaptor.QueryParams.ID.key()))); + OpenCGAResult individualDataResult = getIndividualDBAdaptor(organizationId).get(query, + new QueryOptions(QueryOptions.INCLUDE, + Arrays.asList(IndividualDBAdaptor.QueryParams.UID.key(), IndividualDBAdaptor.QueryParams.ID.key()))); if (individualDataResult.getNumResults() > 0) { throw new CatalogException(msg + "Sample is associated with individual '" + individualDataResult.first().getId() + "'."); } @@ -899,8 +954,11 @@ public OpenCGAResult update(String studyStr, Query query, SampleUpdatePa QueryOptions options, String token) throws CatalogException { Query finalQuery = new Query(ParamUtils.defaultObject(query, Query::new)); - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId, StudyManager.INCLUDE_VARIABLE_SET); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, StudyManager.INCLUDE_VARIABLE_SET, userId, organizationId); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -921,13 +979,13 @@ public OpenCGAResult update(String studyStr, Query query, SampleUpdatePa DBIterator iterator; try { - fixQueryObject(study, finalQuery, userId); + fixQueryObject(organizationId, study, finalQuery, userId); finalQuery.append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - iterator = sampleDBAdaptor.iterator(study.getUid(), finalQuery, INCLUDE_SAMPLE_IDS, userId); + iterator = getSampleDBAdaptor(organizationId).iterator(study.getUid(), finalQuery, INCLUDE_SAMPLE_IDS, userId); } catch (CatalogException e) { - auditManager.auditUpdate(operationId, userId, Enums.Resource.SAMPLE, "", "", study.getId(), study.getUuid(), + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.SAMPLE, "", "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -937,30 +995,33 @@ public OpenCGAResult update(String studyStr, Query query, SampleUpdatePa while (iterator.hasNext()) { Sample sample = iterator.next(); try { - OpenCGAResult updateResult = update(study, sample, updateParams, options, userId); + OpenCGAResult updateResult = update(organizationId, study, sample, updateParams, options, userId); result.append(updateResult); - auditManager.auditUpdate(operationId, userId, Enums.Resource.SAMPLE, sample.getId(), sample.getUuid(), study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.SAMPLE, sample.getId(), sample.getUuid(), + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { Event event = new Event(Event.Type.ERROR, sample.getId(), e.getMessage()); result.getEvents().add(event); result.setNumErrors(result.getNumErrors() + 1); logger.error("Could not update sample {}: {}", sample.getId(), e.getMessage(), e); - auditManager.auditUpdate(operationId, userId, Enums.Resource.SAMPLE, sample.getId(), sample.getUuid(), study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.SAMPLE, sample.getId(), sample.getUuid(), + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); } } - auditManager.finishAuditBatch(operationId); + auditManager.finishAuditBatch(organizationId, operationId); return endResult(result, ignoreException); } public OpenCGAResult update(String studyStr, String sampleId, SampleUpdateParams updateParams, QueryOptions options, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId, StudyManager.INCLUDE_VARIABLE_SET); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, StudyManager.INCLUDE_VARIABLE_SET, userId, organizationId); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -981,7 +1042,7 @@ public OpenCGAResult update(String studyStr, String sampleId, SampleUpda OpenCGAResult result = OpenCGAResult.empty(); String sampleUuid = ""; try { - OpenCGAResult internalResult = internalGet(study.getUid(), sampleId, INCLUDE_SAMPLE_IDS, userId); + OpenCGAResult internalResult = internalGet(organizationId, study.getUid(), sampleId, INCLUDE_SAMPLE_IDS, userId); if (internalResult.getNumResults() == 0) { throw new CatalogException("Sample '" + sampleId + "' not found"); } @@ -991,18 +1052,18 @@ public OpenCGAResult update(String studyStr, String sampleId, SampleUpda sampleId = sample.getId(); sampleUuid = sample.getUuid(); - OpenCGAResult updateResult = update(study, sample, updateParams, options, userId); + OpenCGAResult updateResult = update(organizationId, study, sample, updateParams, options, userId); result.append(updateResult); - auditManager.auditUpdate(operationId, userId, Enums.Resource.SAMPLE, sample.getId(), sample.getUuid(), study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.SAMPLE, sample.getId(), sample.getUuid(), + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { Event event = new Event(Event.Type.ERROR, sampleId, e.getMessage()); result.getEvents().add(event); result.setNumErrors(result.getNumErrors() + 1); logger.error("Could not update sample {}: {}", sampleId, e.getMessage(), e); - auditManager.auditUpdate(operationId, userId, Enums.Resource.SAMPLE, sampleId, sampleUuid, study.getId(), + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.SAMPLE, sampleId, sampleUuid, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -1013,7 +1074,8 @@ public OpenCGAResult update(String studyStr, String sampleId, SampleUpda /** * Update Sample from catalog. * - * @param studyStr Study id in string format. Could be one of [id|user@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy]. + * @param studyStr Study id in string format. Could be one of + * [id|organization@aliasProject:aliasStudy|aliasProject:aliasStudy|aliasStudy] * @param sampleIds List of Sample ids. Could be either the id or uuid. * @param updateParams Data model filled only with the parameters to be updated. * @param options QueryOptions object. @@ -1027,10 +1089,13 @@ public OpenCGAResult update(String studyStr, List sampleIds, Sam return update(studyStr, sampleIds, updateParams, false, options, token); } - public OpenCGAResult update(String studyStr, List sampleIds, SampleUpdateParams updateParams, boolean ignoreException, - QueryOptions options, String token) throws CatalogException { - String userId = userManager.getUserId(token); - Study study = studyManager.resolveId(studyStr, userId, StudyManager.INCLUDE_VARIABLE_SET); + public OpenCGAResult update(String studyStr, List sampleIds, SampleUpdateParams updateParams, + boolean ignoreException, QueryOptions options, String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, StudyManager.INCLUDE_VARIABLE_SET, userId, organizationId); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); @@ -1056,7 +1121,7 @@ public OpenCGAResult update(String studyStr, List sampleIds, Sam String sampleUuid = ""; try { - OpenCGAResult internalResult = internalGet(study.getUid(), id, INCLUDE_SAMPLE_IDS, userId); + OpenCGAResult internalResult = internalGet(organizationId, study.getUid(), id, INCLUDE_SAMPLE_IDS, userId); if (internalResult.getNumResults() == 0) { throw new CatalogException("Sample '" + id + "' not found"); } @@ -1066,28 +1131,28 @@ public OpenCGAResult update(String studyStr, List sampleIds, Sam sampleId = sample.getId(); sampleUuid = sample.getUuid(); - OpenCGAResult updateResult = update(study, sample, updateParams, options, userId); + OpenCGAResult updateResult = update(organizationId, study, sample, updateParams, options, userId); result.append(updateResult); - auditManager.auditUpdate(operationId, userId, Enums.Resource.SAMPLE, sample.getId(), sample.getUuid(), study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.SAMPLE, sample.getId(), sample.getUuid(), + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { Event event = new Event(Event.Type.ERROR, sampleId, e.getMessage()); result.getEvents().add(event); result.setNumErrors(result.getNumErrors() + 1); logger.error("Could not update sample {}: {}", sampleId, e.getMessage(), e); - auditManager.auditUpdate(operationId, userId, Enums.Resource.SAMPLE, sampleId, sampleUuid, study.getId(), + auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.SAMPLE, sampleId, sampleUuid, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); } } - auditManager.finishAuditBatch(operationId); + auditManager.finishAuditBatch(organizationId, operationId); return endResult(result, ignoreException); } - private OpenCGAResult update(Study study, Sample sample, SampleUpdateParams updateParams, QueryOptions options, String userId) - throws CatalogException { + private OpenCGAResult update(String organizationId, Study study, Sample sample, SampleUpdateParams updateParams, QueryOptions options, + String userId) throws CatalogException { options = ParamUtils.defaultObject(options, QueryOptions::new); SampleUpdateParams updateParamsClone; @@ -1132,13 +1197,13 @@ private OpenCGAResult update(Study study, Sample sample, SampleUpdateParams upda // Check permissions... // Only check write annotation permissions if the user wants to update the annotation sets if (updateParamsClone != null && updateParamsClone.getAnnotationSets() != null) { - authorizationManager.checkSamplePermission(study.getUid(), sample.getUid(), userId, + authorizationManager.checkSamplePermission(organizationId, study.getUid(), sample.getUid(), userId, SamplePermissions.WRITE_ANNOTATIONS); } // Only check update permissions if the user wants to update anything apart from the annotation sets if ((parameters.size() == 1 && !parameters.containsKey(SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key())) || parameters.size() > 1) { - authorizationManager.checkSamplePermission(study.getUid(), sample.getUid(), userId, + authorizationManager.checkSamplePermission(organizationId, study.getUid(), sample.getUid(), userId, SamplePermissions.WRITE); } @@ -1148,8 +1213,8 @@ private OpenCGAResult update(Study study, Sample sample, SampleUpdateParams upda if (updateParamsClone != null && StringUtils.isNotEmpty(updateParamsClone.getIndividualId())) { // Check individual id exists - OpenCGAResult individualDataResult = catalogManager.getIndividualManager().internalGet(study.getUid(), - updateParamsClone.getIndividualId(), IndividualManager.INCLUDE_INDIVIDUAL_IDS, userId); + OpenCGAResult individualDataResult = catalogManager.getIndividualManager().internalGet(organizationId, + study.getUid(), updateParamsClone.getIndividualId(), IndividualManager.INCLUDE_INDIVIDUAL_IDS, userId); if (individualDataResult.getNumResults() == 0) { throw new CatalogException("Individual '" + updateParamsClone.getIndividualId() + "' not found."); } @@ -1158,12 +1223,14 @@ private OpenCGAResult update(Study study, Sample sample, SampleUpdateParams upda parameters.put(SampleDBAdaptor.QueryParams.INDIVIDUAL_ID.key(), individualDataResult.first().getId()); } - checkUpdateAnnotations(study, sample, parameters, options, VariableSet.AnnotableDataModels.SAMPLE, sampleDBAdaptor, userId); + checkUpdateAnnotations(organizationId, study, sample, parameters, options, VariableSet.AnnotableDataModels.SAMPLE, + getSampleDBAdaptor(organizationId), userId); - OpenCGAResult update = sampleDBAdaptor.update(sample.getUid(), parameters, study.getVariableSets(), options); + OpenCGAResult update = getSampleDBAdaptor(organizationId).update(sample.getUid(), parameters, study.getVariableSets(), + options); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { // Fetch updated sample - OpenCGAResult queryResult = sampleDBAdaptor.get(study.getUid(), + OpenCGAResult queryResult = getSampleDBAdaptor(organizationId).get(study.getUid(), new Query(SampleDBAdaptor.QueryParams.UID.key(), sample.getUid()), options, userId); update.setResults(queryResult.getResults()); } @@ -1171,19 +1238,22 @@ private OpenCGAResult update(Study study, Sample sample, SampleUpdateParams upda } @Override - public OpenCGAResult rank(String studyStr, Query query, String field, int numResults, boolean asc, String sessionId) + public OpenCGAResult rank(String studyStr, Query query, String field, int numResults, boolean asc, String token) throws CatalogException { query = ParamUtils.defaultObject(query, Query::new); ParamUtils.checkObj(field, "field"); - ParamUtils.checkObj(sessionId, "sessionId"); + ParamUtils.checkObj(token, "token"); - String userId = userManager.getUserId(sessionId); - Study study = catalogManager.getStudyManager().resolveId(studyStr, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyStr, userId, organizationId); // Fix query if it contains any annotation - AnnotationUtils.fixQueryAnnotationSearch(study, userId, query, authorizationManager); + AnnotationUtils.fixQueryAnnotationSearch(organizationId, study, userId, query, authorizationManager); - authorizationManager.checkStudyPermission(study.getUid(), userId, StudyPermissions.Permissions.VIEW_SAMPLES); + authorizationManager.checkStudyPermission(organizationId, study.getUid(), userId, StudyPermissions.Permissions.VIEW_SAMPLES); // TODO: In next release, we will have to check the count parameter from the queryOptions object. boolean count = true; @@ -1191,14 +1261,14 @@ public OpenCGAResult rank(String studyStr, Query query, String field, int numRes OpenCGAResult queryResult = null; if (count) { // We do not need to check for permissions when we show the count of files - queryResult = sampleDBAdaptor.rank(query, field, numResults, asc); + queryResult = getSampleDBAdaptor(organizationId).rank(query, field, numResults, asc); } return ParamUtils.defaultObject(queryResult, OpenCGAResult::new); } @Override - public OpenCGAResult groupBy(@Nullable String studyStr, Query query, List fields, QueryOptions options, String sessionId) + public OpenCGAResult groupBy(@Nullable String studyStr, Query query, List fields, QueryOptions options, String token) throws CatalogException { query = ParamUtils.defaultObject(query, Query::new); options = ParamUtils.defaultObject(options, QueryOptions::new); @@ -1206,38 +1276,42 @@ public OpenCGAResult groupBy(@Nullable String studyStr, Query query, List> getAcls(String studyId, List sampleList, String member, - boolean ignoreException, String token) - throws CatalogException { - return getAcls(studyId, sampleList, StringUtils.isNotEmpty(member) ? Collections.singletonList(member) : Collections.emptyList(), - ignoreException, token); + boolean ignoreException, String token) throws CatalogException { + return getAcls(studyId, sampleList, + StringUtils.isNotEmpty(member) ? Collections.singletonList(member) : Collections.emptyList(), ignoreException, token); } - public OpenCGAResult> getAcls(String studyId, List sampleList, - List members, boolean ignoreException, - String token) throws CatalogException { - String user = userManager.getUserId(token); - Study study = studyManager.resolveId(studyId, user); + public OpenCGAResult> getAcls(String studyStr, List sampleList, List members, + boolean ignoreException, String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, userId, organizationId); String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); ObjectMap auditParams = new ObjectMap() - .append("studyId", studyId) + .append("studyId", studyStr) .append("sampleList", sampleList) .append("members", members) .append("ignoreException", ignoreException) @@ -1247,7 +1321,8 @@ public OpenCGAResult> getAcls(String studyId, Li Map missingMap = new HashMap<>(); try { auditManager.initAuditBatch(operationId); - InternalGetDataResult queryResult = internalGet(study.getUid(), sampleList, INCLUDE_SAMPLE_IDS, user, ignoreException); + InternalGetDataResult queryResult = internalGet(organizationId, study.getUid(), sampleList, INCLUDE_SAMPLE_IDS, userId, + ignoreException); if (queryResult.getMissing() != null) { missingMap = queryResult.getMissing().stream() @@ -1256,11 +1331,11 @@ public OpenCGAResult> getAcls(String studyId, Li List sampleUids = queryResult.getResults().stream().map(Sample::getUid).collect(Collectors.toList()); if (CollectionUtils.isNotEmpty(members)) { - sampleAcls = authorizationManager.getAcl(user, study.getUid(), sampleUids, members, Enums.Resource.SAMPLE, - SamplePermissions.class); + sampleAcls = authorizationManager.getAcl(organizationId, study.getUid(), sampleUids, members, Enums.Resource.SAMPLE, + SamplePermissions.class, userId); } else { - sampleAcls = authorizationManager.getAcl(user, study.getUid(), sampleUids, Enums.Resource.SAMPLE, - SamplePermissions.class); + sampleAcls = authorizationManager.getAcl(organizationId, study.getUid(), sampleUids, Enums.Resource.SAMPLE, + SamplePermissions.class, userId); } // Include non-existing samples to the result list @@ -1271,14 +1346,14 @@ public OpenCGAResult> getAcls(String studyId, Li if (!missingMap.containsKey(sampleId)) { Sample sample = queryResult.getResults().get(counter); resultList.add(sampleAcls.getResults().get(counter)); - auditManager.audit(operationId, user, Enums.Action.FETCH_ACLS, Enums.Resource.SAMPLE, sample.getId(), + auditManager.audit(organizationId, operationId, userId, Enums.Action.FETCH_ACLS, Enums.Resource.SAMPLE, sample.getId(), sample.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS), new ObjectMap()); counter++; } else { resultList.add(new AclEntryList<>()); eventList.add(new Event(Event.Type.ERROR, sampleId, missingMap.get(sampleId).getErrorMsg())); - auditManager.audit(operationId, user, Enums.Action.FETCH_ACLS, Enums.Resource.SAMPLE, sampleId, "", + auditManager.audit(organizationId, operationId, userId, Enums.Action.FETCH_ACLS, Enums.Resource.SAMPLE, sampleId, "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(0, "", missingMap.get(sampleId).getErrorMsg())), new ObjectMap()); } @@ -1290,7 +1365,7 @@ public OpenCGAResult> getAcls(String studyId, Li sampleAcls.setEvents(eventList); } catch (CatalogException e) { for (String sampleId : sampleList) { - auditManager.audit(operationId, user, Enums.Action.FETCH_ACLS, Enums.Resource.SAMPLE, sampleId, "", + auditManager.audit(organizationId, operationId, userId, Enums.Action.FETCH_ACLS, Enums.Resource.SAMPLE, sampleId, "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError()), new ObjectMap()); } @@ -1303,21 +1378,23 @@ public OpenCGAResult> getAcls(String studyId, Li } } } finally { - auditManager.finishAuditBatch(operationId); + auditManager.finishAuditBatch(organizationId, operationId); } return sampleAcls; } - public OpenCGAResult> updateAcl(String studyId, List sampleStringList, - String memberList, SampleAclParams sampleAclParams, - ParamUtils.AclAction action, String token) - throws CatalogException { - String user = userManager.getUserId(token); - Study study = studyManager.resolveId(studyId, user); + public OpenCGAResult> updateAcl(String studyStr, List sampleStringList, String memberList, + SampleAclParams sampleAclParams, ParamUtils.AclAction action, + String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = studyManager.resolveId(studyStr, userId, organizationId); ObjectMap auditParams = new ObjectMap() - .append("studyId", studyId) + .append("studyId", studyStr) .append("sampleStringList", sampleStringList) .append("memberList", memberList) .append("sampleAclParams", sampleAclParams) @@ -1358,7 +1435,8 @@ public OpenCGAResult> updateAcl(String studyId, if (StringUtils.isNotEmpty(sampleAclParams.getIndividual())) { Query query = new Query(IndividualDBAdaptor.QueryParams.ID.key(), sampleAclParams.getIndividual()); QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, IndividualDBAdaptor.QueryParams.SAMPLES.key()); - OpenCGAResult indDataResult = catalogManager.getIndividualManager().search(studyId, query, options, token); + OpenCGAResult indDataResult = catalogManager.getIndividualManager().search(studyStr, query, + options, token); Set sampleSet = new HashSet<>(); for (Individual individual : indDataResult.getResults()) { @@ -1369,7 +1447,7 @@ public OpenCGAResult> updateAcl(String studyId, } if (StringUtils.isNotEmpty(sampleAclParams.getFamily())) { - OpenCGAResult familyDataResult = catalogManager.getFamilyManager().get(studyId, + OpenCGAResult familyDataResult = catalogManager.getFamilyManager().get(studyStr, Arrays.asList(sampleAclParams.getFamily().split(",")), FamilyManager.INCLUDE_FAMILY_MEMBERS, token); Set sampleSet = new HashSet<>(); @@ -1389,8 +1467,8 @@ public OpenCGAResult> updateAcl(String studyId, if (StringUtils.isNotEmpty(sampleAclParams.getFile())) { // // Obtain the samples of the files QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, FileDBAdaptor.QueryParams.SAMPLE_IDS.key()); - OpenCGAResult fileDataResult = catalogManager.getFileManager().internalGet(study.getUid(), - Arrays.asList(StringUtils.split(sampleAclParams.getFile(), ",")), options, user, false); + OpenCGAResult fileDataResult = catalogManager.getFileManager().internalGet(organizationId, study.getUid(), + Arrays.asList(StringUtils.split(sampleAclParams.getFile(), ",")), options, userId, false); Set sampleSet = new HashSet<>(); for (File file : fileDataResult.getResults()) { @@ -1403,7 +1481,8 @@ public OpenCGAResult> updateAcl(String studyId, if (StringUtils.isNotEmpty(sampleAclParams.getCohort())) { Query query = new Query(CohortDBAdaptor.QueryParams.ID.key(), sampleAclParams.getCohort()); QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, CohortDBAdaptor.QueryParams.SAMPLES.key()); - OpenCGAResult cohortDataResult = catalogManager.getCohortManager().search(studyId, query, options, token); + OpenCGAResult cohortDataResult = catalogManager.getCohortManager().search(studyStr, query, options, + token); Set sampleSet = new HashSet<>(); for (Cohort cohort : cohortDataResult.getResults()) { @@ -1413,8 +1492,8 @@ public OpenCGAResult> updateAcl(String studyId, sampleStringList.addAll(sampleSet); } - sampleList = internalGet(study.getUid(), sampleStringList, INCLUDE_SAMPLE_IDS, user, false).getResults(); - authorizationManager.checkCanAssignOrSeePermissions(study.getUid(), user); + sampleList = internalGet(organizationId, study.getUid(), sampleStringList, INCLUDE_SAMPLE_IDS, userId, false).getResults(); + authorizationManager.checkCanAssignOrSeePermissions(organizationId, study.getUid(), userId); // Validate that the members are actually valid members if (memberList != null && !memberList.isEmpty()) { @@ -1422,17 +1501,17 @@ public OpenCGAResult> updateAcl(String studyId, } else { members = Collections.emptyList(); } - checkMembers(study.getUid(), members); + checkMembers(organizationId, study.getUid(), members); authorizationManager.checkNotAssigningPermissionsToAdminsGroup(members); } catch (CatalogException e) { if (sampleStringList != null) { for (String sampleId : sampleStringList) { - auditManager.audit(operationId, user, Enums.Action.UPDATE_ACLS, Enums.Resource.SAMPLE, sampleId, "", + auditManager.audit(organizationId, operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.SAMPLE, sampleId, "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError()), new ObjectMap()); } } - auditManager.finishAuditBatch(operationId); + auditManager.finishAuditBatch(organizationId, operationId); throw e; } @@ -1453,25 +1532,25 @@ public OpenCGAResult> updateAcl(String studyId, try { switch (action) { case SET: - authorizationManager.setAcls(study.getUid(), members, aclParamsList); + authorizationManager.setAcls(organizationId, study.getUid(), members, aclParamsList); break; case ADD: - authorizationManager.addAcls(study.getUid(), members, aclParamsList); + authorizationManager.addAcls(organizationId, study.getUid(), members, aclParamsList); break; case REMOVE: - authorizationManager.removeAcls(members, aclParamsList); + authorizationManager.removeAcls(organizationId, members, aclParamsList); break; case RESET: for (AuthorizationManager.CatalogAclParams aclParam : aclParamsList) { aclParam.setPermissions(null); } - authorizationManager.removeAcls(members, aclParamsList); + authorizationManager.removeAcls(organizationId, members, aclParamsList); break; default: throw new CatalogException("Unexpected error occurred. No valid action found."); } - OpenCGAResult> queryResults = authorizationManager.getAcls(study.getUid(), + OpenCGAResult> queryResults = authorizationManager.getAcls(organizationId, study.getUid(), sampleUids, members, Enums.Resource.SAMPLE, SamplePermissions.class); for (int i = 0; i < queryResults.getResults().size(); i++) { @@ -1480,14 +1559,14 @@ public OpenCGAResult> updateAcl(String studyId, aclResultList.append(queryResults); for (Sample sample : batchSampleList) { - auditManager.audit(operationId, user, Enums.Action.UPDATE_ACLS, Enums.Resource.SAMPLE, sample.getId(), + auditManager.audit(organizationId, operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.SAMPLE, sample.getId(), sample.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS), new ObjectMap()); } } catch (CatalogException e) { // Process current batch for (Sample sample : batchSampleList) { - auditManager.audit(operationId, user, Enums.Action.UPDATE_ACLS, Enums.Resource.SAMPLE, sample.getId(), + auditManager.audit(organizationId, operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.SAMPLE, sample.getId(), sample.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError()), new ObjectMap()); } @@ -1495,66 +1574,29 @@ public OpenCGAResult> updateAcl(String studyId, // Process remaining unprocessed batches while (numProcessed < sampleList.size()) { Sample sample = sampleList.get(numProcessed); - auditManager.audit(operationId, user, Enums.Action.UPDATE_ACLS, Enums.Resource.SAMPLE, sample.getId(), + auditManager.audit(organizationId, operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.SAMPLE, sample.getId(), sample.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError()), new ObjectMap()); } - auditManager.finishAuditBatch(operationId); + auditManager.finishAuditBatch(organizationId, operationId); throw e; } } while (numProcessed < sampleList.size()); - auditManager.finishAuditBatch(operationId); + auditManager.finishAuditBatch(organizationId, operationId); return aclResultList; } - public DataResult facet(String studyId, Query query, QueryOptions options, boolean defaultStats, String token) + private List getIndividualsUidsFromSampleUids(String organizationId, long studyUid, List sampleUids) throws CatalogException { - String userId = userManager.getUserId(token); - // We need to add variableSets and groups to avoid additional queries as it will be used in the catalogSolrManager - Study study = catalogManager.getStudyManager().resolveId(studyId, userId, new QueryOptions(QueryOptions.INCLUDE, - Arrays.asList(StudyDBAdaptor.QueryParams.VARIABLE_SET.key(), StudyDBAdaptor.QueryParams.GROUPS.key()))); - - ParamUtils.defaultObject(query, Query::new); - ParamUtils.defaultObject(options, QueryOptions::new); - - ObjectMap auditParams = new ObjectMap() - .append("studyId", studyId) - .append("query", new Query(query)) - .append("options", options) - .append("defaultStats", defaultStats) - .append("token", token); - - try { - if (defaultStats || StringUtils.isEmpty(options.getString(QueryOptions.FACET))) { - String facet = options.getString(QueryOptions.FACET); - options.put(QueryOptions.FACET, StringUtils.isNotEmpty(facet) ? defaultFacet + ";" + facet : defaultFacet); - } - AnnotationUtils.fixQueryAnnotationSearch(study, userId, query, authorizationManager); - - try (CatalogSolrManager catalogSolrManager = new CatalogSolrManager(catalogManager)) { - DataResult result = catalogSolrManager.facetedQuery(study, CatalogSolrManager.SAMPLE_SOLR_COLLECTION, query, - options, userId); - - auditManager.auditFacet(userId, Enums.Resource.SAMPLE, study.getId(), study.getUuid(), auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); - return result; - } - } catch (CatalogException e) { - auditManager.auditFacet(userId, Enums.Resource.SAMPLE, study.getId(), study.getUuid(), auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(0, "", e.getMessage()))); - throw e; - } - } - - private List getIndividualsUidsFromSampleUids(long studyUid, List sampleUids) throws CatalogException { // Look for all the individuals owning the samples Query query = new Query() .append(IndividualDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(IndividualDBAdaptor.QueryParams.SAMPLE_UIDS.key(), sampleUids); - OpenCGAResult individualDataResult = individualDBAdaptor.get(query, IndividualManager.INCLUDE_INDIVIDUAL_IDS); + OpenCGAResult individualDataResult = getIndividualDBAdaptor(organizationId).get(query, + IndividualManager.INCLUDE_INDIVIDUAL_IDS); return individualDataResult.getResults().stream().map(Individual::getUid).collect(Collectors.toList()); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/StudyManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/StudyManager.java index 9c22e95f0bd..70f933f6430 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/StudyManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/StudyManager.java @@ -38,6 +38,7 @@ import org.opencb.opencga.catalog.io.IOManager; import org.opencb.opencga.catalog.io.IOManagerFactory; import org.opencb.opencga.catalog.utils.AnnotationUtils; +import org.opencb.opencga.catalog.utils.CatalogFqn; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.catalog.utils.UuidUtils; import org.opencb.opencga.core.api.ParamConstants; @@ -46,6 +47,7 @@ import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.config.storage.SampleIndexConfiguration; import org.opencb.opencga.core.models.AclEntryList; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.audit.AuditRecord; import org.opencb.opencga.core.models.clinical.ClinicalAnalysisPermissions; import org.opencb.opencga.core.models.cohort.CohortPermissions; @@ -57,6 +59,7 @@ import org.opencb.opencga.core.models.file.FileStatus; import org.opencb.opencga.core.models.individual.IndividualPermissions; import org.opencb.opencga.core.models.job.JobPermissions; +import org.opencb.opencga.core.models.organizations.Organization; import org.opencb.opencga.core.models.project.Project; import org.opencb.opencga.core.models.sample.SamplePermissions; import org.opencb.opencga.core.models.study.*; @@ -70,7 +73,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.annotation.Nullable; import java.io.IOException; import java.io.InputStream; import java.net.URI; @@ -84,8 +86,6 @@ import java.util.stream.Collectors; import static org.opencb.opencga.catalog.auth.authorization.CatalogAuthorizationManager.checkPermissions; -import static org.opencb.opencga.core.api.ParamConstants.ADMIN_PROJECT; -import static org.opencb.opencga.core.api.ParamConstants.ADMIN_STUDY; import static org.opencb.opencga.core.common.JacksonUtils.getUpdateObjectMapper; /** @@ -102,9 +102,9 @@ public class StudyManager extends AbstractManager { private static final String USER_PATTERN = "[A-Za-z][[-_.]?[A-Za-z0-9]?]*"; private static final String PROJECT_PATTERN = "[A-Za-z0-9][[-_.]?[A-Za-z0-9]?]*"; private static final String STUDY_PATTERN = "[A-Za-z0-9\\-_.]+|\\*"; - private static final Pattern USER_PROJECT_STUDY_PATTERN = Pattern.compile("^(" + USER_PATTERN + ")@(" + PROJECT_PATTERN + "):(" - + STUDY_PATTERN + ")$"); - private static final Pattern PROJECT_STUDY_PATTERN = Pattern.compile("^(" + PROJECT_PATTERN + "):(" + STUDY_PATTERN + ")$"); + + private static final Pattern ORGANIZATION_PROJECT_STUDY_PATTERN = CatalogFqn.ORGANIZATION_PROJECT_STUDY_PATTERN; + private static final Pattern PROJECT_STUDY_PATTERN = CatalogFqn.PROJECT_STUDY_PATTERN; static final QueryOptions INCLUDE_STUDY_UID = new QueryOptions(QueryOptions.INCLUDE, StudyDBAdaptor.QueryParams.UID.key()); public static final QueryOptions INCLUDE_STUDY_IDS = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList( @@ -129,33 +129,34 @@ public class StudyManager extends AbstractManager { logger = LoggerFactory.getLogger(StudyManager.class); } - public String getProjectId(long studyId) throws CatalogException { - return studyDBAdaptor.getProjectIdByStudyUid(studyId); - } +// public String getProjectId(long studyId) throws CatalogException { +// return studyDBAdaptor.getProjectIdByStudyUid(studyId); +// } + - List resolveIds(List studyList, String userId) throws CatalogException { + List resolveIds(List studyList, String userId, String organizationId) throws CatalogException { if (studyList == null || studyList.isEmpty() || (studyList.size() == 1 && studyList.get(0).endsWith("*"))) { String studyStr = "*"; if (studyList != null && !studyList.isEmpty()) { studyStr = studyList.get(0); } - return smartResolutor(studyStr, userId, INCLUDE_STUDY_IDS).getResults(); + return smartResolutor(studyStr, INCLUDE_STUDY_IDS, userId, organizationId).getResults(); } List returnList = new ArrayList<>(studyList.size()); for (String study : studyList) { - returnList.add(resolveId(study, userId, INCLUDE_STUDY_IDS)); + returnList.add(resolveId(study, INCLUDE_STUDY_IDS, userId, organizationId)); } return returnList; } - Study resolveId(String studyStr, String userId) throws CatalogException { - return resolveId(studyStr, userId, INCLUDE_STUDY_IDS); + Study resolveId(String studyStr, String userId, String organizationId) throws CatalogException { + return resolveId(studyStr, INCLUDE_STUDY_IDS, userId, organizationId); } - Study resolveId(String studyStr, String userId, QueryOptions options) throws CatalogException { - OpenCGAResult studyDataResult = smartResolutor(studyStr, userId, options); + Study resolveId(String studyStr, QueryOptions options, String userId, String organizationId) throws CatalogException { + OpenCGAResult studyDataResult = smartResolutor(studyStr, options, userId, organizationId); if (studyDataResult.getNumResults() > 1) { String studyMessage = ""; @@ -163,14 +164,113 @@ Study resolveId(String studyStr, String userId, QueryOptions options) throws Cat studyMessage = " given '" + studyStr + "'"; } throw new CatalogException("More than one study found" + studyMessage + ". Please, be more specific." - + " The accepted pattern is [ownerId@projectId:studyId]"); + + " The accepted pattern is [" + CatalogFqn.STUDY_FQN_FORMAT + "]"); + } + + return studyDataResult.first(); + } + + Study resolveId(CatalogFqn catalogFqn, QueryOptions options, JwtPayload payload) throws CatalogException { + OpenCGAResult studyDataResult = smartResolutor(catalogFqn, options, payload); + + if (studyDataResult.getNumResults() > 1) { + String studyMessage = ""; + if (StringUtils.isNotEmpty(catalogFqn.getProvidedId())) { + studyMessage = " given '" + catalogFqn.getProvidedId() + "'"; + } + throw new CatalogException("More than one study found" + studyMessage + ". Please, be more specific." + + " The accepted pattern is [" + CatalogFqn.STUDY_FQN_FORMAT + "]"); } return studyDataResult.first(); } - private OpenCGAResult smartResolutor(String studyStr, String userId, QueryOptions options) throws CatalogException { - String owner = null; + /** + * Resolve the study the user wants to access. + * + * @param catalogFqn CatalogFqn object. + * @param options Options to include/exclude Study fields. + * @param payload Token payload containing the user requesting the operation. + * @return An OpenCGAResult with the appropriate study. + * @throws CatalogException if the user cannot access the study, or it is unclear which study they want to access. + */ + private OpenCGAResult smartResolutor(CatalogFqn catalogFqn, QueryOptions options, JwtPayload payload) throws CatalogException { + String userId = payload.getUserId(catalogFqn.getOrganizationId()); + + ParamUtils.checkParameter(userId, "userId"); + ParamUtils.checkParameter(catalogFqn.getOrganizationId(), "organizationId"); + + Query query = new Query(); + QueryOptions queryOptions; + if (options == null) { + queryOptions = new QueryOptions(); + } else { + queryOptions = new QueryOptions(options); + } + + String studyStr; + if (StringUtils.isNotEmpty(catalogFqn.getStudyUuid())) { + query.putIfNotEmpty(StudyDBAdaptor.QueryParams.UUID.key(), catalogFqn.getStudyUuid()); + studyStr = catalogFqn.getStudyUuid(); + } else { + studyStr = catalogFqn.getStudyId(); + if (!"*".equals(catalogFqn.getStudyId())) { + query.putIfNotEmpty(StudyDBAdaptor.QueryParams.ID.key(), catalogFqn.getStudyId()); + } + } + query.putIfNotEmpty(StudyDBAdaptor.QueryParams.PROJECT_ID.key(), catalogFqn.getProjectId()); + + if (!queryOptions.isEmpty()) { + // Ensure at least the ids are included + fixQueryOptions(queryOptions, INCLUDE_STUDY_IDS.getAsStringList(QueryOptions.INCLUDE)); + } + + OpenCGAResult studyDataResult; + if (!payload.getOrganization().equals(catalogFqn.getOrganizationId())) { + // If it is the administrator, we allow it without checking the user anymore + authorizationManager.checkIsOpencgaAdministrator(payload); + studyDataResult = getStudyDBAdaptor(catalogFqn.getOrganizationId()).get(query, queryOptions); + } else { + studyDataResult = getStudyDBAdaptor(catalogFqn.getOrganizationId()).get(query, queryOptions, userId); + if (studyDataResult.getNumResults() == 0) { + studyDataResult = getStudyDBAdaptor(catalogFqn.getOrganizationId()).get(query, queryOptions); + if (studyDataResult.getNumResults() != 0) { + throw CatalogAuthorizationException.denyAny(userId, "view", "study"); + } + } + } + + if (studyDataResult.getNumResults() == 0) { + String studyMessage = ""; + if (StringUtils.isNotEmpty(studyStr)) { + studyMessage = " given '" + studyStr + "'"; + } + throw new CatalogException("No study found" + studyMessage + "."); + } else { + return studyDataResult; + } + } + + /** + * Resolve the study the user wants to access. + * + * @param studyStr Study id or fqn. + * @param options Options to include/exclude Study fields. + * @param userId User requesting the operation. + * @param organizationId Organization to which the user belongs. + * @return An OpenCGAResult with the appropriate study. + * @throws CatalogException if the user cannot access the study, or it is unclear which study they want to access. + */ + private OpenCGAResult smartResolutor(String studyStr, QueryOptions options, String userId, String organizationId) + throws CatalogException { + if (StringUtils.isEmpty(userId)) { + throw new CatalogException("Missing internal mandatory field 'userId'"); + } + if (StringUtils.isEmpty(organizationId)) { + throw new CatalogException("Missing internal mandatory field 'organization'"); + } + + String organizationFqn = null; String project = null; Query query = new Query(); @@ -187,10 +287,10 @@ private OpenCGAResult smartResolutor(String studyStr, String userId, Quer } else { String study; - Matcher matcher = USER_PROJECT_STUDY_PATTERN.matcher(studyStr); + Matcher matcher = ORGANIZATION_PROJECT_STUDY_PATTERN.matcher(studyStr); if (matcher.find()) { - // studyStr contains the full path (owner@project:study) - owner = matcher.group(1); + // studyStr contains the full path (org@project:study) + organizationFqn = matcher.group(1); project = matcher.group(2); study = matcher.group(3); } else { @@ -214,23 +314,27 @@ private OpenCGAResult smartResolutor(String studyStr, String userId, Quer } } - query.putIfNotEmpty(StudyDBAdaptor.QueryParams.OWNER.key(), owner); + if (organizationFqn != null && !organizationId.equals(organizationFqn) + && !ParamConstants.ADMIN_ORGANIZATION.equals(organizationId)) { + logger.error("User '{}' belonging to organization '{}' requested access to organization '{}'", userId, organizationId, + organizationFqn); + throw new CatalogAuthorizationException("Cannot access data from a different organization."); + } else { + // If organization is not part of the FQN, assign it with the organization the user belongs to. + organizationFqn = organizationId; + } + query.putIfNotEmpty(StudyDBAdaptor.QueryParams.PROJECT_ID.key(), project); -// query.putIfNotEmpty(StudyDBAdaptor.QueryParams.ALIAS.key(), study); if (!queryOptions.isEmpty()) { // Ensure at least the ids are included fixQueryOptions(queryOptions, INCLUDE_STUDY_IDS.getAsStringList(QueryOptions.INCLUDE)); } - OpenCGAResult studyDataResult = studyDBAdaptor.get(query, queryOptions, userId); + OpenCGAResult studyDataResult = getStudyDBAdaptor(organizationFqn).get(query, queryOptions, userId); if (studyDataResult.getNumResults() == 0) { - if (!query.containsKey(StudyDBAdaptor.QueryParams.FQN.key())) { - // By default, exclude the opencga study for the validation - query.put(StudyDBAdaptor.QueryParams.FQN.key(), "!=" + OPENCGA + "@" + ADMIN_PROJECT + ":" + ADMIN_STUDY); - } - studyDataResult = studyDBAdaptor.get(query, queryOptions); + studyDataResult = getStudyDBAdaptor(organizationFqn).get(query, queryOptions); if (studyDataResult.getNumResults() == 0) { String studyMessage = ""; if (StringUtils.isNotEmpty(studyStr)) { @@ -269,11 +373,12 @@ private void fixQueryOptions(QueryOptions queryOptions, List includeFiel } } - private OpenCGAResult getStudy(long projectUid, String studyUuid, QueryOptions options) throws CatalogDBException { + private OpenCGAResult getStudy(String organizationId, long projectUid, String studyUuid, QueryOptions options) + throws CatalogDBException { Query query = new Query() .append(StudyDBAdaptor.QueryParams.PROJECT_UID.key(), projectUid) .append(StudyDBAdaptor.QueryParams.UUID.key(), studyUuid); - return studyDBAdaptor.get(query, options); + return getStudyDBAdaptor(organizationId).get(query, options); } @Deprecated @@ -296,8 +401,11 @@ public OpenCGAResult create(String projectStr, Study study, QueryOptions ParamUtils.checkObj(study, "study"); ParamUtils.checkIdentifier(study.getId(), "id"); - String userId = catalogManager.getUserManager().getUserId(token); - Project project = catalogManager.getProjectManager().resolveId(projectStr, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn catalogFqn = CatalogFqn.extractFqnFromProject(projectStr, tokenPayload); + String organizationId = catalogFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Project project = catalogManager.getProjectManager().resolveId(catalogFqn, null, tokenPayload).first(); ObjectMap auditParams = new ObjectMap() .append("projectId", projectStr) @@ -308,10 +416,7 @@ public OpenCGAResult create(String projectStr, Study study, QueryOptions try { options = ParamUtils.defaultObject(options, QueryOptions::new); - /* Check project permissions */ - if (!project.getFqn().startsWith(userId + "@")) { - throw new CatalogException("Permission denied: Only the owner of the project can create studies."); - } + authorizationManager.checkIsAtLeastOrganizationOwnerOrAdmin(organizationId, userId); long projectUid = project.getUid(); @@ -334,12 +439,6 @@ public OpenCGAResult create(String projectStr, Study study, QueryOptions study.setAdditionalInfo(ParamUtils.defaultObject(study.getAdditionalInfo(), Collections::emptyList)); study.setAttributes(ParamUtils.defaultObject(study.getAttributes(), HashMap::new)); - study.setClinicalAnalyses(ParamUtils.defaultObject(study.getClinicalAnalyses(), ArrayList::new)); - study.setCohorts(ParamUtils.defaultObject(study.getCohorts(), ArrayList::new)); - study.setFamilies(ParamUtils.defaultObject(study.getFamilies(), ArrayList::new)); - study.setPanels(ParamUtils.defaultObject(study.getPanels(), ArrayList::new)); - study.setSamples(ParamUtils.defaultObject(study.getSamples(), ArrayList::new)); - study.setIndividuals(ParamUtils.defaultObject(study.getIndividuals(), ArrayList::new)); study.setVariableSets(ParamUtils.defaultObject(study.getVariableSets(), ArrayList::new)); LinkedList files = new LinkedList<>(); @@ -350,24 +449,30 @@ public OpenCGAResult create(String projectStr, Study study, QueryOptions files.add(rootFile); files.add(jobsFile); + // Get organization owner and admins + Organization organization = catalogManager.getOrganizationManager().get(organizationId, + OrganizationManager.INCLUDE_ORGANIZATION_ADMINS, token).first(); + Set users = new HashSet<>(); + users.add(organization.getOwner()); + users.addAll(organization.getAdmins()); + List groups = Arrays.asList( - new Group(MEMBERS, Collections.singletonList(userId)), - new Group(ADMINS, Collections.emptyList()) + new Group(MEMBERS, new ArrayList<>(users)), + new Group(ADMINS, new ArrayList<>(users)) ); - study.setFiles(files); study.setGroups(groups); /* CreateStudy */ - studyDBAdaptor.insert(project, study, options); - OpenCGAResult result = getStudy(projectUid, study.getUuid(), options); + getStudyDBAdaptor(organizationId).insert(project, study, files, options); + OpenCGAResult result = getStudy(organizationId, projectUid, study.getUuid(), options); study = result.getResults().get(0); URI uri; try { - uri = catalogIOManager.createStudy(userId, Long.toString(project.getUid()), Long.toString(study.getUid())); + uri = catalogIOManager.createStudy(organizationId, Long.toString(project.getUid()), Long.toString(study.getUid())); } catch (CatalogIOException e) { try { - studyDBAdaptor.delete(study); + getStudyDBAdaptor(organizationId).delete(study); } catch (Exception e1) { logger.error("Can't delete study after failure creating study", e1); } @@ -375,35 +480,38 @@ public OpenCGAResult create(String projectStr, Study study, QueryOptions } // Update uri of study - studyDBAdaptor.update(study.getUid(), new ObjectMap("uri", uri), QueryOptions.empty()); + getStudyDBAdaptor(organizationId).update(study.getUid(), new ObjectMap("uri", uri), QueryOptions.empty()); study.setUri(uri); - long rootFileId = fileDBAdaptor.getId(study.getUid(), ""); //Set studyUri to the root folder too - fileDBAdaptor.update(rootFileId, new ObjectMap("uri", uri), QueryOptions.empty()); + long rootFileId = getFileDBAdaptor(organizationId).getId(study.getUid(), ""); //Set studyUri to the root folder too + getFileDBAdaptor(organizationId).update(rootFileId, new ObjectMap("uri", uri), QueryOptions.empty()); // Read and process installation variable sets - createDefaultVariableSets(study, token); + createDefaultVariableSets(organizationId, study, token); - auditManager.auditCreate(userId, Enums.Resource.STUDY, study.getId(), study.getUuid(), study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.auditCreate(organizationId, userId, Enums.Resource.STUDY, study.getId(), study.getUuid(), study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { result.setResults(Arrays.asList(study)); } return result; } catch (CatalogException e) { - auditManager.auditCreate(userId, Enums.Resource.STUDY, study.getId(), "", study.getId(), "", auditParams, + auditManager.auditCreate(organizationId, userId, Enums.Resource.STUDY, study.getId(), "", study.getId(), "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } public void createDefaultVariableSets(String studyStr, String token) throws CatalogException { - Study study = get(studyStr, new QueryOptions(), token).first(); - createDefaultVariableSets(study, token); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn catalogFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = catalogFqn.getOrganizationId(); + Study study = resolveId(catalogFqn, null, tokenPayload); + createDefaultVariableSets(organizationId, study, token); } - private void createDefaultVariableSets(Study study, String token) throws CatalogException { + private void createDefaultVariableSets(String organizationId, Study study, String token) throws CatalogException { Set variablesets = new Reflections(new ResourcesScanner(), "variablesets/").getResources(Pattern.compile(".*\\.json")); for (String variableSetFile : variablesets) { VariableSet vs; @@ -423,54 +531,21 @@ private void createDefaultVariableSets(Study study, String token) throws Catalog if (study.getVariableSets().stream().anyMatch(tvs -> tvs.getId().equals(vs.getId()))) { logger.debug("Skip already existing variable set " + vs.getId()); } else { - createVariableSet(study, vs, token); + createVariableSet(organizationId, study, vs, token); } } } } - public int getCurrentRelease(Study study, String sessionId) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(sessionId); - authorizationManager.checkCanViewStudy(study.getUid(), userId); - return getCurrentRelease(study); - } - int getCurrentRelease(Study study) throws CatalogException { String[] split = StringUtils.split(study.getFqn(), ":"); - String userId = StringUtils.split(split[0], "@")[0]; - return catalogManager.getProjectManager().resolveId(split[0], userId).getCurrentRelease(); - } + String[] split2 = StringUtils.split(split[0], "@"); + String organization = split2[0]; + String projectId = split2[1]; - MyResourceId getVariableSetId(String variableStr, @Nullable String studyStr, String sessionId) throws CatalogException { - if (StringUtils.isEmpty(variableStr)) { - throw new CatalogException("Missing variableSet parameter"); - } - - String userId; - long studyId; - long variableSetId; - - if (variableStr.contains(",")) { - throw new CatalogException("More than one variable set found. Please, choose just one variable set"); - } - - userId = catalogManager.getUserManager().getUserId(sessionId); - Study study = catalogManager.getStudyManager().resolveId(studyStr, userId); - studyId = study.getUid(); - - Query query = new Query() - .append(StudyDBAdaptor.VariableSetParams.STUDY_UID.key(), study.getUid()) - .append(StudyDBAdaptor.VariableSetParams.ID.key(), variableStr); - QueryOptions queryOptions = new QueryOptions(); - OpenCGAResult variableSetDataResult = studyDBAdaptor.getVariableSets(query, queryOptions); - if (variableSetDataResult.getNumResults() == 0) { - throw new CatalogException("Variable set " + variableStr + " not found in study " + studyStr); - } else if (variableSetDataResult.getNumResults() > 1) { - throw new CatalogException("More than one variable set found under " + variableStr + " in study " + studyStr); - } - variableSetId = variableSetDataResult.first().getUid(); - - return new MyResourceId(userId, studyId, variableSetId); + Query query = new Query(ProjectDBAdaptor.QueryParams.ID.key(), projectId); + QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, ProjectDBAdaptor.QueryParams.CURRENT_RELEASE.key()); + return getProjectDBAdaptor(organization).get(query, options).first().getCurrentRelease(); } /** @@ -483,34 +558,38 @@ MyResourceId getVariableSetId(String variableStr, @Nullable String studyStr, Str * @throws CatalogException CatalogException */ public OpenCGAResult get(String studyStr, QueryOptions options, String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn catalogFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = catalogFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + options = ParamUtils.defaultObject(options, QueryOptions::new); - String userId = catalogManager.getUserManager().getUserId(token); ObjectMap auditParams = new ObjectMap() .append("studyStr", studyStr) .append("options", options) .append("token", token); try { StopWatch stopWatch = StopWatch.createStarted(); - Study study = catalogManager.getStudyManager().resolveId(studyStr, userId, options); - auditManager.auditInfo(userId, Enums.Resource.STUDY, study.getId(), study.getUuid(), study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + Study study = catalogManager.getStudyManager().resolveId(studyStr, options, userId, organizationId); + auditManager.auditInfo(organizationId, userId, Enums.Resource.STUDY, study.getId(), study.getUuid(), study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); OpenCGAResult studyDataResult = new OpenCGAResult<>((int) stopWatch.getTime(TimeUnit.MILLISECONDS), null, 1, Collections.singletonList(study), 1); return filterResults(studyDataResult); } catch (CatalogException e) { - auditManager.auditInfo(userId, Enums.Resource.STUDY, studyStr, "", "", "", auditParams, + auditManager.auditInfo(organizationId, userId, Enums.Resource.STUDY, studyStr, "", "", "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } - public OpenCGAResult get(List studyList, QueryOptions queryOptions, boolean ignoreException, String sessionId) + public OpenCGAResult get(List studyList, QueryOptions queryOptions, boolean ignoreException, String token) throws CatalogException { OpenCGAResult result = OpenCGAResult.empty(Study.class); for (String study : studyList) { try { - OpenCGAResult studyObj = get(study, queryOptions, sessionId); + OpenCGAResult studyObj = get(study, queryOptions, token); result.append(studyObj); } catch (CatalogException e) { String warning = "Missing " + study + ": " + e.getMessage(); @@ -530,70 +609,78 @@ public OpenCGAResult get(List studyList, QueryOptions queryOption /** * Fetch all the study objects matching the query. * - * @param projectStr Project id or alias. - * @param query Query to catalog. - * @param options Query options, like "include", "exclude", "limit" and "skip" - * @param sessionId sessionId + * @param projectStr Project id, uuid or fqn. + * @param query Query to catalog. + * @param options Query options, like "include", "exclude", "limit" and "skip" + * @param token token * @return All matching elements. * @throws CatalogException CatalogException */ - public OpenCGAResult search(String projectStr, Query query, QueryOptions options, String sessionId) throws CatalogException { - ParamUtils.checkParameter(projectStr, "project"); - ParamUtils.defaultObject(query, Query::new); - ParamUtils.defaultObject(options, QueryOptions::new); - - String auxProject = null; - String auxOwner = null; - if (StringUtils.isNotEmpty(projectStr)) { - String[] split = projectStr.split("@"); - if (split.length == 1) { - auxProject = projectStr; - } else if (split.length == 2) { - auxOwner = split[0]; - auxProject = split[1]; - } else { - throw new CatalogException(projectStr + " does not follow the expected pattern [ownerId@projectId]"); - } - } + public OpenCGAResult search(String projectStr, Query query, QueryOptions options, String token) throws CatalogException { + query = ParamUtils.defaultObject(query, Query::new); + QueryOptions qOptions = options != null ? new QueryOptions(options) : new QueryOptions(); - query.putIfNotNull(StudyDBAdaptor.QueryParams.PROJECT_ID.key(), auxProject); - query.putIfNotNull(StudyDBAdaptor.QueryParams.OWNER.key(), auxOwner); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn projectFqn = CatalogFqn.extractFqnFromProject(projectStr, tokenPayload); + String organizationId = projectFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); - return search(query, options, sessionId); + ObjectMap auditParams = new ObjectMap() + .append("projectStr", projectStr) + .append("query", query) + .append("options", options) + .append("token", token); + try { + query.putIfNotEmpty(StudyDBAdaptor.QueryParams.PROJECT_ID.key(), projectFqn.getProjectId()); + query.putIfNotEmpty(StudyDBAdaptor.QueryParams.PROJECT_UUID.key(), projectFqn.getProjectUuid()); + fixQueryObject(query); + + OpenCGAResult studyDataResult = getStudyDBAdaptor(organizationId).get(query, qOptions, userId); + auditManager.auditSearch(organizationId, userId, Enums.Resource.STUDY, "", "", auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + return filterResults(studyDataResult); + } catch (CatalogException e) { + auditManager.auditSearch(organizationId, userId, Enums.Resource.STUDY, "", "", auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + throw e; + } } /** * Fetch all the study objects matching the query. * - * @param query Query to catalog. - * @param options Query options, like "include", "exclude", "limit" and "skip" - * @param token sessionId + * @param organizationId Organization id.. + * @param query Query to catalog. + * @param options Query options, like "include", "exclude", "limit" and "skip" + * @param token token * @return All matching elements. * @throws CatalogException CatalogException */ - public OpenCGAResult search(Query query, QueryOptions options, String token) throws CatalogException { + public OpenCGAResult searchInOrganization(String organizationId, Query query, QueryOptions options, String token) + throws CatalogException { query = ParamUtils.defaultObject(query, Query::new); QueryOptions qOptions = options != null ? new QueryOptions(options) : new QueryOptions(); - String userId = catalogManager.getUserManager().getUserId(token); - ObjectMap auditParams = new ObjectMap() + .append("organizationId", organizationId) .append("query", query) .append("options", options) .append("token", token); - if (!qOptions.containsKey("include") || qOptions.get("include") == null || qOptions.getAsStringList("include").isEmpty()) { - qOptions.addToListOption("exclude", "projects.studies.attributes.studyConfiguration"); - } + + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + organizationId = StringUtils.isNotEmpty(organizationId) ? organizationId : tokenPayload.getOrganization(); + String userId = tokenPayload.getUserId(organizationId); try { fixQueryObject(query); - OpenCGAResult studyDataResult = studyDBAdaptor.get(query, qOptions, userId); - auditManager.auditSearch(userId, Enums.Resource.STUDY, "", "", auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + OpenCGAResult studyDataResult = getStudyDBAdaptor(organizationId).get(query, qOptions, userId); + auditManager.auditSearch(organizationId, userId, Enums.Resource.STUDY, "", "", auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS), + new ObjectMap("numMatches", studyDataResult.getNumMatches())); return filterResults(studyDataResult); } catch (CatalogException e) { - auditManager.auditSearch(userId, Enums.Resource.STUDY, "", "", auditParams, + auditManager.auditSearch(organizationId, userId, Enums.Resource.STUDY, "", "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -624,27 +711,24 @@ private OpenCGAResult filterResults(OpenCGAResult result) { */ public OpenCGAResult update(String studyId, StudyUpdateParams parameters, QueryOptions options, String token) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(token); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = resolveId(studyFqn, QueryOptions.empty(), tokenPayload); ObjectMap auditParams = new ObjectMap() + .append("organizationId", organizationId) .append("studyId", studyId) .append("updateParams", parameters) .append("options", options) .append("token", token); - Study study; - try { - study = resolveId(studyId, userId); - } catch (CatalogException e) { - auditManager.auditUpdate(userId, Enums.Resource.STUDY, studyId, "", "", "", auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); - throw e; - } try { options = ParamUtils.defaultObject(options, QueryOptions::new); ParamUtils.checkObj(parameters, "Parameters"); - authorizationManager.checkCanEditStudy(study.getUid(), userId); + authorizationManager.checkCanEditStudy(organizationId, study.getUid(), userId); if (StringUtils.isNotEmpty(parameters.getAlias())) { ParamUtils.checkIdentifier(parameters.getAlias(), "alias"); @@ -664,28 +748,31 @@ public OpenCGAResult update(String studyId, StudyUpdateParams parameters, throw new CatalogException("Jackson casting error: " + e.getMessage(), e); } - OpenCGAResult updateResult = studyDBAdaptor.update(study.getUid(), update, options); - auditManager.auditUpdate(userId, Enums.Resource.STUDY, study.getId(), study.getUuid(), study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + OpenCGAResult updateResult = getStudyDBAdaptor(organizationId).update(study.getUid(), update, options); + auditManager.auditUpdate(organizationId, userId, Enums.Resource.STUDY, study.getId(), study.getUuid(), study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { // Fetch updated study - OpenCGAResult result = studyDBAdaptor.get(study.getUid(), options); + OpenCGAResult result = getStudyDBAdaptor(organizationId).get(study.getUid(), options); updateResult.setResults(result.getResults()); } return updateResult; } catch (CatalogException e) { - auditManager.auditUpdate(userId, Enums.Resource.STUDY, study.getId(), study.getUuid(), study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditUpdate(organizationId, userId, Enums.Resource.STUDY, study.getId(), study.getUuid(), study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } public OpenCGAResult createPermissionRule(String studyId, Enums.Entity entry, PermissionRule permissionRule, String token) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(token); - Study study = resolveId(studyId, userId, INCLUDE_STUDY_IDS); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = resolveId(studyFqn, INCLUDE_STUDY_IDS, tokenPayload); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) @@ -696,19 +783,20 @@ public OpenCGAResult createPermissionRule(String studyId, Enums. ParamUtils.checkObj(entry, "entry"); ParamUtils.checkObj(permissionRule, "permission rule"); - authorizationManager.checkCanUpdatePermissionRules(study.getUid(), userId); - validatePermissionRules(study.getUid(), entry, permissionRule); + authorizationManager.checkCanUpdatePermissionRules(organizationId, study.getUid(), userId); + validatePermissionRules(organizationId, study.getUid(), entry, permissionRule); - OpenCGAResult result = studyDBAdaptor.createPermissionRule(study.getUid(), entry, permissionRule); + OpenCGAResult result = getStudyDBAdaptor(organizationId).createPermissionRule(study.getUid(), entry, + permissionRule); - auditManager.audit(userId, Enums.Action.ADD_STUDY_PERMISSION_RULE, Enums.Resource.STUDY, study.getId(), + auditManager.audit(organizationId, userId, Enums.Action.ADD_STUDY_PERMISSION_RULE, Enums.Resource.STUDY, study.getId(), study.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return new OpenCGAResult<>(result.getTime(), result.getEvents(), 1, Collections.singletonList(permissionRule), 1, result.getNumInserted(), result.getNumUpdated(), result.getNumDeleted(), result.getNumErrors(), new ObjectMap()); } catch (CatalogException e) { - auditManager.audit(userId, Enums.Action.ADD_STUDY_PERMISSION_RULE, Enums.Resource.STUDY, study.getId(), + auditManager.audit(organizationId, userId, Enums.Action.ADD_STUDY_PERMISSION_RULE, Enums.Resource.STUDY, study.getId(), study.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; @@ -717,8 +805,11 @@ public OpenCGAResult createPermissionRule(String studyId, Enums. public void markDeletedPermissionRule(String studyId, Enums.Entity entry, String permissionRuleId, PermissionRule.DeleteAction deleteAction, String token) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(token); - Study study = resolveId(studyId, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = resolveId(studyFqn, QueryOptions.empty(), tokenPayload); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) @@ -731,14 +822,14 @@ public void markDeletedPermissionRule(String studyId, Enums.Entity entry, String ParamUtils.checkObj(deleteAction, "Delete action"); ParamUtils.checkObj(permissionRuleId, "permission rule id"); - authorizationManager.checkCanUpdatePermissionRules(study.getUid(), userId); - studyDBAdaptor.markDeletedPermissionRule(study.getUid(), entry, permissionRuleId, deleteAction); + authorizationManager.checkCanUpdatePermissionRules(organizationId, study.getUid(), userId); + getStudyDBAdaptor(organizationId).markDeletedPermissionRule(study.getUid(), entry, permissionRuleId, deleteAction); - auditManager.audit(userId, Enums.Action.REMOVE_STUDY_PERMISSION_RULE, Enums.Resource.STUDY, study.getId(), + auditManager.audit(organizationId, userId, Enums.Action.REMOVE_STUDY_PERMISSION_RULE, Enums.Resource.STUDY, study.getId(), study.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { - auditManager.audit(userId, Enums.Action.REMOVE_STUDY_PERMISSION_RULE, Enums.Resource.STUDY, study.getId(), + auditManager.audit(organizationId, userId, Enums.Action.REMOVE_STUDY_PERMISSION_RULE, Enums.Resource.STUDY, study.getId(), study.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; @@ -746,37 +837,41 @@ public void markDeletedPermissionRule(String studyId, Enums.Entity entry, String } public OpenCGAResult getPermissionRules(String studyId, Enums.Entity entry, String token) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(token); - Study study = resolveId(studyId, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = resolveId(studyFqn, QueryOptions.empty(), tokenPayload); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) .append("entry", entry) .append("token", token); try { - authorizationManager.checkCanViewStudy(study.getUid(), userId); - OpenCGAResult result = studyDBAdaptor.getPermissionRules(study.getUid(), entry); + authorizationManager.checkCanViewStudy(organizationId, study.getUid(), userId); + OpenCGAResult result = getStudyDBAdaptor(organizationId).getPermissionRules(study.getUid(), entry); - auditManager.audit(userId, Enums.Action.FETCH_STUDY_PERMISSION_RULES, Enums.Resource.STUDY, study.getId(), + auditManager.audit(organizationId, userId, Enums.Action.FETCH_STUDY_PERMISSION_RULES, Enums.Resource.STUDY, study.getId(), study.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return result; } catch (CatalogException e) { - auditManager.audit(userId, Enums.Action.FETCH_STUDY_PERMISSION_RULES, Enums.Resource.STUDY, study.getId(), + auditManager.audit(organizationId, userId, Enums.Action.FETCH_STUDY_PERMISSION_RULES, Enums.Resource.STUDY, study.getId(), study.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } - public OpenCGAResult rank(long projectId, Query query, String field, int numResults, boolean asc, String sessionId) + public OpenCGAResult rank(String organizationId, long projectId, Query query, String field, int numResults, boolean asc, String token) throws CatalogException { query = ParamUtils.defaultObject(query, Query::new); ParamUtils.checkObj(field, "field"); ParamUtils.checkObj(projectId, "projectId"); - String userId = catalogManager.getUserManager().getUserId(sessionId); - authorizationManager.checkCanViewProject(projectId, userId); + JwtPayload payload = catalogManager.getUserManager().validateToken(token); + String userId = payload.getUserId(organizationId); + authorizationManager.checkCanViewProject(organizationId, projectId, userId); // TODO: In next release, we will have to check the count parameter from the queryOptions object. boolean count = true; @@ -784,42 +879,46 @@ public OpenCGAResult rank(long projectId, Query query, String field, int numResu OpenCGAResult queryResult = null; if (count) { // We do not need to check for permissions when we show the count of files - queryResult = studyDBAdaptor.rank(query, field, numResults, asc); + queryResult = getStudyDBAdaptor(organizationId).rank(query, field, numResults, asc); } return ParamUtils.defaultObject(queryResult, OpenCGAResult::new); } - public OpenCGAResult groupBy(long projectId, Query query, String field, QueryOptions options, String sessionId) + public OpenCGAResult groupBy(String organizationId, long projectId, Query query, String field, QueryOptions options, String token) throws CatalogException { - return groupBy(projectId, query, Collections.singletonList(field), options, sessionId); + return groupBy(organizationId, projectId, query, Collections.singletonList(field), options, token); } - public OpenCGAResult groupBy(long projectId, Query query, List fields, QueryOptions options, String sessionId) - throws CatalogException { + public OpenCGAResult groupBy(String organizationId, long projectId, Query query, List fields, QueryOptions options, + String token) throws CatalogException { query = ParamUtils.defaultObject(query, Query::new); options = ParamUtils.defaultObject(options, QueryOptions::new); ParamUtils.checkObj(fields, "fields"); ParamUtils.checkObj(projectId, "projectId"); - String userId = catalogManager.getUserManager().getUserId(sessionId); - authorizationManager.checkCanViewProject(projectId, userId); + JwtPayload payload = catalogManager.getUserManager().validateToken(token); + String userId = payload.getUserId(organizationId); + authorizationManager.checkCanViewProject(organizationId, projectId, userId); // TODO: In next release, we will have to check the count parameter from the queryOptions object. boolean count = true; OpenCGAResult queryResult = null; if (count) { // We do not need to check for permissions when we show the count of files - queryResult = studyDBAdaptor.groupBy(query, fields, options); + queryResult = getStudyDBAdaptor(organizationId).groupBy(query, fields, options); } return ParamUtils.defaultObject(queryResult, OpenCGAResult::new); } - public OpenCGAResult getSummary(String studyStr, QueryOptions queryOptions, String sessionId) throws CatalogException { + public OpenCGAResult getSummary(String studyStr, QueryOptions queryOptions, String token) throws CatalogException { long startTime = System.currentTimeMillis(); - Study study = get(studyStr, queryOptions, sessionId).first(); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + Study study = resolveId(studyFqn, queryOptions, tokenPayload); StudySummary studySummary = new StudySummary() .setAlias(study.getId()) @@ -832,7 +931,7 @@ public OpenCGAResult getSummary(String studyStr, QueryOptions quer .setInternal(study.getInternal()) .setVariableSets(study.getVariableSets()); - Long nFiles = fileDBAdaptor.count( + Long nFiles = getFileDBAdaptor(organizationId).count( new Query(FileDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()) .append(FileDBAdaptor.QueryParams.TYPE.key(), File.Type.FILE) .append(FileDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), "!=" + FileStatus.TRASHED + ";!=" @@ -840,16 +939,20 @@ public OpenCGAResult getSummary(String studyStr, QueryOptions quer .getNumMatches(); studySummary.setFiles(nFiles); - Long nSamples = sampleDBAdaptor.count(new Query(SampleDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid())).getNumMatches(); + Long nSamples = getSampleDBAdaptor(organizationId).count(new Query(SampleDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid())) + .getNumMatches(); studySummary.setSamples(nSamples); - Long nJobs = jobDBAdaptor.count(new Query(JobDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid())).getNumMatches(); + Long nJobs = getJobDBAdaptor(organizationId).count(new Query(JobDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid())) + .getNumMatches(); studySummary.setJobs(nJobs); - Long nCohorts = cohortDBAdaptor.count(new Query(CohortDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid())).getNumMatches(); + Long nCohorts = getCohortDBAdaptor(organizationId).count(new Query(CohortDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid())) + .getNumMatches(); studySummary.setCohorts(nCohorts); - Long nIndividuals = individualDBAdaptor.count(new Query(IndividualDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid())) + Long nIndividuals = getIndividualDBAdaptor(organizationId) + .count(new Query(IndividualDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid())) .getNumMatches(); studySummary.setIndividuals(nIndividuals); @@ -876,15 +979,17 @@ public List> getSummary(List studyList, Quer return results; } - public OpenCGAResult createGroup(String studyStr, String groupId, List users, String sessionId) - throws CatalogException { + public OpenCGAResult createGroup(String studyStr, String groupId, List users, String token) throws CatalogException { ParamUtils.checkParameter(groupId, "group id"); - return createGroup(studyStr, new Group(groupId, users, null), sessionId); + return createGroup(studyStr, new Group(groupId, users, null), token); } public OpenCGAResult createGroup(String studyId, Group group, String token) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(token); - Study study = resolveId(studyId, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = resolveId(studyFqn, QueryOptions.empty(), tokenPayload); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) @@ -905,10 +1010,10 @@ public OpenCGAResult createGroup(String studyId, Group group, String toke group.setId("@" + group.getId()); } - authorizationManager.checkCreateDeleteGroupPermissions(study.getUid(), userId, group.getId()); + authorizationManager.checkCreateDeleteGroupPermissions(organizationId, study.getUid(), userId, group.getId()); // Check group exists - if (existsGroup(study.getUid(), group.getId())) { + if (existsGroup(organizationId, study.getUid(), group.getId())) { throw new CatalogException("The group " + group.getId() + " already exists."); } @@ -916,7 +1021,7 @@ public OpenCGAResult createGroup(String studyId, Group group, String toke if (CollectionUtils.isNotEmpty(users)) { // We remove possible duplicates users = users.stream().collect(Collectors.toSet()).stream().collect(Collectors.toList()); - userDBAdaptor.checkIds(users); + getUserDBAdaptor(organizationId).checkIds(users); } else { users = Collections.emptyList(); } @@ -924,56 +1029,64 @@ public OpenCGAResult createGroup(String studyId, Group group, String toke // Add those users to the members group if (CollectionUtils.isNotEmpty(users)) { - studyDBAdaptor.addUsersToGroup(study.getUid(), MEMBERS, users); + getStudyDBAdaptor(organizationId).addUsersToGroup(study.getUid(), MEMBERS, users); } // Create the group - OpenCGAResult result = studyDBAdaptor.createGroup(study.getUid(), group); + OpenCGAResult result = getStudyDBAdaptor(organizationId).createGroup(study.getUid(), group); - OpenCGAResult queryResult = studyDBAdaptor.getGroup(study.getUid(), group.getId(), null); + OpenCGAResult queryResult = getStudyDBAdaptor(organizationId).getGroup(study.getUid(), group.getId(), null); queryResult.setTime(queryResult.getTime() + result.getTime()); - auditManager.audit(userId, Enums.Action.ADD_STUDY_GROUP, Enums.Resource.STUDY, study.getId(), study.getUuid(), + auditManager.audit(organizationId, userId, Enums.Action.ADD_STUDY_GROUP, Enums.Resource.STUDY, study.getId(), study.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return queryResult; } catch (CatalogException e) { - auditManager.audit(userId, Enums.Action.ADD_STUDY_GROUP, Enums.Resource.STUDY, study.getId(), study.getUuid(), + auditManager.audit(organizationId, userId, Enums.Action.ADD_STUDY_GROUP, Enums.Resource.STUDY, study.getId(), study.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } public OpenCGAResult getGroup(String studyId, String groupId, String token) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(token); - Study study = resolveId(studyId, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = resolveId(studyFqn, QueryOptions.empty(), tokenPayload); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) .append("groupId", groupId) .append("token", token); try { - authorizationManager.checkCanViewStudy(study.getUid(), userId); + authorizationManager.checkCanViewStudy(organizationId, study.getUid(), userId); // Fix the groupId if (groupId != null && !groupId.startsWith("@")) { groupId = "@" + groupId; } - OpenCGAResult result = studyDBAdaptor.getGroup(study.getUid(), groupId, Collections.emptyList()); - auditManager.audit(userId, Enums.Action.FETCH_STUDY_GROUPS, Enums.Resource.STUDY, study.getId(), study.getUuid(), - study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + OpenCGAResult result = getStudyDBAdaptor(organizationId).getGroup(study.getUid(), groupId, Collections.emptyList()); + auditManager.audit(organizationId, userId, Enums.Action.FETCH_STUDY_GROUPS, Enums.Resource.STUDY, study.getId(), + study.getUuid(), study.getId(), study.getUuid(), auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return result; } catch (CatalogException e) { - auditManager.audit(userId, Enums.Action.FETCH_STUDY_GROUPS, Enums.Resource.STUDY, study.getId(), study.getUuid(), - study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.audit(organizationId, userId, Enums.Action.FETCH_STUDY_GROUPS, Enums.Resource.STUDY, study.getId(), + study.getUuid(), study.getId(), study.getUuid(), auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } public OpenCGAResult getCustomGroups(String studyId, String groupId, String token) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(token); - Study study = resolveId(studyId, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = resolveId(studyFqn, QueryOptions.empty(), tokenPayload); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) @@ -983,14 +1096,14 @@ public OpenCGAResult getCustomGroups(String studyId, String groupId StopWatch stopWatch = new StopWatch(); stopWatch.start(); - authorizationManager.checkIsOwnerOrAdmin(study.getUid(), userId); + authorizationManager.checkIsAtLeastStudyAdministrator(organizationId, study.getUid(), userId); // Fix the groupId if (groupId != null && !groupId.startsWith("@")) { groupId = "@" + groupId; } - OpenCGAResult result = studyDBAdaptor.getGroup(study.getUid(), groupId, Collections.emptyList()); + OpenCGAResult result = getStudyDBAdaptor(organizationId).getGroup(study.getUid(), groupId, Collections.emptyList()); // Extract all users from all groups Set userIds = new HashSet<>(); @@ -999,10 +1112,8 @@ public OpenCGAResult getCustomGroups(String studyId, String groupId } Query userQuery = new Query(UserDBAdaptor.QueryParams.ID.key(), new ArrayList<>(userIds)); - QueryOptions userOptions = new QueryOptions(QueryOptions.EXCLUDE, Arrays.asList( - UserDBAdaptor.QueryParams.PROJECTS.key(), UserDBAdaptor.QueryParams.SHARED_PROJECTS.key() - )); - OpenCGAResult userResult = userDBAdaptor.get(userQuery, userOptions); + QueryOptions userOptions = new QueryOptions(); + OpenCGAResult userResult = getUserDBAdaptor(organizationId).get(userQuery, userOptions); Map userMap = new HashMap<>(); for (User user : userResult.getResults()) { userMap.put(user.getId(), user); @@ -1024,24 +1135,29 @@ public OpenCGAResult getCustomGroups(String studyId, String groupId customGroupList, result.getNumMatches(), result.getNumInserted(), result.getNumUpdated(), result.getNumDeleted(), result.getNumErrors(), result.getAttributes(), result.getFederationNode()); - auditManager.audit(userId, Enums.Action.FETCH_STUDY_GROUPS, Enums.Resource.STUDY, study.getId(), study.getUuid(), - study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.audit(organizationId, userId, Enums.Action.FETCH_STUDY_GROUPS, Enums.Resource.STUDY, study.getId(), + study.getUuid(), study.getId(), study.getUuid(), auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); stopWatch.stop(); finalResult.setTime((int) stopWatch.getTime(TimeUnit.MILLISECONDS)); return finalResult; } catch (CatalogException e) { - auditManager.audit(userId, Enums.Action.FETCH_STUDY_GROUPS, Enums.Resource.STUDY, study.getId(), study.getUuid(), - study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.audit(organizationId, userId, Enums.Action.FETCH_STUDY_GROUPS, Enums.Resource.STUDY, study.getId(), + study.getUuid(), study.getId(), study.getUuid(), auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } public OpenCGAResult updateGroup(String studyId, String groupId, ParamUtils.BasicUpdateAction action, GroupUpdateParams updateParams, String token) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(token); - Study study = resolveId(studyId, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = resolveId(studyFqn, QueryOptions.empty(), tokenPayload); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) @@ -1059,7 +1175,7 @@ public OpenCGAResult updateGroup(String studyId, String groupId, ParamUti groupId = "@" + groupId; } - authorizationManager.checkUpdateGroupPermissions(study.getUid(), userId, groupId, action); + authorizationManager.checkUpdateGroupPermissions(organizationId, study.getUid(), userId, groupId, action); if (CollectionUtils.isNotEmpty(updateParams.getUsers())) { List tmpUsers = updateParams.getUsers(); @@ -1072,7 +1188,7 @@ public OpenCGAResult updateGroup(String studyId, String groupId, ParamUti .collect(Collectors.toList()); } if (tmpUsers.size() > 0) { - userDBAdaptor.checkIds(tmpUsers); + getUserDBAdaptor(organizationId).checkIds(tmpUsers); } } else { updateParams.setUsers(Collections.emptyList()); @@ -1082,59 +1198,74 @@ public OpenCGAResult updateGroup(String studyId, String groupId, ParamUti case SET: if (MEMBERS.equals(groupId)) { throw new CatalogException("Operation not valid. Valid actions over the '@members' group are ADD or REMOVE."); + } else if (ADMINS.equals(groupId)) { + Set admins = catalogManager.getOrganizationManager().getOrganizationOwnerAndAdmins(organizationId); + if (!updateParams.getUsers().containsAll(admins)) { + throw new CatalogException("Can't set the users provided to the " + ADMINS + " group. Organization owner " + + "and admins cannot be removed from that group. Please ensure these users are also added: " + + String.join(", ", admins)); + } } - studyDBAdaptor.setUsersToGroup(study.getUid(), groupId, updateParams.getUsers()); - studyDBAdaptor.addUsersToGroup(study.getUid(), MEMBERS, updateParams.getUsers()); + getStudyDBAdaptor(organizationId).setUsersToGroup(study.getUid(), groupId, updateParams.getUsers()); + getStudyDBAdaptor(organizationId).addUsersToGroup(study.getUid(), MEMBERS, updateParams.getUsers()); break; case ADD: - studyDBAdaptor.addUsersToGroup(study.getUid(), groupId, updateParams.getUsers()); + getStudyDBAdaptor(organizationId).addUsersToGroup(study.getUid(), groupId, updateParams.getUsers()); if (!MEMBERS.equals(groupId)) { - studyDBAdaptor.addUsersToGroup(study.getUid(), MEMBERS, updateParams.getUsers()); + getStudyDBAdaptor(organizationId).addUsersToGroup(study.getUid(), MEMBERS, updateParams.getUsers()); } break; case REMOVE: - if (MEMBERS.equals(groupId)) { - // Check we are not trying to remove the owner of the study from the group - String owner = getOwner(study); - if (updateParams.getUsers().contains(owner)) { - throw new CatalogException("Cannot remove owner of the study from the '@members' group"); + // Check is not one of the organization admins + if (MEMBERS.equals(groupId) || ADMINS.equals(groupId)) { + Set admins = catalogManager.getOrganizationManager().getOrganizationOwnerAndAdmins(organizationId); + for (String user : updateParams.getUsers()) { + if (admins.contains(user)) { + throw new CatalogException("Cannot remove user '" + user + "'. The user is either the organization owner " + + "or one of the organization administrators."); + } } + } + if (MEMBERS.equals(groupId)) { // We remove the users from all the groups and acls - authorizationManager.resetPermissionsFromAllEntities(study.getUid(), updateParams.getUsers()); - studyDBAdaptor.removeUsersFromAllGroups(study.getUid(), updateParams.getUsers()); + authorizationManager.resetPermissionsFromAllEntities(organizationId, study.getUid(), updateParams.getUsers()); + getStudyDBAdaptor(organizationId).removeUsersFromAllGroups(study.getUid(), updateParams.getUsers()); } else { - studyDBAdaptor.removeUsersFromGroup(study.getUid(), groupId, updateParams.getUsers()); + getStudyDBAdaptor(organizationId).removeUsersFromGroup(study.getUid(), groupId, updateParams.getUsers()); } break; default: throw new CatalogException("Unknown action " + action + " found."); } - auditManager.audit(userId, Enums.Action.UPDATE_USERS_FROM_STUDY_GROUP, Enums.Resource.STUDY, study.getId(), + auditManager.audit(organizationId, userId, Enums.Action.UPDATE_USERS_FROM_STUDY_GROUP, Enums.Resource.STUDY, study.getId(), study.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); - return studyDBAdaptor.getGroup(study.getUid(), groupId, Collections.emptyList()); + return getStudyDBAdaptor(organizationId).getGroup(study.getUid(), groupId, Collections.emptyList()); } catch (CatalogException e) { - auditManager.audit(userId, Enums.Action.UPDATE_USERS_FROM_STUDY_GROUP, Enums.Resource.STUDY, study.getId(), + auditManager.audit(organizationId, userId, Enums.Action.UPDATE_USERS_FROM_STUDY_GROUP, Enums.Resource.STUDY, study.getId(), study.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } - public OpenCGAResult updateSummaryIndex(String studyStr, RecessiveGeneSummaryIndex summaryIndex, - String token) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(token); - Study study = resolveId(studyStr, userId); + public OpenCGAResult updateSummaryIndex(String studyStr, RecessiveGeneSummaryIndex summaryIndex, String token) + throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = resolveId(studyFqn, QueryOptions.empty(), tokenPayload); ObjectMap auditParams = new ObjectMap() .append("studyId", studyStr) .append("recessiveGeneSummaryIndex", summaryIndex) .append("token", token); try { - authorizationManager.checkIsOwnerOrAdmin(study.getUid(), userId); + authorizationManager.checkIsAtLeastStudyAdministrator(organizationId, study.getUid(), userId); ParamUtils.checkObj(summaryIndex, "RecessiveGeneSummaryIndex"); summaryIndex.setModificationDate(ParamUtils.checkDateOrGetCurrentDate(summaryIndex.getModificationDate(), "creationDate")); @@ -1147,15 +1278,15 @@ public OpenCGAResult updateSummaryIndex(String studyStr, RecessiveGeneSummary throw new CatalogException("Jackson casting error: " + e.getMessage(), e); } - OpenCGAResult result = studyDBAdaptor.update(study.getUid(), update, QueryOptions.empty()); + OpenCGAResult result = getStudyDBAdaptor(organizationId).update(study.getUid(), update, QueryOptions.empty()); - auditManager.audit(userId, Enums.Action.UPDATE_INTERNAL, Enums.Resource.STUDY, study.getId(), + auditManager.audit(organizationId, userId, Enums.Action.UPDATE_INTERNAL, Enums.Resource.STUDY, study.getId(), study.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return result; } catch (CatalogException e) { - auditManager.audit(userId, Enums.Action.UPDATE_INTERNAL, Enums.Resource.STUDY, study.getId(), + auditManager.audit(organizationId, userId, Enums.Action.UPDATE_INTERNAL, Enums.Resource.STUDY, study.getId(), study.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; @@ -1201,8 +1332,11 @@ public OpenCGAResult updateSummaryIndex(String studyStr, RecessiveGeneSummary // } public OpenCGAResult deleteGroup(String studyId, String groupId, String token) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(token); - Study study = resolveId(studyId, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = resolveId(studyFqn, QueryOptions.empty(), tokenPayload); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) @@ -1218,58 +1352,56 @@ public OpenCGAResult deleteGroup(String studyId, String groupId, String t groupId = "@" + groupId; } - authorizationManager.checkCreateDeleteGroupPermissions(study.getUid(), userId, groupId); + authorizationManager.checkCreateDeleteGroupPermissions(organizationId, study.getUid(), userId, groupId); - OpenCGAResult group = studyDBAdaptor.getGroup(study.getUid(), groupId, Collections.emptyList()); + OpenCGAResult group = getStudyDBAdaptor(organizationId).getGroup(study.getUid(), groupId, Collections.emptyList()); // Remove the permissions the group might have had StudyAclParams aclParams = new StudyAclParams(null, null); - updateAcl(Collections.singletonList(studyId), groupId, aclParams, ParamUtils.AclAction.RESET, token); + updateAcl(studyId, groupId, aclParams, ParamUtils.AclAction.RESET, token); - studyDBAdaptor.deleteGroup(study.getUid(), groupId); + getStudyDBAdaptor(organizationId).deleteGroup(study.getUid(), groupId); - auditManager.audit(userId, Enums.Action.REMOVE_STUDY_GROUP, Enums.Resource.STUDY, study.getId(), study.getUuid(), - study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.audit(organizationId, userId, Enums.Action.REMOVE_STUDY_GROUP, Enums.Resource.STUDY, study.getId(), + study.getUuid(), study.getId(), study.getUuid(), auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return group; } catch (CatalogException e) { - auditManager.audit(userId, Enums.Action.REMOVE_STUDY_GROUP, Enums.Resource.STUDY, study.getId(), study.getUuid(), - study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.audit(organizationId, userId, Enums.Action.REMOVE_STUDY_GROUP, Enums.Resource.STUDY, study.getId(), + study.getUuid(), study.getId(), study.getUuid(), auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } - public OpenCGAResult getVariableSetSummary(String studyStr, String variableSetStr, String sessionId) + public OpenCGAResult getVariableSetSummary(String studyStr, String variableSetStr, String token) throws CatalogException { - MyResourceId resource = getVariableSetId(variableSetStr, studyStr, sessionId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = resolveId(studyFqn, StudyManager.INCLUDE_VARIABLE_SET, tokenPayload); - String userId = resource.getUser(); - - OpenCGAResult variableSet = studyDBAdaptor.getVariableSet(resource.getResourceId(), new QueryOptions(), userId); - if (variableSet.getNumResults() == 0) { - logger.error("getVariableSetSummary: Could not find variable set id {}. {} results returned", variableSetStr, - variableSet.getNumResults()); - throw new CatalogDBException("Variable set " + variableSetStr + " not found."); - } + VariableSet variableSet = extractVariableSet(organizationId, study, variableSetStr, userId); int dbTime = 0; + VariableSetSummary variableSetSummary = new VariableSetSummary(variableSet.getUid(), variableSet.getId()); - VariableSetSummary variableSetSummary = new VariableSetSummary(resource.getResourceId(), variableSet.first().getId()); - - OpenCGAResult annotationSummary = sampleDBAdaptor.getAnnotationSummary(resource.getStudyId(), - resource.getResourceId()); + OpenCGAResult annotationSummary = getSampleDBAdaptor(organizationId).getAnnotationSummary(study.getUid(), + variableSet.getUid()); dbTime += annotationSummary.getTime(); variableSetSummary.setSamples(annotationSummary.getResults()); - annotationSummary = cohortDBAdaptor.getAnnotationSummary(resource.getStudyId(), resource.getResourceId()); + annotationSummary = getCohortDBAdaptor(organizationId).getAnnotationSummary(study.getUid(), variableSet.getUid()); dbTime += annotationSummary.getTime(); variableSetSummary.setCohorts(annotationSummary.getResults()); - annotationSummary = individualDBAdaptor.getAnnotationSummary(resource.getStudyId(), resource.getResourceId()); + annotationSummary = getIndividualDBAdaptor(organizationId).getAnnotationSummary(study.getUid(), variableSet.getUid()); dbTime += annotationSummary.getTime(); variableSetSummary.setIndividuals(annotationSummary.getResults()); - annotationSummary = familyDBAdaptor.getAnnotationSummary(resource.getStudyId(), resource.getResourceId()); + annotationSummary = getFamilyDBAdaptor(organizationId).getAnnotationSummary(study.getUid(), variableSet.getUid()); dbTime += annotationSummary.getTime(); variableSetSummary.setFamilies(annotationSummary.getResults()); @@ -1280,12 +1412,13 @@ public OpenCGAResult getVariableSetSummary(String studyStr, /* * Variables Methods */ - private OpenCGAResult createVariableSet(Study study, VariableSet variableSet, String token) + private OpenCGAResult createVariableSet(String organizationId, Study study, VariableSet variableSet, String token) throws CatalogException { ParamUtils.checkParameter(variableSet.getId(), "id"); ParamUtils.checkObj(variableSet.getVariables(), "Variables from VariableSet"); - String userId = catalogManager.getUserManager().getUserId(token); - authorizationManager.checkCanCreateUpdateDeleteVariableSets(study.getUid(), userId); + JwtPayload jwtPayload = catalogManager.getUserManager().validateToken(token); + String userId = jwtPayload.getUserId(organizationId); + authorizationManager.checkCanCreateUpdateDeleteVariableSets(organizationId, study.getUid(), userId); variableSet.setDescription(ParamUtils.defaultString(variableSet.getDescription(), "")); variableSet.setAttributes(ParamUtils.defaultObject(variableSet.getAttributes(), new HashMap<>())); @@ -1307,8 +1440,9 @@ private OpenCGAResult createVariableSet(Study study, VariableSet va variableSet.setRelease(getCurrentRelease(study)); AnnotationUtils.checkVariableSet(variableSet); - OpenCGAResult result = studyDBAdaptor.createVariableSet(study.getUid(), variableSet); - OpenCGAResult queryResult = studyDBAdaptor.getVariableSet(study.getUid(), variableSet.getId(), QueryOptions.empty()); + OpenCGAResult result = getStudyDBAdaptor(organizationId).createVariableSet(study.getUid(), variableSet); + OpenCGAResult queryResult = getStudyDBAdaptor(organizationId).getVariableSet(study.getUid(), variableSet.getId(), + QueryOptions.empty()); return OpenCGAResult.merge(Arrays.asList(result, queryResult)); } @@ -1323,29 +1457,35 @@ public OpenCGAResult createVariableSet(String studyId, String id, S } public OpenCGAResult createVariableSet(String studyId, VariableSet variableSet, String token) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(token); - Study study = resolveId(studyId, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = resolveId(studyFqn, QueryOptions.empty(), tokenPayload); ObjectMap auditParams = new ObjectMap() .append("study", studyId) .append("variableSet", variableSet) .append("token", token); try { - OpenCGAResult queryResult = createVariableSet(study, variableSet, token); - auditManager.audit(userId, Enums.Action.ADD_VARIABLE_SET, Enums.Resource.STUDY, queryResult.first().getId(), "", + OpenCGAResult queryResult = createVariableSet(organizationId, study, variableSet, token); + auditManager.audit(organizationId, userId, Enums.Action.ADD_VARIABLE_SET, Enums.Resource.STUDY, queryResult.first().getId(), "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return queryResult; } catch (CatalogException e) { - auditManager.audit(userId, Enums.Action.ADD_VARIABLE_SET, Enums.Resource.STUDY, variableSet.getId(), "", study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.audit(organizationId, userId, Enums.Action.ADD_VARIABLE_SET, Enums.Resource.STUDY, variableSet.getId(), "", + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } public OpenCGAResult getVariableSet(String studyId, String variableSetId, QueryOptions options, String token) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(token); - Study study = resolveId(studyId, userId, StudyManager.INCLUDE_VARIABLE_SET); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = resolveId(studyFqn, StudyManager.INCLUDE_VARIABLE_SET, tokenPayload); ObjectMap auditParams = new ObjectMap() .append("studyId", studyId) @@ -1354,41 +1494,28 @@ public OpenCGAResult getVariableSet(String studyId, String variable .append("token", token); try { options = ParamUtils.defaultObject(options, QueryOptions::new); - VariableSet variableSet = extractVariableSet(study, variableSetId, userId); - OpenCGAResult result = studyDBAdaptor.getVariableSet(variableSet.getUid(), options, userId); + VariableSet variableSet = extractVariableSet(organizationId, study, variableSetId, userId); + OpenCGAResult result = getStudyDBAdaptor(organizationId).getVariableSet(variableSet.getUid(), options, userId); - auditManager.audit(userId, Enums.Action.FETCH_VARIABLE_SET, Enums.Resource.STUDY, variableSet.getId(), "", + auditManager.audit(organizationId, userId, Enums.Action.FETCH_VARIABLE_SET, Enums.Resource.STUDY, variableSet.getId(), "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return result; } catch (CatalogException e) { - auditManager.audit(userId, Enums.Action.FETCH_VARIABLE_SET, Enums.Resource.STUDY, variableSetId, "", study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.audit(organizationId, userId, Enums.Action.FETCH_VARIABLE_SET, Enums.Resource.STUDY, variableSetId, "", + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } - public OpenCGAResult searchVariableSets(String studyStr, Query query, QueryOptions options, String sessionId) - throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(sessionId); - Study study = resolveId(studyStr, userId); -// authorizationManager.checkStudyPermission(studyId, userId, StudyAclEntry.StudyPermissions.VIEW_VARIABLE_SET); - options = ParamUtils.defaultObject(options, QueryOptions::new); - query = ParamUtils.defaultObject(query, Query::new); - if (query.containsKey(StudyDBAdaptor.VariableSetParams.UID.key())) { - // Id could be either the id or the name - MyResourceId resource = getVariableSetId(query.getString(StudyDBAdaptor.VariableSetParams.UID.key()), studyStr, sessionId); - query.put(StudyDBAdaptor.VariableSetParams.UID.key(), resource.getResourceId()); - } - query.put(StudyDBAdaptor.VariableSetParams.STUDY_UID.key(), study.getUid()); - return studyDBAdaptor.getVariableSets(query, options, userId); - } - public OpenCGAResult deleteVariableSet(String studyId, String variableSetId, boolean force, String token) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(token); - Study study = resolveId(studyId, userId, StudyManager.INCLUDE_VARIABLE_SET); - VariableSet variableSet = extractVariableSet(study, variableSetId, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = resolveId(studyFqn, StudyManager.INCLUDE_VARIABLE_SET, tokenPayload); + VariableSet variableSet = extractVariableSet(organizationId, study, variableSetId, userId); ObjectMap auditParams = new ObjectMap() .append("study", studyId) @@ -1396,14 +1523,14 @@ public OpenCGAResult deleteVariableSet(String studyId, String varia .append("force", force) .append("token", token); try { - authorizationManager.checkCanCreateUpdateDeleteVariableSets(study.getUid(), userId); - OpenCGAResult writeResult = studyDBAdaptor.deleteVariableSet(study.getUid(), variableSet, force); - auditManager.audit(userId, Enums.Action.DELETE_VARIABLE_SET, Enums.Resource.STUDY, variableSet.getId(), "", + authorizationManager.checkCanCreateUpdateDeleteVariableSets(organizationId, study.getUid(), userId); + OpenCGAResult writeResult = getStudyDBAdaptor(organizationId).deleteVariableSet(study.getUid(), variableSet, force); + auditManager.audit(organizationId, userId, Enums.Action.DELETE_VARIABLE_SET, Enums.Resource.STUDY, variableSet.getId(), "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return writeResult; } catch (CatalogException e) { - auditManager.audit(userId, Enums.Action.DELETE_VARIABLE_SET, Enums.Resource.STUDY, variableSet.getId(), "", + auditManager.audit(organizationId, userId, Enums.Action.DELETE_VARIABLE_SET, Enums.Resource.STUDY, variableSet.getId(), "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -1411,9 +1538,12 @@ public OpenCGAResult deleteVariableSet(String studyId, String varia public OpenCGAResult addFieldToVariableSet(String studyId, String variableSetId, Variable variable, String token) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(token); - Study study = resolveId(studyId, userId, StudyManager.INCLUDE_VARIABLE_SET); - VariableSet variableSet = extractVariableSet(study, variableSetId, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = resolveId(studyFqn, StudyManager.INCLUDE_VARIABLE_SET, tokenPayload); + VariableSet variableSet = extractVariableSet(organizationId, study, variableSetId, userId); ObjectMap auditParams = new ObjectMap() .append("study", studyId) @@ -1428,52 +1558,61 @@ public OpenCGAResult addFieldToVariableSet(String studyId, String v variable.setId(variable.getName()); } - authorizationManager.checkCanCreateUpdateDeleteVariableSets(study.getUid(), userId); - OpenCGAResult result = studyDBAdaptor.addFieldToVariableSet(study.getUid(), variableSet.getUid(), variable, userId); - auditManager.audit(userId, Enums.Action.ADD_VARIABLE_TO_VARIABLE_SET, Enums.Resource.STUDY, variableSet.getId(), "", - study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + authorizationManager.checkCanCreateUpdateDeleteVariableSets(organizationId, study.getUid(), userId); + OpenCGAResult result = getStudyDBAdaptor(organizationId).addFieldToVariableSet(study.getUid(), + variableSet.getUid(), variable, userId); + auditManager.audit(organizationId, userId, Enums.Action.ADD_VARIABLE_TO_VARIABLE_SET, Enums.Resource.STUDY, variableSet.getId(), + "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); - OpenCGAResult queryResult = studyDBAdaptor.getVariableSet(variableSet.getUid(), QueryOptions.empty()); + OpenCGAResult queryResult = getStudyDBAdaptor(organizationId).getVariableSet(variableSet.getUid(), + QueryOptions.empty()); queryResult.setTime(queryResult.getTime() + result.getTime()); return queryResult; } catch (CatalogException e) { - auditManager.audit(userId, Enums.Action.ADD_VARIABLE_TO_VARIABLE_SET, Enums.Resource.STUDY, variableSet.getId(), "", - study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.audit(organizationId, userId, Enums.Action.ADD_VARIABLE_TO_VARIABLE_SET, Enums.Resource.STUDY, variableSet.getId(), + "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } public OpenCGAResult removeFieldFromVariableSet(String studyId, String variableSetId, String variableId, String token) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(token); - Study study = resolveId(studyId, userId, StudyManager.INCLUDE_VARIABLE_SET); - VariableSet variableSet = extractVariableSet(study, variableSetId, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = resolveId(studyFqn, StudyManager.INCLUDE_VARIABLE_SET, tokenPayload); + VariableSet variableSet = extractVariableSet(organizationId, study, variableSetId, userId); ObjectMap auditParams = new ObjectMap() + .append("organizationId", organizationId) .append("study", studyId) .append("variableSetId", variableSetId) .append("variableId", variableId) .append("token", token); try { - authorizationManager.checkCanCreateUpdateDeleteVariableSets(study.getUid(), userId); - OpenCGAResult result = studyDBAdaptor.removeFieldFromVariableSet(study.getUid(), variableSet.getUid(), variableId, userId); - auditManager.audit(userId, Enums.Action.REMOVE_VARIABLE_FROM_VARIABLE_SET, Enums.Resource.STUDY, + authorizationManager.checkCanCreateUpdateDeleteVariableSets(organizationId, study.getUid(), userId); + OpenCGAResult result = getStudyDBAdaptor(organizationId).removeFieldFromVariableSet(study.getUid(), + variableSet.getUid(), variableId, userId); + auditManager.audit(organizationId, userId, Enums.Action.REMOVE_VARIABLE_FROM_VARIABLE_SET, Enums.Resource.STUDY, variableSet.getId(), "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); - OpenCGAResult queryResult = studyDBAdaptor.getVariableSet(variableSet.getUid(), QueryOptions.empty()); + OpenCGAResult queryResult = getStudyDBAdaptor(organizationId).getVariableSet(variableSet.getUid(), + QueryOptions.empty()); queryResult.setTime(queryResult.getTime() + result.getTime()); return queryResult; } catch (CatalogException e) { - auditManager.audit(userId, Enums.Action.REMOVE_VARIABLE_FROM_VARIABLE_SET, Enums.Resource.STUDY, + auditManager.audit(organizationId, userId, Enums.Action.REMOVE_VARIABLE_FROM_VARIABLE_SET, Enums.Resource.STUDY, variableSet.getId(), "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } - private VariableSet extractVariableSet(Study study, String variableSetId, String userId) throws CatalogException { + private VariableSet extractVariableSet(String organizationId, Study study, String variableSetId, String userId) + throws CatalogException { if (study == null || study.getVariableSets() == null || study.getVariableSets().isEmpty()) { throw new CatalogException(variableSetId + " not found."); } @@ -1485,35 +1624,33 @@ private VariableSet extractVariableSet(Study study, String variableSetId, String .append(StudyDBAdaptor.VariableSetParams.ID.key(), variableSetId); // We query again because we only want to return the variable set if the user can access it - OpenCGAResult queryResult = studyDBAdaptor.getVariableSets(query, new QueryOptions(), userId); + OpenCGAResult queryResult = getStudyDBAdaptor(organizationId).getVariableSets(query, new QueryOptions(), userId); if (queryResult.getNumResults() == 0) { throw new CatalogException(variableSetId + " not found."); } return queryResult.first(); } - public OpenCGAResult renameFieldFromVariableSet(String studyStr, String variableSetStr, String oldName, String newName, - String sessionId) throws CatalogException { - throw new UnsupportedOperationException("Operation not yet supported"); - -// MyResourceId resource = getVariableSetId(variableSetStr, studyStr, sessionId); -// String userId = resource.getUser(); -// -// authorizationManager.checkCanCreateUpdateDeleteVariableSets(resource.getStudyId(), userId); -// OpenCGAResult queryResult = studyDBAdaptor.renameFieldVariableSet(resource.getResourceId(), oldName, newName,userId); -// auditManager.recordDeletion(AuditRecord.Resource.variableSet, resource.getResourceId(), userId, queryResult.first(), null, null); -// return queryResult; - } - - // ************************** ACLs ******************************** // public OpenCGAResult> getAcls(List studyIdList, String member, boolean ignoreException, String token) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(token); - List studyList = resolveIds(studyIdList, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + ParamUtils.checkObj(studyIdList, "studyIdList"); + List fqnList = new ArrayList<>(studyIdList.size()); + for (String studyStr : studyIdList) { + fqnList.add(CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload)); + } + // Validate all studies belong to the same organization + List organizationIds = fqnList.stream().map(CatalogFqn::getOrganizationId).distinct().collect(Collectors.toList()); + if (organizationIds.size() > 1) { + throw new CatalogException("More than organization found for the studies provided: " + StringUtils.join(organizationIds, ", ")); + } + String organizationId = organizationIds.get(0); + String userId = tokenPayload.getUserId(organizationId); ObjectMap auditParams = new ObjectMap() + .append("organizationId", organizationId) .append("studyIdList", studyIdList) .append("member", member) .append("ignoreException", ignoreException) @@ -1522,23 +1659,23 @@ public OpenCGAResult> getAcls(List> studyAclList = OpenCGAResult.empty(); - for (int i = 0; i < studyList.size(); i++) { - Study study = studyList.get(i); + for (CatalogFqn studyFqn : fqnList) { + Study study = resolveId(studyFqn, null, tokenPayload); long studyId = study.getUid(); try { OpenCGAResult> allStudyAcls; if (StringUtils.isNotEmpty(member)) { - allStudyAcls = authorizationManager.getStudyAcl(userId, studyId, member); + allStudyAcls = authorizationManager.getStudyAcl(organizationId, studyId, member, userId); } else { - allStudyAcls = authorizationManager.getAllStudyAcls(userId, studyId); + allStudyAcls = authorizationManager.getAllStudyAcls(organizationId, studyId, userId); } studyAclList.append(allStudyAcls); - auditManager.audit(operationUuid, userId, Enums.Action.FETCH_ACLS, Enums.Resource.STUDY, study.getId(), + auditManager.audit(organizationId, operationUuid, userId, Enums.Action.FETCH_ACLS, Enums.Resource.STUDY, study.getId(), study.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS), new ObjectMap()); } catch (CatalogException e) { - auditManager.audit(operationUuid, userId, Enums.Action.FETCH_ACLS, Enums.Resource.STUDY, study.getId(), + auditManager.audit(organizationId, operationUuid, userId, Enums.Action.FETCH_ACLS, Enums.Resource.STUDY, study.getId(), study.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError()), new ObjectMap()); if (ignoreException) { @@ -1552,30 +1689,23 @@ public OpenCGAResult> getAcls(List> updateAcl(String studyId, String memberIds, StudyAclParams aclParams, + public OpenCGAResult> updateAcl(String studyStr, String memberIds, StudyAclParams aclParams, ParamUtils.AclAction action, String token) throws CatalogException { - return updateAcl(Collections.singletonList(studyId), memberIds, aclParams, action, token); - } - - public OpenCGAResult> updateAcl(List studyIdList, String memberIds, - StudyAclParams aclParams, ParamUtils.AclAction action, - String token) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(token); - List studies = resolveIds(studyIdList, userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = resolveId(studyFqn, QueryOptions.empty(), tokenPayload); ObjectMap auditParams = new ObjectMap() - .append("studyIdList", studyIdList) + .append("studyStr", studyStr) .append("memberIds", memberIds) .append("aclParams", aclParams) .append("action", action) .append("token", token); String operationUuid = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); try { - if (studyIdList == null || studyIdList.isEmpty()) { - throw new CatalogException("Missing study parameter"); - } - if (action == null) { throw new CatalogException("Invalid action found. Please choose a valid action to be performed."); } @@ -1620,9 +1750,8 @@ public OpenCGAResult> updateAcl(List< } // Check the user has the permissions needed to change permissions - for (Study study : studies) { - authorizationManager.checkCanAssignOrSeePermissions(study.getUid(), userId); - } + authorizationManager.checkCanAssignOrSeePermissions(organizationId, study.getUid(), userId); + // Validate that the members are actually valid members List members; @@ -1632,88 +1761,55 @@ public OpenCGAResult> updateAcl(List< members = Collections.emptyList(); } authorizationManager.checkNotAssigningPermissionsToAdminsGroup(members); - for (Study study : studies) { - checkMembers(study.getUid(), members); - } - - List studyUidList = studies - .stream() - .map(Study::getUid) - .collect(Collectors.toList()); + checkMembers(organizationId, study.getUid(), members); switch (action) { case SET: - authorizationManager.setStudyAcls(studyUidList, members, permissions); + authorizationManager.setStudyAcls(organizationId, Collections.singletonList(study.getUid()), members, permissions); break; case ADD: - authorizationManager.addStudyAcls(studyUidList, members, permissions); + authorizationManager.addStudyAcls(organizationId, Collections.singletonList(study.getUid()), members, permissions); break; case REMOVE: - authorizationManager.removeStudyAcls(studyUidList, members, permissions); + authorizationManager.removeStudyAcls(organizationId, Collections.singletonList(study.getUid()), members, permissions); break; case RESET: - authorizationManager.removeStudyAcls(studyUidList, members, null); + authorizationManager.removeStudyAcls(organizationId, Collections.singletonList(study.getUid()), members, null); break; default: throw new CatalogException("Unexpected error occurred. No valid action found."); } - OpenCGAResult> remainingAcls = authorizationManager.getStudyAcl(studyUidList, - members); + OpenCGAResult> remainingAcls = authorizationManager.getStudyAcl(organizationId, + study.getUid(), members); - for (Study study : studies) { - auditManager.audit(operationUuid, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.STUDY, study.getId(), - study.getUuid(), study.getId(), study.getUuid(), auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS), new ObjectMap()); - } + auditManager.audit(organizationId, operationUuid, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.STUDY, study.getId(), + study.getUuid(), study.getId(), study.getUuid(), auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS), new ObjectMap()); return remainingAcls; } catch (CatalogException e) { - for (Study study : studies) { - auditManager.audit(operationUuid, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.STUDY, study.getId(), - study.getUuid(), study.getId(), study.getUuid(), auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError()), new ObjectMap()); - } + auditManager.audit(organizationId, operationUuid, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.STUDY, study.getId(), + study.getUuid(), study.getId(), study.getUuid(), auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError()), new ObjectMap()); throw e; } } - public Map facet(String studyStr, String fileFields, String sampleFields, String individualFields, String cohortFields, - String familyFields, String jobFields, boolean defaultStats, String sessionId) - throws CatalogException, IOException { - Map result = new HashMap<>(); - result.put("sample", catalogManager.getSampleManager().facet(studyStr, new Query(), setFacetFields(sampleFields), defaultStats, - sessionId)); - result.put("file", catalogManager.getFileManager().facet(studyStr, new Query(), setFacetFields(fileFields), defaultStats, - sessionId)); - result.put("individual", catalogManager.getIndividualManager().facet(studyStr, new Query(), setFacetFields(individualFields), - defaultStats, sessionId)); - result.put("family", catalogManager.getFamilyManager().facet(studyStr, new Query(), setFacetFields(familyFields), defaultStats, - sessionId)); - result.put("cohort", catalogManager.getCohortManager().facet(studyStr, new Query(), setFacetFields(cohortFields), defaultStats, - sessionId)); - result.put("job", catalogManager.getJobManager().facet(studyStr, new Query(), setFacetFields(jobFields), defaultStats, - sessionId)); - - return result; - } - - private QueryOptions setFacetFields(String fields) { - QueryOptions queryOptions = new QueryOptions(); - queryOptions.putIfNotEmpty(QueryOptions.FACET, fields); - return queryOptions; - } - // ************************** Protected internal methods ******************************** // public void setVariantEngineConfigurationOptions(String studyStr, ObjectMap options, String token) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(token); - Study study = get(studyStr, new QueryOptions(QueryOptions.INCLUDE, Arrays.asList( - StudyDBAdaptor.QueryParams.UID.key(), - StudyDBAdaptor.QueryParams.INTERNAL_CONFIGURATION_VARIANT_ENGINE.key())), token).first(); - - authorizationManager.checkIsOwnerOrAdmin(study.getUid(), userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = resolveId(studyFqn, + new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(StudyDBAdaptor.QueryParams.UID.key(), + StudyDBAdaptor.QueryParams.INTERNAL_CONFIGURATION_VARIANT_ENGINE.key())), + tokenPayload); + + authorizationManager.checkIsAtLeastStudyAdministrator(organizationId, study.getUid(), userId); StudyVariantEngineConfiguration configuration = study.getInternal().getConfiguration().getVariantEngine(); if (configuration == null) { configuration = new StudyVariantEngineConfiguration(); @@ -1721,17 +1817,55 @@ public void setVariantEngineConfigurationOptions(String studyStr, ObjectMap opti configuration.setOptions(options); ObjectMap parameters = new ObjectMap(StudyDBAdaptor.QueryParams.INTERNAL_CONFIGURATION_VARIANT_ENGINE.key(), configuration); - studyDBAdaptor.update(study.getUid(), parameters, QueryOptions.empty()); + getStudyDBAdaptor(organizationId).update(study.getUid(), parameters, QueryOptions.empty()); + } + + public void setVariantEngineSetupOptions(String studyStr, VariantSetupResult variantSetupResult, String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = resolveId(studyFqn, + new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(StudyDBAdaptor.QueryParams.UID.key(), + StudyDBAdaptor.QueryParams.INTERNAL_CONFIGURATION_VARIANT_ENGINE.key())), + tokenPayload); + + authorizationManager.checkIsAtLeastStudyAdministrator(organizationId, study.getUid(), userId); + StudyVariantEngineConfiguration configuration = study.getInternal().getConfiguration().getVariantEngine(); + if (configuration == null) { + configuration = new StudyVariantEngineConfiguration(); + } + if (configuration.getOptions() == null) { + configuration.setOptions(new ObjectMap()); + } + VariantSetupResult prevSetupValue = configuration.getSetup(); + if (prevSetupValue != null && prevSetupValue.getOptions() != null) { + // Variant setup was already executed. + // Remove the options from the previous execution before adding the new ones + // Check that both key/value matches, to avoid removing options that might have been modified manually + for (Map.Entry entry : prevSetupValue.getOptions().entrySet()) { + configuration.getOptions().remove(entry.getKey(), entry.getValue()); + } + } + configuration.getOptions().putAll(variantSetupResult.getOptions()); + configuration.setSetup(variantSetupResult); + + ObjectMap parameters = new ObjectMap(StudyDBAdaptor.QueryParams.INTERNAL_CONFIGURATION_VARIANT_ENGINE.key(), configuration); + getStudyDBAdaptor(organizationId).update(study.getUid(), parameters, QueryOptions.empty()); } public void setVariantEngineConfigurationSampleIndex(String studyStr, SampleIndexConfiguration sampleIndexConfiguration, String token) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(token); - Study study = get(studyStr, new QueryOptions(QueryOptions.INCLUDE, Arrays.asList( - StudyDBAdaptor.QueryParams.UID.key(), - StudyDBAdaptor.QueryParams.INTERNAL_CONFIGURATION_VARIANT_ENGINE.key())), token).first(); - - authorizationManager.checkIsOwnerOrAdmin(study.getUid(), userId); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = resolveId(studyFqn, + new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(StudyDBAdaptor.QueryParams.UID.key(), + StudyDBAdaptor.QueryParams.INTERNAL_CONFIGURATION_VARIANT_ENGINE.key())), + tokenPayload); + + authorizationManager.checkIsAtLeastStudyAdministrator(organizationId, study.getUid(), userId); StudyVariantEngineConfiguration configuration = study.getInternal().getConfiguration().getVariantEngine(); if (configuration == null) { configuration = new StudyVariantEngineConfiguration(); @@ -1739,19 +1873,20 @@ public void setVariantEngineConfigurationSampleIndex(String studyStr, SampleInde configuration.setSampleIndex(sampleIndexConfiguration); ObjectMap parameters = new ObjectMap(StudyDBAdaptor.QueryParams.INTERNAL_CONFIGURATION_VARIANT_ENGINE.key(), configuration); - studyDBAdaptor.update(study.getUid(), parameters, QueryOptions.empty()); + getStudyDBAdaptor(organizationId).update(study.getUid(), parameters, QueryOptions.empty()); } // ************************** Private methods ******************************** // - private boolean existsGroup(long studyId, String groupId) throws CatalogDBException { + private boolean existsGroup(String organizationId, long studyId, String groupId) throws CatalogDBException { Query query = new Query() .append(StudyDBAdaptor.QueryParams.UID.key(), studyId) .append(StudyDBAdaptor.QueryParams.GROUP_ID.key(), groupId); - return studyDBAdaptor.count(query).getNumMatches() > 0; + return getStudyDBAdaptor(organizationId).count(query).getNumMatches() > 0; } - private void validatePermissionRules(long studyId, Enums.Entity entry, PermissionRule permissionRule) throws CatalogException { + private void validatePermissionRules(String organizationId, long studyId, Enums.Entity entry, PermissionRule permissionRule) + throws CatalogException { ParamUtils.checkIdentifier(permissionRule.getId(), "PermissionRules"); if (permissionRule.getPermissions() == null || permissionRule.getPermissions().isEmpty()) { @@ -1784,7 +1919,7 @@ private void validatePermissionRules(long studyId, Enums.Entity entry, Permissio throw new CatalogException("Unexpected entry found"); } - checkMembers(studyId, permissionRule.getMembers()); + checkMembers(organizationId, studyId, permissionRule.getMembers()); } private void validatePermissions(List permissions, Function valueOf) throws CatalogException { @@ -1798,23 +1933,8 @@ private void validatePermissions(List permissions, Function uploadTemplate(String studyStr, String filename, InputStream inputStream, String token) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(token); - Study study = resolveId(studyStr, userId, QueryOptions.empty()); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = resolveId(studyFqn, QueryOptions.empty(), tokenPayload); String templateId = "template." + TimeUtils.getTime() + "." + RandomStringUtils.random(6, true, false); ObjectMap auditParams = new ObjectMap() .append("studyStr", studyStr) + .append("filename", filename) .append("token", token); try { StopWatch stopWatch = StopWatch.createStarted(); - authorizationManager.checkIsOwnerOrAdmin(study.getUid(), userId); + authorizationManager.checkIsAtLeastStudyAdministrator(organizationId, study.getUid(), userId); ParamUtils.checkParameter(filename, "File name"); if (!filename.endsWith(".zip") && !filename.endsWith(".tar.gz")) { @@ -1897,12 +2021,12 @@ public OpenCGAResult uploadTemplate(String studyStr, String filename, In return new OpenCGAResult<>((int) stopWatch.getTime(TimeUnit.MILLISECONDS), Collections.emptyList(), 1, Collections.singletonList(templateId), 1); } catch (CatalogException e) { - auditManager.auditCreate(userId, Enums.Action.UPLOAD_TEMPLATE, Enums.Resource.STUDY, templateId, "", study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditCreate(organizationId, userId, Enums.Action.UPLOAD_TEMPLATE, Enums.Resource.STUDY, templateId, "", + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } catch (Exception e) { - auditManager.auditCreate(userId, Enums.Action.UPLOAD_TEMPLATE, Enums.Resource.STUDY, templateId, "", study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, + auditManager.auditCreate(organizationId, userId, Enums.Action.UPLOAD_TEMPLATE, Enums.Resource.STUDY, templateId, "", + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(-1, "template upload", e.getMessage()))); throw e; } finally { @@ -1924,8 +2048,11 @@ public OpenCGAResult uploadTemplate(String studyStr, String filename, In * @throws CatalogException if there is any issue with the upload. */ public OpenCGAResult deleteTemplate(String studyStr, String templateId, String token) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(token); - Study study = resolveId(studyStr, userId, QueryOptions.empty()); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = resolveId(studyFqn, QueryOptions.empty(), tokenPayload); ObjectMap auditParams = new ObjectMap() .append("studyStr", studyStr) @@ -1934,7 +2061,7 @@ public OpenCGAResult deleteTemplate(String studyStr, String templateId, try { StopWatch stopWatch = StopWatch.createStarted(); - authorizationManager.checkIsOwnerOrAdmin(study.getUid(), userId); + authorizationManager.checkIsAtLeastStudyAdministrator(organizationId, study.getUid(), userId); // We obtain the basic studyPath where we will upload the file temporarily java.nio.file.Path studyPath = Paths.get(study.getUri()); @@ -1954,12 +2081,12 @@ public OpenCGAResult deleteTemplate(String studyStr, String templateId, return new OpenCGAResult<>((int) stopWatch.getTime(TimeUnit.MILLISECONDS), null, 1, Collections.singletonList(true), 1); } catch (CatalogException e) { - auditManager.auditCreate(userId, Enums.Action.DELETE_TEMPLATE, Enums.Resource.STUDY, templateId, "", study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditCreate(organizationId, userId, Enums.Action.DELETE_TEMPLATE, Enums.Resource.STUDY, templateId, "", + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } catch (Exception e) { - auditManager.auditCreate(userId, Enums.Action.DELETE_TEMPLATE, Enums.Resource.STUDY, templateId, "", study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, + auditManager.auditCreate(organizationId, userId, Enums.Action.DELETE_TEMPLATE, Enums.Resource.STUDY, templateId, "", + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(-1, "template delete", e.getMessage()))); throw e; } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java index a789f25b443..7e8e8cdffeb 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java @@ -17,23 +17,21 @@ package org.opencb.opencga.catalog.managers; import org.apache.commons.lang3.StringUtils; -import org.opencb.commons.datastore.core.Event; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.result.Error; import org.opencb.commons.utils.ListUtils; import org.opencb.opencga.catalog.auth.authentication.AuthenticationManager; -import org.opencb.opencga.catalog.auth.authentication.AzureADAuthenticationManager; import org.opencb.opencga.catalog.auth.authentication.CatalogAuthenticationManager; -import org.opencb.opencga.catalog.auth.authentication.LDAPAuthenticationManager; +import org.opencb.opencga.catalog.auth.authentication.azure.AuthenticationFactory; import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager; import org.opencb.opencga.catalog.db.DBAdaptorFactory; -import org.opencb.opencga.catalog.db.api.ProjectDBAdaptor; import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; import org.opencb.opencga.catalog.db.api.UserDBAdaptor; import org.opencb.opencga.catalog.exceptions.*; import org.opencb.opencga.catalog.io.CatalogIOManager; +import org.opencb.opencga.catalog.utils.CatalogFqn; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.catalog.utils.UuidUtils; import org.opencb.opencga.core.api.ParamConstants; @@ -41,12 +39,13 @@ import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.AuthenticationOrigin; import org.opencb.opencga.core.config.Configuration; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.audit.AuditRecord; import org.opencb.opencga.core.models.common.Enums; -import org.opencb.opencga.core.models.project.Project; +import org.opencb.opencga.core.models.common.InternalStatus; +import org.opencb.opencga.core.models.organizations.Organization; import org.opencb.opencga.core.models.study.Group; import org.opencb.opencga.core.models.study.GroupUpdateParams; -import org.opencb.opencga.core.models.study.Study; import org.opencb.opencga.core.models.user.*; import org.opencb.opencga.core.response.OpenCGAResult; import org.slf4j.Logger; @@ -64,75 +63,23 @@ */ public class UserManager extends AbstractManager { - static final QueryOptions INCLUDE_ACCOUNT = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList( - UserDBAdaptor.QueryParams.ID.key(), UserDBAdaptor.QueryParams.ACCOUNT.key())); + static final QueryOptions INCLUDE_INTERNAL = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(UserDBAdaptor.QueryParams.ID.key(), + UserDBAdaptor.QueryParams.INTERNAL.key(), UserDBAdaptor.QueryParams.DEPRECATED_ACCOUNT.key())); protected static Logger logger = LoggerFactory.getLogger(UserManager.class); private final CatalogIOManager catalogIOManager; - private final String INTERNAL_AUTHORIZATION = CatalogAuthenticationManager.INTERNAL; - private final Map authenticationManagerMap; + private final AuthenticationFactory authenticationFactory; UserManager(AuthorizationManager authorizationManager, AuditManager auditManager, CatalogManager catalogManager, - DBAdaptorFactory catalogDBAdaptorFactory, CatalogIOManager catalogIOManager, Configuration configuration) + DBAdaptorFactory catalogDBAdaptorFactory, CatalogIOManager catalogIOManager, + AuthenticationFactory authenticationFactory, Configuration configuration) throws CatalogException { super(authorizationManager, auditManager, catalogManager, catalogDBAdaptorFactory, configuration); this.catalogIOManager = catalogIOManager; - - String secretKey = configuration.getAdmin().getSecretKey(); - long expiration = configuration.getAuthentication().getExpiration(); - - authenticationManagerMap = new LinkedHashMap<>(); - if (configuration.getAuthentication().getAuthenticationOrigins() != null) { - for (AuthenticationOrigin authenticationOrigin : configuration.getAuthentication().getAuthenticationOrigins()) { - if (authenticationOrigin.getId() != null) { - switch (authenticationOrigin.getType()) { - case LDAP: - authenticationManagerMap.put(authenticationOrigin.getId(), - new LDAPAuthenticationManager(authenticationOrigin, secretKey, expiration)); - break; - case AzureAD: - authenticationManagerMap.put(authenticationOrigin.getId(), - new AzureADAuthenticationManager(authenticationOrigin)); - break; - case OPENCGA: - authenticationManagerMap.put(authenticationOrigin.getId(), - new CatalogAuthenticationManager(catalogDBAdaptorFactory, configuration.getEmail(), secretKey, - expiration)); - break; - default: - logger.warn("Unexpected authentication origin type '{}' for id '{}' found in the configuration file. " - + "Authentication origin will be ignored.", authenticationOrigin.getType(), - authenticationOrigin.getId()); - break; - } - } - } - } - // Even if internal authentication is not present in the configuration file, create it - authenticationManagerMap.putIfAbsent(INTERNAL_AUTHORIZATION, - new CatalogAuthenticationManager(catalogDBAdaptorFactory, configuration.getEmail(), secretKey, expiration)); - AuthenticationOrigin authenticationOrigin = new AuthenticationOrigin(); - if (configuration.getAuthentication().getAuthenticationOrigins() == null) { - configuration.getAuthentication().setAuthenticationOrigins(Arrays.asList(authenticationOrigin)); - } else { - // Check if OPENCGA authentication is already present in catalog configuration - boolean catalogPresent = false; - for (AuthenticationOrigin origin : configuration.getAuthentication().getAuthenticationOrigins()) { - if (AuthenticationOrigin.AuthenticationType.OPENCGA == origin.getType()) { - catalogPresent = true; - break; - } - } - if (!catalogPresent) { - List linkedList = new LinkedList<>(); - linkedList.addAll(configuration.getAuthentication().getAuthenticationOrigins()); - linkedList.add(authenticationOrigin); - configuration.getAuthentication().setAuthenticationOrigins(linkedList); - } - } + this.authenticationFactory = authenticationFactory; } - public void changePassword(String userId, String oldPassword, String newPassword) throws CatalogException { + public void changePassword(String organizationId, String userId, String oldPassword, String newPassword) throws CatalogException { ParamUtils.checkParameter(userId, "userId"); ParamUtils.checkParameter(oldPassword, "oldPassword"); ParamUtils.checkParameter(newPassword, "newPassword"); @@ -140,92 +87,131 @@ public void changePassword(String userId, String oldPassword, String newPassword if (oldPassword.equals(newPassword)) { throw new CatalogException("New password is the same as the old password."); } + if (!PasswordUtils.isStrongPassword(newPassword)) { + throw new CatalogException("Invalid password. " + PasswordUtils.PASSWORD_REQUIREMENT); + } - userDBAdaptor.checkId(userId); - String authOrigin = getAuthenticationOriginId(userId); - authenticationManagerMap.get(authOrigin).changePassword(userId, oldPassword, newPassword); - auditManager.auditUser(userId, Enums.Action.CHANGE_USER_PASSWORD, userId, + getUserDBAdaptor(organizationId).checkId(userId); + String authOrigin = getAuthenticationOriginId(organizationId, userId); + authenticationFactory.changePassword(organizationId, authOrigin, userId, oldPassword, newPassword); + auditManager.auditUser(organizationId, userId, Enums.Action.CHANGE_USER_PASSWORD, userId, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { - auditManager.auditUser(userId, Enums.Action.CHANGE_USER_PASSWORD, userId, + auditManager.auditUser(organizationId, userId, Enums.Action.CHANGE_USER_PASSWORD, userId, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } - public OpenCGAResult create(User user, String password, @Nullable String token) throws CatalogException { - // Check if the users can be registered publicly or just the admin. + public OpenCGAResult create(User user, String password, String token) throws CatalogException { + if (StringUtils.isEmpty(user.getOrganization())) { + if (StringUtils.isEmpty(token)) { + throw CatalogParameterException.isNull("user.organization"); + } + JwtPayload payload = new JwtPayload(token); + if (ParamConstants.ADMIN_ORGANIZATION.equals(payload.getOrganization())) { + // Administrators always need to provide the user organization + throw CatalogParameterException.isNull("user.organization"); + } + logger.warn("User organization was missing. Setting it with the token's organization id '{}'.", payload.getOrganization()); + user.setOrganization(payload.getOrganization()); + } + String organizationId = user.getOrganization(); + + if (token == null && (!ParamConstants.ADMIN_ORGANIZATION.equals(organizationId) || !OPENCGA.equals(user.getId()))) { + throw new CatalogException("Missing token parameter"); + } + if (OPENCGA.equals(user.getId()) && !ParamConstants.ADMIN_ORGANIZATION.equals(organizationId)) { + throw new CatalogException("Creating '" + OPENCGA + "' user is forbidden in any organization."); + } + + Organization organization = getOrganizationDBAdaptor(organizationId).get(OrganizationManager.INCLUDE_ORGANIZATION_CONFIGURATION) + .first(); + ObjectMap auditParams = new ObjectMap("user", user); // Initialise fields ParamUtils.checkObj(user, "User"); ParamUtils.checkValidUserId(user.getId()); - ParamUtils.checkParameter(user.getName(), "name"); + user.setName(ParamUtils.defaultString(user.getName(), user.getId())); user.setEmail(ParamUtils.defaultString(user.getEmail(), "")); if (StringUtils.isNotEmpty(user.getEmail())) { checkEmail(user.getEmail()); } - user.setOrganization(ParamUtils.defaultObject(user.getOrganization(), "")); - ParamUtils.checkObj(user.getAccount(), "account"); - user.getAccount().setType(ParamUtils.defaultObject(user.getAccount().getType(), Account.AccountType.GUEST)); - user.getAccount().setCreationDate(TimeUtils.getTime()); - user.getAccount().setExpirationDate(ParamUtils.defaultString(user.getAccount().getExpirationDate(), "")); - user.setInternal(new UserInternal(new UserStatus(UserStatus.READY))); + user.setCreationDate(ParamUtils.checkDateOrGetCurrentDate(user.getCreationDate(), + UserDBAdaptor.QueryParams.CREATION_DATE.key())); + user.setModificationDate(ParamUtils.checkDateOrGetCurrentDate(user.getModificationDate(), + UserDBAdaptor.QueryParams.MODIFICATION_DATE.key())); + + user.setInternal(ParamUtils.defaultObject(user.getInternal(), UserInternal::new)); + user.getInternal().setStatus(new UserStatus(InternalStatus.READY)); user.setQuota(ParamUtils.defaultObject(user.getQuota(), UserQuota::new)); user.setProjects(ParamUtils.defaultObject(user.getProjects(), Collections::emptyList)); - user.setSharedProjects(ParamUtils.defaultObject(user.getSharedProjects(), Collections::emptyList)); user.setConfigs(ParamUtils.defaultObject(user.getConfigs(), HashMap::new)); user.setFilters(ParamUtils.defaultObject(user.getFilters(), LinkedList::new)); user.setAttributes(ParamUtils.defaultObject(user.getAttributes(), Collections::emptyMap)); + // Init account + user.getInternal().setAccount(ParamUtils.defaultObject(user.getInternal().getAccount(), Account::new)); + Account account = user.getInternal().getAccount(); + account.setPassword(ParamUtils.defaultObject(account.getPassword(), Password::new)); + if (StringUtils.isEmpty(account.getExpirationDate())) { + account.setExpirationDate(organization.getConfiguration().getDefaultUserExpirationDate()); + } else { + // Validate expiration date is not over + ParamUtils.checkDateIsNotExpired(account.getExpirationDate(), UserDBAdaptor.QueryParams.INTERNAL_ACCOUNT_EXPIRATION_DATE.key()); + } if (StringUtils.isEmpty(password)) { - // The authentication origin must be different than internal - Set authOrigins = configuration.getAuthentication().getAuthenticationOrigins() - .stream() - .map(AuthenticationOrigin::getId) - .collect(Collectors.toSet()); - if (!authOrigins.contains(user.getAccount().getAuthentication().getId())) { - throw new CatalogException("Unknown authentication origin id '" + user.getAccount().getAuthentication() + "'"); + Map authOriginMap = authenticationFactory.getOrganizationAuthenticationManagers(organizationId); + if (!authOriginMap.containsKey(account.getAuthentication().getId())) { + throw new CatalogException("Unknown authentication origin id '" + account.getAuthentication() + "'"); } } else { - user.getAccount().setAuthentication(new Account.AuthenticationOrigin(INTERNAL_AUTHORIZATION, false)); + account.setAuthentication(new Account.AuthenticationOrigin(CatalogAuthenticationManager.OPENCGA, false)); } - String userId = user.getId(); - if (!OPENCGA.equals(user.getId())) { - userId = authenticationManagerMap.get(INTERNAL_AUTHORIZATION).getUserId(token); - if (!OPENCGA.equals(userId)) { - String errorMsg = "The registration is closed to the public: Please talk to your administrator."; - auditManager.auditCreate(userId, Enums.Resource.USER, user.getId(), "", "", "", auditParams, + // Set password expiration + if (AuthenticationOrigin.AuthenticationType.OPENCGA.name().equals(account.getAuthentication().getId())) { + account.getPassword().setLastModified(TimeUtils.getTime()); + } + if (!AuthenticationOrigin.AuthenticationType.OPENCGA.name().equals(account.getAuthentication().getId()) + || configuration.getAccount().getPasswordExpirationDays() <= 0) { + // User password doesn't expire or it's not managed by OpenCGA + account.getPassword().setExpirationDate(null); + } else { + Date date = TimeUtils.addDaysToCurrentDate(configuration.getAccount().getPasswordExpirationDays()); + account.getPassword().setExpirationDate(TimeUtils.getTime(date)); + } + + if (!ParamConstants.ADMIN_ORGANIZATION.equals(organizationId) || !OPENCGA.equals(user.getId())) { + JwtPayload jwtPayload = validateToken(token); + // If it's not one of the SUPERADMIN users or the owner or one of the admins of the organisation, we should not allow it + if (!authorizationManager.isAtLeastOrganizationOwnerOrAdmin(organizationId, jwtPayload.getUserId(organizationId))) { + String errorMsg = "Please ask your administrator to create your account."; + auditManager.auditCreate(organizationId, user.getId(), Enums.Resource.USER, user.getId(), "", "", "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(0, "", errorMsg))); throw new CatalogException(errorMsg); } } - checkUserExists(user.getId()); + checkUserExists(organizationId, user.getId()); try { if (StringUtils.isNotEmpty(password) && !PasswordUtils.isStrongPassword(password)) { - throw new CatalogException("Invalid password. Check password strength for user " + user.getId()); + throw new CatalogException("Invalid password. " + PasswordUtils.PASSWORD_REQUIREMENT); } if (user.getProjects() != null && !user.getProjects().isEmpty()) { throw new CatalogException("Creating user and projects in a single transaction is forbidden"); } - catalogIOManager.createUser(user.getId()); - userDBAdaptor.insert(user, password, QueryOptions.empty()); + getUserDBAdaptor(organizationId).insert(user, password, QueryOptions.empty()); - auditManager.auditCreate(userId, Enums.Resource.USER, user.getId(), "", "", "", auditParams, + auditManager.auditCreate(organizationId, user.getId(), Enums.Resource.USER, user.getId(), "", "", "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); - return userDBAdaptor.get(user.getId(), QueryOptions.empty()); + return getUserDBAdaptor(organizationId).get(user.getId(), QueryOptions.empty()); } catch (CatalogIOException | CatalogDBException e) { - if (userDBAdaptor.exists(user.getId())) { - logger.error("ERROR! DELETING USER! " + user.getId()); - catalogIOManager.deleteUser(user.getId()); - } - - auditManager.auditCreate(userId, Enums.Resource.USER, user.getId(), "", "", "", auditParams, + auditManager.auditCreate(organizationId, user.getId(), Enums.Resource.USER, user.getId(), "", "", "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; @@ -241,21 +227,83 @@ public OpenCGAResult create(User user, String password, @Nullable String t * @param password Encrypted Password * @param organization Optional organization * @param quota Maximum user disk quota - * @param type User account type. Full or guest. * @param token JWT token. * @return The created user * @throws CatalogException If user already exists, or unable to create a new user. */ - public OpenCGAResult create(String id, String name, String email, String password, String organization, Long quota, - Account.AccountType type, String token) throws CatalogException { + public OpenCGAResult create(String id, String name, String email, String password, String organization, Long quota, String token) + throws CatalogException { User user = new User(id, name, email, organization, new UserInternal(new UserStatus())) - .setAccount(new Account(type, "", "", null)) .setQuota(new UserQuota().setMaxDisk(quota != null ? quota : -1)); return create(user, password, token); } - public void syncAllUsersOfExternalGroup(String study, String authOrigin, String token) throws CatalogException { - if (!OPENCGA.equals(authenticationManagerMap.get(INTERNAL_AUTHORIZATION).getUserId(token))) { + /** + * Search users from Organization. Token must belong to at least an Organization administrator. + * + * @param organizationId Organization id. + * @param query Query object. + * @param options QueryOptions object. + * @param token JWT token. + * @return OpenCGAResult with the list of users. + * @throws CatalogException if the token does not belong to an Organization administrator or there are any parameters wrong. + */ + public OpenCGAResult search(@Nullable String organizationId, Query query, QueryOptions options, String token) + throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + ObjectMap auditParams = new ObjectMap() + .append("organizationId", organizationId) + .append("query", query) + .append("options", options) + .append("token", token); + + options = ParamUtils.defaultObject(options, QueryOptions::new); + String myOrganizationId = StringUtils.isNotEmpty(organizationId) ? organizationId : tokenPayload.getOrganization(); + try { + authorizationManager.checkIsAtLeastOrganizationOwnerOrAdmin(myOrganizationId, tokenPayload.getUserId(myOrganizationId)); + + // Fix query params + if (query.containsKey(ParamConstants.USER_AUTHENTICATION_ORIGIN)) { + query.put(UserDBAdaptor.QueryParams.INTERNAL_ACCOUNT_AUTHENTICATION_ID.key(), + query.get(ParamConstants.USER_AUTHENTICATION_ORIGIN)); + query.remove(ParamConstants.USER_AUTHENTICATION_ORIGIN); + } + + OpenCGAResult result = getUserDBAdaptor(myOrganizationId).get(query, options); + auditManager.auditSearch(myOrganizationId, tokenPayload.getUserId(myOrganizationId), Enums.Resource.USER, "", "", auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + return result; + } catch (Exception e) { + auditManager.auditSearch(myOrganizationId, tokenPayload.getUserId(myOrganizationId), Enums.Resource.USER, "", "", auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(0, "User search", e.getMessage()))); + throw e; + } + } + + public JwtPayload validateToken(String token) throws CatalogException { + JwtPayload jwtPayload = new JwtPayload(token); + ParamUtils.checkParameter(jwtPayload.getUserId(), "jwt user"); + ParamUtils.checkParameter(jwtPayload.getOrganization(), "jwt organization"); + + String authOrigin; + if (ParamConstants.ANONYMOUS_USER_ID.equals(jwtPayload.getUserId())) { + authOrigin = CatalogAuthenticationManager.OPENCGA; + } else { + OpenCGAResult userResult = getUserDBAdaptor(jwtPayload.getOrganization()).get(jwtPayload.getUserId(), + INCLUDE_INTERNAL); + if (userResult.getNumResults() == 0) { + throw new CatalogException("User '" + jwtPayload.getUserId() + "' could not be found."); + } + User user = userResult.first(); + authOrigin = user.getInternal().getAccount().getAuthentication().getId(); + } + + authenticationFactory.validateToken(jwtPayload.getOrganization(), authOrigin, token); + return jwtPayload; + } + + public void syncAllUsersOfExternalGroup(String organizationId, String study, String authOrigin, String token) throws CatalogException { + if (!OPENCGA.equals(authenticationFactory.getUserId(organizationId, authOrigin, token))) { throw new CatalogAuthorizationException("Only the root user can perform this action"); } @@ -270,8 +318,8 @@ public void syncAllUsersOfExternalGroup(String study, String authOrigin, String List userList; try { - userList = authenticationManagerMap.get(group.getSyncedFrom().getAuthOrigin()) - .getUsersFromRemoteGroup(group.getSyncedFrom().getRemoteGroup()); + userList = authenticationFactory.getUsersFromRemoteGroup(organizationId, group.getSyncedFrom().getAuthOrigin(), + group.getSyncedFrom().getRemoteGroup()); } catch (CatalogException e) { // There was some kind of issue for which we could not retrieve the group information. logger.info("Removing all users from group '{}' belonging to group '{}' in the external authentication origin", @@ -285,6 +333,7 @@ public void syncAllUsersOfExternalGroup(String study, String authOrigin, String Iterator iterator = userList.iterator(); while (iterator.hasNext()) { User user = iterator.next(); + user.setOrganization(organizationId); try { create(user, null, token); logger.info("User '{}' ({}) successfully created", user.getId(), user.getName()); @@ -307,7 +356,8 @@ public void syncAllUsersOfExternalGroup(String study, String authOrigin, String logger.info("Associating members to the internal OpenCGA group"); updateParams = new GroupUpdateParams(new ArrayList<>(userList.stream().map(User::getId).collect(Collectors.toSet()))); } - catalogManager.getStudyManager().updateGroup(study, group.getId(), ParamUtils.BasicUpdateAction.SET, updateParams, token); + catalogManager.getStudyManager().updateGroup(study, group.getId(), ParamUtils.BasicUpdateAction.SET, + updateParams, token); } } if (!foundAny) { @@ -319,19 +369,22 @@ public void syncAllUsersOfExternalGroup(String study, String authOrigin, String * Register all the users belonging to a remote group. If internalGroup and study are not null, it will also associate the remote group * to the internalGroup defined. * - * @param authOrigin Authentication origin. - * @param remoteGroup Group name of the remote authentication origin. - * @param internalGroup Group name in Catalog that will be associated to the remote group. - * @param study Study where the internal group will be associated. - * @param sync Boolean indicating whether the remote group will be synced with the internal group or not. - * @param token JWT token. The token should belong to the root user. + * @param organizationId Organization id. + * @param authOrigin Authentication origin. + * @param remoteGroup Group name of the remote authentication origin. + * @param internalGroup Group name in Catalog that will be associated to the remote group. + * @param study Study where the internal group will be associated. + * @param sync Boolean indicating whether the remote group will be synced with the internal group or not. + * @param token JWT token. The token should belong to the root user. * @throws CatalogException If any of the parameters is wrong or there is any internal error. */ - public void importRemoteGroupOfUsers(String authOrigin, String remoteGroup, @Nullable String internalGroup, @Nullable String study, - boolean sync, String token) throws CatalogException { - String userId = getUserId(token); + public void importRemoteGroupOfUsers(String organizationId, String authOrigin, String remoteGroup, @Nullable String internalGroup, + @Nullable String study, boolean sync, String token) throws CatalogException { + JwtPayload payload = validateToken(token); + String userId = payload.getUserId(organizationId); ObjectMap auditParams = new ObjectMap() + .append("organizationId", organizationId) .append("authOrigin", authOrigin) .append("remoteGroup", remoteGroup) .append("internalGroup", internalGroup) @@ -339,17 +392,13 @@ public void importRemoteGroupOfUsers(String authOrigin, String remoteGroup, @Nul .append("sync", sync) .append("token", token); try { - if (!OPENCGA.equals(authenticationManagerMap.get(INTERNAL_AUTHORIZATION).getUserId(token))) { + if (!OPENCGA.equals(authenticationFactory.getUserId(organizationId, authOrigin, token))) { throw new CatalogAuthorizationException("Only the root user can perform this action"); } ParamUtils.checkParameter(authOrigin, "Authentication origin"); ParamUtils.checkParameter(remoteGroup, "Remote group"); - if (!authenticationManagerMap.containsKey(authOrigin)) { - throw new CatalogException("Unknown authentication origin"); - } - List userList; if (sync) { // We don't create any user as they will be automatically populated during login @@ -358,8 +407,9 @@ public void importRemoteGroupOfUsers(String authOrigin, String remoteGroup, @Nul logger.info("Fetching users from authentication origin '{}'", authOrigin); // Register the users - userList = authenticationManagerMap.get(authOrigin).getUsersFromRemoteGroup(remoteGroup); + userList = authenticationFactory.getUsersFromRemoteGroup(organizationId, authOrigin, remoteGroup); for (User user : userList) { + user.setOrganization(organizationId); try { create(user, null, token); logger.info("User '{}' successfully created", user.getId()); @@ -389,8 +439,8 @@ public void importRemoteGroupOfUsers(String authOrigin, String remoteGroup, @Nul .setSyncedFrom(groupSync); catalogManager.getStudyManager().createGroup(study, group, token); logger.info("Group '{}' created and synchronised with external group", internalGroup); - auditManager.audit(userId, Enums.Action.IMPORT_EXTERNAL_GROUP_OF_USERS, Enums.Resource.USER, group.getId(), - "", study, "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.audit(organizationId, userId, Enums.Action.IMPORT_EXTERNAL_GROUP_OF_USERS, Enums.Resource.USER, + group.getId(), "", study, "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { logger.error("Could not register group '{}' in study '{}'\n{}", internalGroup, study, e.getMessage(), e); throw new CatalogException("Could not register group '" + internalGroup + "' in study '" + study + "': " @@ -398,7 +448,7 @@ public void importRemoteGroupOfUsers(String authOrigin, String remoteGroup, @Nul } } } catch (CatalogException e) { - auditManager.audit(userId, Enums.Action.IMPORT_EXTERNAL_GROUP_OF_USERS, Enums.Resource.USER, "", "", "", "", + auditManager.audit(organizationId, userId, Enums.Action.IMPORT_EXTERNAL_GROUP_OF_USERS, Enums.Resource.USER, "", "", "", "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -407,17 +457,19 @@ public void importRemoteGroupOfUsers(String authOrigin, String remoteGroup, @Nul /** * Register all the ids. If internalGroup and study are not null, it will also associate the users to the internalGroup defined. * - * @param authOrigin Authentication origin. - * @param idList List of entity ids existing in the authentication origin. - * @param isApplication boolean indicating whether the id list belong to external applications or users. - * @param internalGroup Group name in Catalog that will be associated to the remote group. - * @param study Study where the internal group will be associated. - * @param token JWT token. The token should belong to the root user. + * @param organizationId Organization id. + * @param authOrigin Authentication origin. + * @param idList List of entity ids existing in the authentication origin. + * @param isApplication boolean indicating whether the id list belong to external applications or users. + * @param internalGroup Group name in Catalog that will be associated to the remote group. + * @param study Study where the internal group will be associated. + * @param token JWT token. The token should belong to the root user. * @throws CatalogException If any of the parameters is wrong or there is any internal error. */ - public void importRemoteEntities(String authOrigin, List idList, boolean isApplication, @Nullable String internalGroup, - @Nullable String study, String token) throws CatalogException { + public void importRemoteEntities(String organizationId, String authOrigin, List idList, boolean isApplication, + @Nullable String internalGroup, @Nullable String study, String token) throws CatalogException { ObjectMap auditParams = new ObjectMap() + .append("organizationId", organizationId) .append("authOrigin", authOrigin) .append("idList", idList) .append("isApplication", isApplication) @@ -425,7 +477,8 @@ public void importRemoteEntities(String authOrigin, List idList, boolean .append("study", study) .append("token", token); - String userId = getUserId(token); + JwtPayload payload = validateToken(token); + String userId = payload.getUserId(organizationId); try { if (!OPENCGA.equals(userId)) { @@ -435,28 +488,27 @@ public void importRemoteEntities(String authOrigin, List idList, boolean ParamUtils.checkParameter(authOrigin, "Authentication origin"); ParamUtils.checkObj(idList, "ids"); - if (!authenticationManagerMap.containsKey(authOrigin)) { - throw new CatalogException("Unknown authentication origin"); - } - if (!isApplication) { logger.info("Fetching user information from authentication origin '{}'", authOrigin); - List parsedUserList = authenticationManagerMap.get(authOrigin).getRemoteUserInformation(idList); + List parsedUserList = authenticationFactory.getRemoteUserInformation(organizationId, authOrigin, idList); for (User user : parsedUserList) { + user.setOrganization(organizationId); create(user, null, token); - auditManager.audit(userId, Enums.Action.IMPORT_EXTERNAL_USERS, Enums.Resource.USER, user.getId(), "", "", - "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.audit(organizationId, userId, Enums.Action.IMPORT_EXTERNAL_USERS, Enums.Resource.USER, user.getId(), "", + "", "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); logger.info("User '{}' successfully created", user.getId()); } } else { for (String applicationId : idList) { - User application = new User(applicationId, new Account() - .setType(Account.AccountType.GUEST) - .setAuthentication(new Account.AuthenticationOrigin(authOrigin, true))) + Account account = new Account() + .setAuthentication(new Account.AuthenticationOrigin(authOrigin, true)); + User application = new User(applicationId) + .setInternal(new UserInternal(new UserStatus(UserStatus.READY), account)) .setEmail("mail@mail.co.uk"); + application.setOrganization(organizationId); create(application, null, token); - auditManager.audit(userId, Enums.Action.IMPORT_EXTERNAL_USERS, Enums.Resource.USER, application.getId(), "", - "", "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.audit(organizationId, userId, Enums.Action.IMPORT_EXTERNAL_USERS, Enums.Resource.USER, application.getId(), + "", "", "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); logger.info("User (application) '{}' successfully created", application.getId()); } } @@ -485,7 +537,7 @@ public void importRemoteEntities(String authOrigin, List idList, boolean } } } catch (CatalogException e) { - auditManager.audit(userId, Enums.Action.IMPORT_EXTERNAL_USERS, Enums.Resource.USER, "", "", "", "", auditParams, + auditManager.audit(organizationId, userId, Enums.Action.IMPORT_EXTERNAL_USERS, Enums.Resource.USER, "", "", "", "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -494,14 +546,15 @@ public void importRemoteEntities(String authOrigin, List idList, boolean /** * Gets the user information. * - * @param userId User id - * @param options QueryOptions - * @param token SessionId of the user performing this operation. + * @param organizationId Organization id. + * @param userId User id + * @param options QueryOptions + * @param token SessionId of the user performing this operation. * @return The requested user * @throws CatalogException CatalogException */ - public OpenCGAResult get(String userId, QueryOptions options, String token) throws CatalogException { - return get(Collections.singletonList(userId), options, token); + public OpenCGAResult get(String organizationId, String userId, QueryOptions options, String token) throws CatalogException { + return get(organizationId, Collections.singletonList(userId), options, token); // ParamUtils.checkParameter(userId, "userId"); // ParamUtils.checkParameter(token, "sessionId"); // options = ParamUtils.defaultObject(options, QueryOptions::new); @@ -512,7 +565,7 @@ public OpenCGAResult get(String userId, QueryOptions options, String token // .append("token", token); // try { // userId = getCatalogUserId(userId, token); -// OpenCGAResult userDataResult = userDBAdaptor.get(userId, options); +// OpenCGAResult userDataResult = getUserDBAdaptor(organizationId).get(userId, options); // // // Remove some unnecessary and prohibited parameters // for (User user : userDataResult.getResults()) { @@ -540,134 +593,70 @@ public OpenCGAResult get(String userId, QueryOptions options, String token /** * Gets the user information. * - * @param userIdList List of user id - * @param options QueryOptions - * @param token Token belonging to the user itself or administrator of any study shared with the user list requested. + * @param organizationId Organization id. + * @param userIdList List of user id + * @param options QueryOptions + * @param token Token belonging to the user itself or administrator of any study shared with the user list requested. * @return The requested users * @throws CatalogException CatalogException */ - public OpenCGAResult get(List userIdList, QueryOptions options, String token) + public OpenCGAResult get(@Nullable String organizationId, List userIdList, QueryOptions options, String token) throws CatalogException { ParamUtils.checkNotEmptyArray(userIdList, "userId"); ParamUtils.checkParameter(token, "token"); options = ParamUtils.defaultObject(options, QueryOptions::new); ObjectMap auditParams = new ObjectMap() + .append("organizationId", organizationId) .append("userIdList", userIdList) .append("options", options) .append("token", token); - String userId = getUserId(token); + JwtPayload jwtPayload = validateToken(token); + + if (StringUtils.isEmpty(organizationId)) { + organizationId = jwtPayload.getOrganization(); + } + String userId = jwtPayload.getUserId(organizationId); String operationUuid = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); auditManager.initAuditBatch(operationUuid); try { - OpenCGAResult userDataResult; - - if (userIdList.size() == 1 && userId.equals(userIdList.get(0))) { - userDataResult = userDBAdaptor.get(userId, options); - auditManager.auditInfo(operationUuid, userId, Enums.Resource.USER, userId, "", "", "", auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); - return userDataResult; - } - // We will obtain the users this user is administrating - QueryOptions adminOptions = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList( - UserDBAdaptor.QueryParams.PROJECTS.key() + "." + ProjectDBAdaptor.QueryParams.STUDIES.key() + "." - + StudyDBAdaptor.QueryParams.GROUPS.key(), UserDBAdaptor.QueryParams.SHARED_PROJECTS.key() + "." - + ProjectDBAdaptor.QueryParams.STUDIES.key() + "." + StudyDBAdaptor.QueryParams.GROUPS.key())); - userDataResult = userDBAdaptor.get(userId, adminOptions); - User admin = userDataResult.first(); - - Set users = new HashSet<>(); - boolean isAdmin = false; - - if (admin.getProjects() != null) { - for (Project project : admin.getProjects()) { - if (project.getStudies() != null) { - for (Study study : project.getStudies()) { - isAdmin = true; - for (Group group : study.getGroups()) { - if (StudyManager.MEMBERS.equals(group.getId())) { - users.addAll(group.getUserIds()); - break; - } - } - } - } - } - } - if (admin.getSharedProjects() != null) { - for (Project project : admin.getSharedProjects()) { - if (project.getStudies() != null) { - for (Study study : project.getStudies()) { - boolean isAdminInStudy = false; - Set usersInStudy = new HashSet<>(); - for (Group group : study.getGroups()) { - if (StudyManager.ADMINS.equals(group.getId()) && group.getUserIds().contains(userId)) { - isAdminInStudy = true; - } - if (StudyManager.MEMBERS.equals(group.getId())) { - usersInStudy.addAll(group.getUserIds()); - } - } - if (isAdminInStudy) { - isAdmin = true; - users.addAll(usersInStudy); - } - } - } + // 1. If the user is an opencga administrator or the organization owner or admin + // 2. Or the user is requesting its own data + // - return info + if (authorizationManager.isAtLeastOrganizationOwnerOrAdmin(organizationId, userId) + || (userIdList.size() == 1 && userId.equals(userIdList.get(0)))) { + Query query = new Query(UserDBAdaptor.QueryParams.ID.key(), userIdList); + OpenCGAResult userDataResult = getUserDBAdaptor(organizationId).get(query, options); + if (userDataResult.getNumResults() < userIdList.size()) { + Set returnedUsers = userDataResult.getResults().stream().map(User::getId).collect(Collectors.toSet()); + String missingUsers = userIdList.stream().filter(u -> !returnedUsers.contains(u)).collect(Collectors.joining(", ")); + throw new CatalogException("Some users were not found: " + missingUsers); } - } - - if (!isAdmin) { - throw new CatalogAuthorizationException("Only owners or administrators can see other user information"); - } - - // Filter only the users the userId can get information for - List auxUserList = userIdList.stream().filter(users::contains).collect(Collectors.toList()); - - Query query = new Query(UserDBAdaptor.QueryParams.ID.key(), auxUserList); - OpenCGAResult result = userDBAdaptor.get(query, options); - Map userMap = new HashMap<>(); - for (User user : result.getResults()) { - userMap.put(user.getId(), user); - } - - // Ensure order and audit - List finalUserList = new ArrayList<>(userIdList.size()); - List eventList = new ArrayList<>(userIdList.size()); - for (String tmpUserId : userIdList) { - if (userMap.containsKey(tmpUserId)) { - finalUserList.add(userMap.get(tmpUserId)); - auditManager.auditInfo(operationUuid, userId, Enums.Resource.USER, tmpUserId, "", "", "", auditParams, + for (String tmpUserId : userIdList) { + auditManager.auditInfo(organizationId, operationUuid, userId, Enums.Resource.USER, tmpUserId, "", "", "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); - } else { - finalUserList.add(new User().setId(tmpUserId)); - - String msg = "'" + userId + "' is not administrating a study of user '" + tmpUserId + "' or user does not exist."; - eventList.add(new Event(Event.Type.ERROR, msg)); - - auditManager.auditInfo(operationUuid, userId, Enums.Resource.USER, tmpUserId, "", "", "", auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(-1, tmpUserId, msg))); } + return userDataResult; + } else { + throw CatalogAuthorizationException.notOrganizationOwnerOrAdmin("retrieve user's information."); } - - result.setResults(finalUserList); - result.setEvents(eventList); - - return result; } catch (CatalogException e) { for (String tmpUserId : userIdList) { - auditManager.auditInfo(operationUuid, userId, Enums.Resource.USER, tmpUserId, "", "", "", auditParams, + auditManager.auditInfo(organizationId, operationUuid, userId, Enums.Resource.USER, tmpUserId, "", "", "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); } throw e; } finally { - auditManager.finishAuditBatch(operationUuid); + auditManager.finishAuditBatch(organizationId, operationUuid); } } - public OpenCGAResult update(String userId, ObjectMap parameters, QueryOptions options, String token) throws CatalogException { - String loggedUser = getUserId(token); + public OpenCGAResult update(String userId, ObjectMap parameters, QueryOptions options, String token) + throws CatalogException { + JwtPayload payload = validateToken(token); + String organizationId = payload.getOrganization(); + String loggedUser = payload.getUserId(); ObjectMap auditParams = new ObjectMap() .append("userId", userId) @@ -680,9 +669,9 @@ public OpenCGAResult update(String userId, ObjectMap parameters, QueryOpti ParamUtils.checkObj(parameters, "parameters"); ParamUtils.checkParameter(token, "token"); - userId = getCatalogUserId(userId, token); + userId = getValidUserId(userId, payload); for (String s : parameters.keySet()) { - if (!s.matches("name|email|organization|attributes")) { + if (!s.matches("name|email")) { throw new CatalogDBException("Parameter '" + s + "' can't be changed"); } } @@ -690,19 +679,19 @@ public OpenCGAResult update(String userId, ObjectMap parameters, QueryOpti if (parameters.containsKey("email")) { checkEmail(parameters.getString("email")); } - OpenCGAResult updateResult = userDBAdaptor.update(userId, parameters); - auditManager.auditUpdate(loggedUser, Enums.Resource.USER, userId, "", "", "", auditParams, + OpenCGAResult updateResult = getUserDBAdaptor(organizationId).update(userId, parameters); + auditManager.auditUpdate(organizationId, loggedUser, Enums.Resource.USER, userId, "", "", "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { // Fetch updated user - OpenCGAResult result = userDBAdaptor.get(userId, options); + OpenCGAResult result = getUserDBAdaptor(organizationId).get(userId, options); updateResult.setResults(result.getResults()); } return updateResult; } catch (CatalogException e) { - auditManager.auditUpdate(loggedUser, Enums.Resource.USER, userId, "", "", "", auditParams, + auditManager.auditUpdate(organizationId, loggedUser, Enums.Resource.USER, userId, "", "", "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -711,23 +700,27 @@ public OpenCGAResult update(String userId, ObjectMap parameters, QueryOpti /** * Delete entries from Catalog. * - * @param userIdList Comma separated list of ids corresponding to the objects to delete - * @param options Deleting options. - * @param token Token + * @param organizationId Organization id. + * @param userIdList Comma separated list of ids corresponding to the objects to delete + * @param options Deleting options. + * @param token Token * @return A list with the deleted objects * @throws CatalogException CatalogException. */ - public OpenCGAResult delete(String userIdList, QueryOptions options, String token) throws CatalogException { + public OpenCGAResult delete(String organizationId, String userIdList, QueryOptions options, String token) + throws CatalogException { ParamUtils.checkParameter(userIdList, "userIdList"); ParamUtils.checkParameter(token, "token"); String operationUuid = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); ObjectMap auditParams = new ObjectMap() + .append("organizationId", organizationId) .append("userIdList", userIdList) .append("options", options) .append("token", token); - String tokenUser = getUserId(token); + JwtPayload payload = validateToken(token); + String tokenUser = payload.getUserId(organizationId); List userIds = Arrays.asList(userIdList.split(",")); OpenCGAResult deletedUsers = OpenCGAResult.empty(); @@ -735,20 +728,20 @@ public OpenCGAResult delete(String userIdList, QueryOptions options, Strin // Only if the user asking the deletion is the ADMINISTRATOR or the user to be deleted itself... if (OPENCGA.equals(tokenUser) || userId.equals(tokenUser)) { try { - OpenCGAResult result = userDBAdaptor.delete(userId, options); + OpenCGAResult result = getUserDBAdaptor(organizationId).delete(userId, options); - auditManager.auditDelete(operationUuid, tokenUser, Enums.Resource.USER, userId, "", "", "", auditParams, + auditManager.auditDelete(organizationId, operationUuid, tokenUser, Enums.Resource.USER, userId, "", "", "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); Query query = new Query() .append(UserDBAdaptor.QueryParams.ID.key(), userId) .append(UserDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), UserStatus.DELETED); - OpenCGAResult deletedUser = userDBAdaptor.get(query, QueryOptions.empty()); + OpenCGAResult deletedUser = getUserDBAdaptor(organizationId).get(query, QueryOptions.empty()); deletedUser.setTime(deletedUser.getTime() + result.getTime()); deletedUsers.append(deletedUser); } catch (CatalogException e) { - auditManager.auditDelete(operationUuid, tokenUser, Enums.Resource.USER, userId, "", "", "", auditParams, + auditManager.auditDelete(organizationId, operationUuid, tokenUser, Enums.Resource.USER, userId, "", "", "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); } } @@ -759,19 +752,21 @@ public OpenCGAResult delete(String userIdList, QueryOptions options, Strin /** * Delete the entries satisfying the query. * - * @param query Query of the objects to be deleted. - * @param options Deleting options. - * @param sessionId sessionId. + * @param organizationId Organization id. + * @param query Query of the objects to be deleted. + * @param options Deleting options. + * @param sessionId sessionId. * @return A list with the deleted objects. * @throws CatalogException CatalogException * @throws IOException IOException. */ - public OpenCGAResult delete(Query query, QueryOptions options, String sessionId) throws CatalogException, IOException { + public OpenCGAResult delete(String organizationId, Query query, QueryOptions options, String sessionId) + throws CatalogException, IOException { QueryOptions queryOptions = new QueryOptions(QueryOptions.INCLUDE, UserDBAdaptor.QueryParams.ID.key()); - OpenCGAResult userDataResult = userDBAdaptor.get(query, queryOptions); + OpenCGAResult userDataResult = getUserDBAdaptor(organizationId).get(query, queryOptions); List userIds = userDataResult.getResults().stream().map(User::getId).collect(Collectors.toList()); String userIdStr = StringUtils.join(userIds, ","); - return delete(userIdStr, options, sessionId); + return delete(organizationId, userIdStr, options, sessionId); } public OpenCGAResult restore(String ids, QueryOptions options, String sessionId) throws CatalogException { @@ -781,53 +776,125 @@ public OpenCGAResult restore(String ids, QueryOptions options, String sess public OpenCGAResult resetPassword(String userId, String token) throws CatalogException { ParamUtils.checkParameter(userId, "userId"); ParamUtils.checkParameter(token, "token"); + JwtPayload jwtPayload = validateToken(token); + String organizationId = jwtPayload.getOrganization(); try { - String authenticatedUserId = getUserId(token); - authorizationManager.checkIsInstallationAdministrator(authenticatedUserId); - String authOrigin = getAuthenticationOriginId(userId); - OpenCGAResult writeResult = authenticationManagerMap.get(authOrigin).resetPassword(userId); + authorizationManager.checkIsAtLeastOrganizationOwnerOrAdmin(organizationId, jwtPayload.getUserId()); + String authOrigin = getAuthenticationOriginId(organizationId, userId); + OpenCGAResult writeResult = authenticationFactory.resetPassword(organizationId, authOrigin, userId); - auditManager.auditUser(userId, Enums.Action.RESET_USER_PASSWORD, userId, + auditManager.auditUser(organizationId, jwtPayload.getUserId(organizationId), Enums.Action.RESET_USER_PASSWORD, userId, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return writeResult; } catch (CatalogException e) { - auditManager.auditUser(userId, Enums.Action.RESET_USER_PASSWORD, userId, + auditManager.auditUser(organizationId, jwtPayload.getUserId(organizationId), Enums.Action.RESET_USER_PASSWORD, userId, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } public AuthenticationResponse loginAsAdmin(String password) throws CatalogException { - return login(OPENCGA, password); + return login(ParamConstants.ADMIN_ORGANIZATION, OPENCGA, password); } - public AuthenticationResponse login(String username, String password) throws CatalogException { + public AuthenticationResponse login(String organizationId, String username, String password) throws CatalogException { ParamUtils.checkParameter(username, "userId"); ParamUtils.checkParameter(password, "password"); String authId = null; AuthenticationResponse response = null; - OpenCGAResult userOpenCGAResult = userDBAdaptor.get(username, INCLUDE_ACCOUNT); + if (StringUtils.isEmpty(organizationId)) { + // Try to automatically set the organization id + if (OPENCGA.equals(username)) { + organizationId = ParamConstants.ADMIN_ORGANIZATION; + } else { + List organizationIds = getCatalogDBAdaptorFactory().getOrganizationIds(); + if (organizationIds.size() == 2) { + organizationId = organizationIds.stream().filter(s -> !ParamConstants.ADMIN_ORGANIZATION.equals(s)).findFirst().get(); + } else { + throw CatalogParameterException.isNull("organization"); + } + } + } + + OpenCGAResult userOpenCGAResult = getUserDBAdaptor(organizationId).get(username, INCLUDE_INTERNAL); if (userOpenCGAResult.getNumResults() == 1) { - authId = userOpenCGAResult.first().getAccount().getAuthentication().getId(); - if (!authenticationManagerMap.containsKey(authId)) { - throw new CatalogException("Could not authenticate user '" + username + "'. The authentication origin '" + authId - + "' could not be found."); + User user = userOpenCGAResult.first(); + // Only local OPENCGA users that are not superadmins can be automatically banned or their accounts be expired + boolean userCanBeBanned = !ParamConstants.ADMIN_ORGANIZATION.equals(organizationId) + && CatalogAuthenticationManager.OPENCGA.equals(user.getInternal().getAccount().getAuthentication().getId()); + // We check + if (userCanBeBanned) { + // Check user is not banned, suspended or has an expired account + if (UserStatus.BANNED.equals(user.getInternal().getStatus().getId())) { + throw CatalogAuthenticationException.userIsBanned(username); + } + if (UserStatus.SUSPENDED.equals(user.getInternal().getStatus().getId())) { + throw CatalogAuthenticationException.userIsSuspended(username); + } + Account account2 = user.getInternal().getAccount(); + if (account2.getPassword().getExpirationDate() != null) { + Account account1 = user.getInternal().getAccount(); + Date passwordExpirationDate = TimeUtils.toDate(account1.getPassword().getExpirationDate()); + if (passwordExpirationDate == null) { + throw new CatalogException("Unexpected null 'passwordExpirationDate' for user '" + username + "'."); + } + if (passwordExpirationDate.before(new Date())) { + Account account = user.getInternal().getAccount(); + throw CatalogAuthenticationException.passwordExpired(username, account.getPassword().getExpirationDate()); + } + } + if (user.getInternal().getAccount().getExpirationDate() != null) { + Date date = TimeUtils.toDate(user.getInternal().getAccount().getExpirationDate()); + if (date == null) { + throw new CatalogException("Unexpected null 'expirationDate' for user '" + username + "'."); + } + if (date.before(new Date())) { + throw CatalogAuthenticationException.accountIsExpired(username, + user.getInternal().getAccount().getExpirationDate()); + } + } } + User user1 = userOpenCGAResult.first(); + authId = user1.getInternal().getAccount().getAuthentication().getId(); try { - response = authenticationManagerMap.get(authId).authenticate(username, password); + response = authenticationFactory.authenticate(organizationId, authId, username, password); } catch (CatalogAuthenticationException e) { - auditManager.auditUser(username, Enums.Action.LOGIN, username, + if (userCanBeBanned) { + // We can only lock the account if it is not the root user + UserInternal userInternal = userOpenCGAResult.first().getInternal(); + int failedAttempts = userInternal.getAccount().getFailedAttempts(); + ObjectMap updateParams = new ObjectMap(UserDBAdaptor.QueryParams.INTERNAL_ACCOUNT_FAILED_ATTEMPTS.key(), + failedAttempts + 1); + if (failedAttempts >= (configuration.getAccount().getMaxLoginAttempts() - 1)) { + // Ban the account + updateParams.append(UserDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), UserStatus.BANNED); + } + getUserDBAdaptor(organizationId).update(username, updateParams); + } + + auditManager.auditUser(organizationId, username, Enums.Action.LOGIN, username, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } + + // If it was a local user and the counter of failed attempts was greater than 0, we reset it + if (userCanBeBanned) { + UserInternal userInternal = userOpenCGAResult.first().getInternal(); + if (userInternal.getAccount().getFailedAttempts() > 0) { + // Reset login failed attempts counter + ObjectMap updateParams = new ObjectMap(UserDBAdaptor.QueryParams.INTERNAL_ACCOUNT_FAILED_ATTEMPTS.key(), 0); + getUserDBAdaptor(organizationId).update(username, updateParams); + } + } } else { // We attempt to login the user with the different authentication managers - for (Map.Entry entry : authenticationManagerMap.entrySet()) { + for (Map.Entry entry + : authenticationFactory.getOrganizationAuthenticationManagers(organizationId).entrySet()) { AuthenticationManager authenticationManager = entry.getValue(); try { - response = authenticationManager.authenticate(username, password); + response = authenticationManager.authenticate(organizationId, username, password); authId = entry.getKey(); break; } catch (CatalogAuthenticationException e) { @@ -837,31 +904,34 @@ public AuthenticationResponse login(String username, String password) throws Cat } if (response == null) { - auditManager.auditUser(username, Enums.Action.LOGIN, username, + auditManager.auditUser(organizationId, username, Enums.Action.LOGIN, username, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(0, "", "Incorrect user or password."))); throw CatalogAuthenticationException.incorrectUserOrPassword(); } - auditManager.auditUser(username, Enums.Action.LOGIN, username, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); - String userId = authenticationManagerMap.get(authId).getUserId(response.getToken()); - if (!INTERNAL_AUTHORIZATION.equals(authId)) { + auditManager.auditUser(organizationId, username, Enums.Action.LOGIN, username, + new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + String userId = authenticationFactory.getUserId(organizationId, authId, response.getToken()); + if (!CatalogAuthenticationManager.OPENCGA.equals(authId) && !CatalogAuthenticationManager.INTERNAL.equals(authId)) { // External authorization try { // If the user is not registered, an exception will be raised - userDBAdaptor.checkId(userId); + getUserDBAdaptor(organizationId).checkId(userId); } catch (CatalogDBException e) { // The user does not exist so we register it - User user = authenticationManagerMap.get(authId).getRemoteUserInformation(Collections.singletonList(userId)).get(0); + User user = authenticationFactory.getRemoteUserInformation(organizationId, authId, Collections.singletonList(userId)) + .get(0); + user.setOrganization(organizationId); // Generate a root token to be able to create the user even if the installation is private - String rootToken = authenticationManagerMap.get(INTERNAL_AUTHORIZATION).createToken(OPENCGA); + String rootToken = authenticationFactory.createToken(organizationId, CatalogAuthenticationManager.OPENCGA, OPENCGA); create(user, null, rootToken); } try { - List remoteGroups = authenticationManagerMap.get(authId).getRemoteGroups(response.getToken()); + List remoteGroups = authenticationFactory.getRemoteGroups(organizationId, authId, response.getToken()); // Resync synced groups of user in OpenCGA - studyDBAdaptor.resyncUserWithSyncedGroups(userId, remoteGroups, authId); + getStudyDBAdaptor(organizationId).resyncUserWithSyncedGroups(userId, remoteGroups, authId); } catch (CatalogException e) { logger.error("Could not update synced groups for user '" + userId + "'\n" + e.getMessage(), e); } @@ -870,19 +940,37 @@ public AuthenticationResponse login(String username, String password) throws Cat return response; } + public AuthenticationResponse loginAnonymous(String organizationId) throws CatalogException { + ParamUtils.checkParameter(organizationId, "organization id"); + + // Check user anonymous has access to any study within the organization + Query query = new Query(StudyDBAdaptor.QueryParams.GROUP_USER_IDS.key(), ParamConstants.ANONYMOUS_USER_ID); + OpenCGAResult count = getStudyDBAdaptor(organizationId).count(query); + if (count.getNumMatches() == 0) { + throw CatalogAuthenticationException.userNotFound(organizationId, ParamConstants.ANONYMOUS_USER_ID); + } + + String token = authenticationFactory.createToken(organizationId, CatalogAuthenticationManager.OPENCGA, + ParamConstants.ANONYMOUS_USER_ID); + return new AuthenticationResponse(token); + } + /** * Create a new token if the token provided corresponds to the user and it is not expired yet. * - * @param token active token. + * @param token active token. * @return a new AuthenticationResponse object. * @throws CatalogException if the token does not correspond to the user or the token is expired. */ public AuthenticationResponse refreshToken(String token) throws CatalogException { + JwtPayload payload = new JwtPayload(token); + String organizationId = payload.getOrganization(); AuthenticationResponse response = null; CatalogAuthenticationException exception = null; String userId = ""; // We attempt to renew the token with the different authentication managers - for (Map.Entry entry : authenticationManagerMap.entrySet()) { + for (Map.Entry entry + : authenticationFactory.getOrganizationAuthenticationManagers(organizationId).entrySet()) { AuthenticationManager authenticationManager = entry.getValue(); try { response = authenticationManager.refreshToken(token); @@ -890,71 +978,137 @@ public AuthenticationResponse refreshToken(String token) throws CatalogException break; } catch (CatalogAuthenticationException e) { logger.debug("Could not refresh token with '{}' provider: {}", entry.getKey(), e.getMessage(), e); - if (INTERNAL_AUTHORIZATION.equals(entry.getKey())) { + if (CatalogAuthenticationManager.OPENCGA.equals(entry.getKey())) { exception = e; } } } if (response == null && exception != null) { - auditManager.auditUser(userId, Enums.Action.REFRESH_TOKEN, userId, + auditManager.auditUser(organizationId, userId, Enums.Action.REFRESH_TOKEN, userId, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, exception.getError())); throw exception; } - auditManager.auditUser(userId, Enums.Action.REFRESH_TOKEN, userId, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.auditUser(organizationId, userId, Enums.Action.REFRESH_TOKEN, userId, + new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return response; } + public OpenCGAResult changeStatus(String organizationId, String userId, String status, QueryOptions options, String token) + throws CatalogException { + JwtPayload tokenPayload = validateToken(token); + String userIdOrganization = StringUtils.isNotEmpty(organizationId) ? organizationId : tokenPayload.getOrganization(); + + ObjectMap auditParams = new ObjectMap() + .append("organizationId", organizationId) + .append("userId", userId) + .append("status", status) + .append("options", options) + .append("token", token); + try { + authorizationManager.checkIsAtLeastOrganizationOwnerOrAdmin(userIdOrganization, tokenPayload.getUserId(userIdOrganization)); + options = ParamUtils.defaultObject(options, QueryOptions::new); + + // Validate user exists + getUserDBAdaptor(userIdOrganization).checkId(userId); + + // Validate status is valid + if (!UserStatus.READY.equals(status) && !UserStatus.SUSPENDED.equals(status)) { + throw new CatalogParameterException("Invalid status '" + status + "'. Valid values are: " + UserStatus.READY + ", " + + UserStatus.SUSPENDED); + } + + if (UserStatus.SUSPENDED.equals(status)) { + // Get organization information + Set ownerAndAdmins = catalogManager.getOrganizationManager().getOrganizationOwnerAndAdmins(userIdOrganization); + if (ownerAndAdmins.contains(userId)) { + if (tokenPayload.getUserId().equals(userId)) { + // The user is trying to suspend himself + throw new CatalogAuthorizationException("You can't suspend your own account."); + } + if (!authorizationManager.isAtLeastOrganizationOwner(userIdOrganization, tokenPayload.getUserId(userIdOrganization))) { + // One of the admins is trying to suspend the owner or one of the admins + throw new CatalogAuthorizationException("Only the owner of the organization can suspend administrators."); + } + } + } + + // Update user status and reset failed attempts to 0 + ObjectMap updateParams = new ObjectMap(UserDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), status); + if (UserStatus.READY.equals(status)) { + updateParams.put(UserDBAdaptor.QueryParams.INTERNAL_ACCOUNT_FAILED_ATTEMPTS.key(), 0); + } + OpenCGAResult result = getUserDBAdaptor(userIdOrganization).update(userId, updateParams); + + auditManager.auditUpdate(organizationId, tokenPayload.getUserId(userIdOrganization), Enums.Resource.USER, userId, "", "", "", + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + + if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { + // Fetch updated user + OpenCGAResult tmpResult = getUserDBAdaptor(userIdOrganization).get(userId, options); + result.setResults(tmpResult.getResults()); + } + + return result; + } catch (Exception e) { + auditManager.auditUpdate(organizationId, tokenPayload.getUserId(userIdOrganization), Enums.Resource.USER, userId, "", "", "", + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(-1, "Could not update user status", + e.getMessage()))); + throw e; + } + } + /** * This method will be only callable by the system. It generates a new session id for the user. * - * @param userId user id for which a session will be generated. - * @param attributes attributes to be put as part of the claims section in the JWT. - * @param expiration Expiration time in seconds. If null, default expiration time will be used. - * @param token Password or active session of the OpenCGA admin. + * @param organizationId Organization id. + * @param userId user id for which a session will be generated. + * @param attributes attributes to be put as part of the claims section in the JWT. + * @param expiration Expiration time in seconds. If null, default expiration time will be used. + * @param token Password or active session of the OpenCGA admin. * @return an objectMap containing the new sessionId * @throws CatalogException if the password is not correct or the userId does not exist. */ - public String getToken(String userId, Map attributes, Long expiration, String token) throws CatalogException { - if (!OPENCGA.equals(getUserId(token))) { - throw new CatalogException("Only user '" + OPENCGA + "' is allowed to create tokens"); - } + public String getToken(String organizationId, String userId, Map attributes, Long expiration, String token) + throws CatalogException { + JwtPayload jwtPayload = validateToken(token); + authorizationManager.checkIsOpencgaAdministrator(jwtPayload, "create tokens"); + if (expiration != null && expiration <= 0) { throw new CatalogException("Expiration time must be higher than 0"); } - AuthenticationManager authManager = getAuthenticationManagerForUser(userId); + AuthenticationManager authManager = getAuthenticationManagerForUser(organizationId, userId); return expiration != null - ? authManager.createToken(userId, attributes, expiration) - : authManager.createToken(userId, attributes); + ? authManager.createToken(organizationId, userId, attributes, expiration) + : authManager.createToken(organizationId, userId, attributes); } /** * This method will be only callable by the system. It generates a new session id for the user. * - * @param userId user id for which a session will be generated. - * @param attributes attributes to be put as part of the claims section in the JWT. - * @param token Password or active session of the OpenCGA admin. + * @param organizationId Organization id. + * @param userId user id for which a session will be generated. + * @param attributes attributes to be put as part of the claims section in the JWT. + * @param token Password or active session of the OpenCGA admin. * @return an objectMap containing the new sessionId * @throws CatalogException if the password is not correct or the userId does not exist. */ - public String getNonExpiringToken(String userId, Map attributes, String token) throws CatalogException { - if (!OPENCGA.equals(getUserId(token))) { - throw new CatalogException("Only user '" + OPENCGA + "' is allowed to create tokens"); - } - AuthenticationManager authManager = getAuthenticationManagerForUser(userId); - return authManager.createNonExpiringToken(userId, attributes); + public String getNonExpiringToken(String organizationId, String userId, Map attributes, String token) + throws CatalogException { + JwtPayload payload = validateToken(token); + authorizationManager.checkIsOpencgaAdministrator(payload, "create tokens"); + + AuthenticationManager authManager = getAuthenticationManagerForUser(organizationId, userId); + return authManager.createNonExpiringToken(organizationId, userId, attributes); } - private AuthenticationManager getAuthenticationManagerForUser(String user) throws CatalogException { - OpenCGAResult userOpenCGAResult = userDBAdaptor.get(user, INCLUDE_ACCOUNT); + private AuthenticationManager getAuthenticationManagerForUser(String organizationId, String user) throws CatalogException { + OpenCGAResult userOpenCGAResult = getUserDBAdaptor(organizationId).get(user, INCLUDE_INTERNAL); if (userOpenCGAResult.getNumResults() == 1) { - String authId = userOpenCGAResult.first().getAccount().getAuthentication().getId(); - if (!authenticationManagerMap.containsKey(authId)) { - throw new CatalogException("Could not authenticate user '" + user + "'. The authentication origin '" + authId - + "' could not be found."); - } - return authenticationManagerMap.get(authId); + User user1 = userOpenCGAResult.first(); + String authId = user1.getInternal().getAccount().getAuthentication().getId(); + return authenticationFactory.getOrganizationAuthenticationManager(organizationId, authId); } else { throw new CatalogException("User '" + user + "' not found."); } @@ -964,21 +1118,25 @@ private AuthenticationManager getAuthenticationManagerForUser(String user) throw * Add a new filter to the user account. *

* - * @param userId user id to whom the filter will be associated. - * @param id Filter id. - * @param description Filter description. - * @param resource Resource where the filter should be applied. - * @param query Query object. - * @param queryOptions Query options object. - * @param token session id of the user asking to store the filter. + * @param userId user id to whom the filter will be associated. + * @param id Filter id. + * @param description Filter description. + * @param resource Resource where the filter should be applied. + * @param query Query object. + * @param queryOptions Query options object. + * @param token session id of the user asking to store the filter. * @return the created filter. * @throws CatalogException if there already exists a filter with that same name for the user or if the user corresponding to the * session id is not the same as the provided user id. */ public OpenCGAResult addFilter(String userId, String id, String description, Enums.Resource resource, Query query, QueryOptions queryOptions, String token) throws CatalogException { + JwtPayload payload = validateToken(token); + String authenticatedUserId = payload.getUserId(); + String organizationId = payload.getOrganization(); + ParamUtils.checkParameter(userId, "userId"); - ParamUtils.checkParameter(token, "sessionId"); + ParamUtils.checkParameter(token, "token"); ParamUtils.checkParameter(id, "id"); ParamUtils.checkObj(resource, "resource"); ParamUtils.checkObj(query, "Query"); @@ -996,23 +1154,23 @@ public OpenCGAResult addFilter(String userId, String id, String desc .append("queryOptions", queryOptions) .append("token", token); try { - userId = getCatalogUserId(userId, token); - userDBAdaptor.checkId(userId); + userId = getValidUserId(userId, payload); + getUserDBAdaptor(organizationId).checkId(userId); Query queryExists = new Query() .append(UserDBAdaptor.QueryParams.ID.key(), userId) .append(UserDBAdaptor.QueryParams.FILTERS_ID.key(), id); - if (userDBAdaptor.count(queryExists).getNumMatches() > 0) { + if (getUserDBAdaptor(organizationId).count(queryExists).getNumMatches() > 0) { throw new CatalogException("There already exists a filter called " + id + " for user " + userId); } UserFilter filter = new UserFilter(id, description, resource, query, queryOptions); - OpenCGAResult result = userDBAdaptor.addFilter(userId, filter); - auditManager.auditUser(userId, Enums.Action.CHANGE_USER_CONFIG, userId, auditParams, + OpenCGAResult result = getUserDBAdaptor(organizationId).addFilter(userId, filter); + auditManager.auditUser(organizationId, authenticatedUserId, Enums.Action.CHANGE_USER_CONFIG, userId, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return new OpenCGAResult<>(result.getTime(), Collections.emptyList(), 1, Collections.singletonList(filter), 1); } catch (CatalogException e) { - auditManager.auditUser(userId, Enums.Action.CHANGE_USER_CONFIG, userId, auditParams, + auditManager.auditUser(organizationId, authenticatedUserId, Enums.Action.CHANGE_USER_CONFIG, userId, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -1022,18 +1180,18 @@ public OpenCGAResult addFilter(String userId, String id, String desc * Update the filter information. *

* - * @param userId user id to whom the filter should be updated. - * @param name Filter name. - * @param params Map containing the parameters to be updated. - * @param token session id of the user asking to update the filter. + * @param userId user id to whom the filter should be updated. + * @param name Filter name. + * @param params Map containing the parameters to be updated. + * @param token session id of the user asking to update the filter. * @return the updated filter. * @throws CatalogException if the filter could not be updated because the filter name is not correct or if the user corresponding to * the session id is not the same as the provided user id. */ public OpenCGAResult updateFilter(String userId, String name, ObjectMap params, String token) throws CatalogException { - ParamUtils.checkParameter(userId, "userId"); - ParamUtils.checkParameter(token, "token"); - ParamUtils.checkParameter(name, "name"); + JwtPayload payload = validateToken(token); + String authenticatedUserId = payload.getUserId(); + String organizationId = payload.getOrganization(); ObjectMap auditParams = new ObjectMap() .append("userId", userId) @@ -1041,26 +1199,30 @@ public OpenCGAResult updateFilter(String userId, String name, Object .append("params", params) .append("token", token); try { - userId = getCatalogUserId(userId, token); - userDBAdaptor.checkId(userId); + ParamUtils.checkParameter(userId, "userId"); + ParamUtils.checkParameter(token, "token"); + ParamUtils.checkParameter(name, "name"); + + userId = getValidUserId(userId, payload); + getUserDBAdaptor(organizationId).checkId(userId); Query queryExists = new Query() .append(UserDBAdaptor.QueryParams.ID.key(), userId) .append(UserDBAdaptor.QueryParams.FILTERS_ID.key(), name); - if (userDBAdaptor.count(queryExists).getNumMatches() == 0) { + if (getUserDBAdaptor(organizationId).count(queryExists).getNumMatches() == 0) { throw new CatalogException("There is no filter called " + name + " for user " + userId); } - OpenCGAResult result = userDBAdaptor.updateFilter(userId, name, params); - UserFilter filter = getFilter(userId, name); + OpenCGAResult result = getUserDBAdaptor(organizationId).updateFilter(userId, name, params); + UserFilter filter = getFilter(getUserDBAdaptor(organizationId), userId, name); if (filter == null) { throw new CatalogException("Internal error: The filter " + name + " could not be found."); } - auditManager.auditUser(userId, Enums.Action.CHANGE_USER_CONFIG, userId, auditParams, + auditManager.auditUser(organizationId, authenticatedUserId, Enums.Action.CHANGE_USER_CONFIG, userId, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return new OpenCGAResult<>(result.getTime(), Collections.emptyList(), 1, Collections.singletonList(filter), 1); } catch (CatalogException e) { - auditManager.auditUser(userId, Enums.Action.CHANGE_USER_CONFIG, userId, auditParams, + auditManager.auditUser(organizationId, authenticatedUserId, Enums.Action.CHANGE_USER_CONFIG, userId, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -1078,29 +1240,33 @@ public OpenCGAResult updateFilter(String userId, String name, Object * is not the same as the provided user id. */ public OpenCGAResult deleteFilter(String userId, String name, String token) throws CatalogException { - ParamUtils.checkParameter(userId, "userId"); - ParamUtils.checkParameter(token, "token"); - ParamUtils.checkParameter(name, "name"); + JwtPayload payload = validateToken(token); + String authenticatedUserId = payload.getUserId(); + String organizationId = payload.getOrganization(); ObjectMap auditParams = new ObjectMap() .append("userId", userId) .append("name", name) .append("token", token); try { - userId = getCatalogUserId(userId, token); - userDBAdaptor.checkId(userId); + ParamUtils.checkParameter(userId, "userId"); + ParamUtils.checkParameter(token, "token"); + ParamUtils.checkParameter(name, "name"); + + userId = getValidUserId(userId, payload); + getUserDBAdaptor(organizationId).checkId(userId); - UserFilter filter = getFilter(userId, name); + UserFilter filter = getFilter(getUserDBAdaptor(organizationId), userId, name); if (filter == null) { throw new CatalogException("There is no filter called " + name + " for user " + userId); } - OpenCGAResult result = userDBAdaptor.deleteFilter(userId, name); - auditManager.auditUser(userId, Enums.Action.CHANGE_USER_CONFIG, userId, auditParams, + OpenCGAResult result = getUserDBAdaptor(organizationId).deleteFilter(userId, name); + auditManager.auditUser(organizationId, authenticatedUserId, Enums.Action.CHANGE_USER_CONFIG, userId, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return new OpenCGAResult<>(result.getTime(), Collections.emptyList(), 1, Collections.singletonList(filter), 1); } catch (CatalogException e) { - auditManager.auditUser(userId, Enums.Action.CHANGE_USER_CONFIG, userId, auditParams, + auditManager.auditUser(organizationId, authenticatedUserId, Enums.Action.CHANGE_USER_CONFIG, userId, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -1117,20 +1283,22 @@ public OpenCGAResult deleteFilter(String userId, String name, String * @throws CatalogException if the user corresponding to the session id is not the same as the provided user id. */ public OpenCGAResult getFilter(String userId, String name, String token) throws CatalogException { - ParamUtils.checkParameter(userId, "userId"); - ParamUtils.checkParameter(token, "sessionId"); - ParamUtils.checkParameter(name, "name"); + JwtPayload payload = validateToken(token); + String authenticatedUserId = payload.getUserId(); + String organizationId = payload.getOrganization(); ObjectMap auditParams = new ObjectMap() .append("userId", userId) .append("name", name) .append("token", token); try { - userId = getCatalogUserId(userId, token); - userDBAdaptor.checkId(userId); + ParamUtils.checkParameter(userId, "userId"); + ParamUtils.checkParameter(name, "name"); + + userId = getValidUserId(userId, payload); - UserFilter filter = getFilter(userId, name); - auditManager.auditUser(userId, Enums.Action.FETCH_USER_CONFIG, userId, auditParams, + UserFilter filter = getFilter(getUserDBAdaptor(organizationId), userId, name); + auditManager.auditUser(organizationId, authenticatedUserId, Enums.Action.FETCH_USER_CONFIG, userId, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); if (filter == null) { @@ -1139,12 +1307,31 @@ public OpenCGAResult getFilter(String userId, String name, String to return new OpenCGAResult<>(0, Collections.emptyList(), 1, Collections.singletonList(filter), 1); } } catch (CatalogException e) { - auditManager.auditUser(userId, Enums.Action.FETCH_USER_CONFIG, userId, auditParams, + auditManager.auditUser(organizationId, authenticatedUserId, Enums.Action.FETCH_USER_CONFIG, userId, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } + private UserFilter getFilter(UserDBAdaptor userDBAdaptor, String userId, String name) throws CatalogException { + Query query = new Query() + .append(UserDBAdaptor.QueryParams.ID.key(), userId); + QueryOptions queryOptions = new QueryOptions(QueryOptions.INCLUDE, UserDBAdaptor.QueryParams.FILTERS.key()); + OpenCGAResult userDataResult = userDBAdaptor.get(query, queryOptions); + + if (userDataResult.getNumResults() != 1) { + throw new CatalogException("Internal error: User " + userId + " not found."); + } + + for (UserFilter filter : userDataResult.first().getFilters()) { + if (name.equals(filter.getId())) { + return filter; + } + } + + return null; + } + /** * Retrieves all the user filters. * @@ -1154,32 +1341,35 @@ public OpenCGAResult getFilter(String userId, String name, String to * @throws CatalogException if the user corresponding to the session id is not the same as the provided user id. */ public OpenCGAResult getAllFilters(String userId, String token) throws CatalogException { - ParamUtils.checkParameter(userId, "userId"); - ParamUtils.checkParameter(token, "sessionId"); + JwtPayload payload = validateToken(token); + String authenticatedUserId = payload.getUserId(); + String organizationId = payload.getOrganization(); ObjectMap auditParams = new ObjectMap() .append("userId", userId) .append("token", token); try { - userId = getCatalogUserId(userId, token); - userDBAdaptor.checkId(userId); + ParamUtils.checkParameter(userId, "userId"); + + userId = getValidUserId(userId, payload); + getUserDBAdaptor(organizationId).checkId(userId); Query query = new Query() .append(UserDBAdaptor.QueryParams.ID.key(), userId); QueryOptions queryOptions = new QueryOptions(QueryOptions.INCLUDE, UserDBAdaptor.QueryParams.FILTERS.key()); - OpenCGAResult userDataResult = userDBAdaptor.get(query, queryOptions); + OpenCGAResult userDataResult = getUserDBAdaptor(organizationId).get(query, queryOptions); if (userDataResult.getNumResults() != 1) { throw new CatalogException("Internal error: User " + userId + " not found."); } List filters = userDataResult.first().getFilters(); - auditManager.auditUser(userId, Enums.Action.FETCH_USER_CONFIG, userId, auditParams, + auditManager.auditUser(organizationId, authenticatedUserId, Enums.Action.FETCH_USER_CONFIG, userId, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return new OpenCGAResult<>(0, Collections.emptyList(), filters.size(), filters, filters.size()); } catch (CatalogException e) { - auditManager.auditUser(userId, Enums.Action.FETCH_USER_CONFIG, userId, auditParams, + auditManager.auditUser(organizationId, authenticatedUserId, Enums.Action.FETCH_USER_CONFIG, userId, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -1197,10 +1387,9 @@ public OpenCGAResult getAllFilters(String userId, String token) thro * @throws CatalogException if the user corresponding to the session id is not the same as the provided user id. */ public OpenCGAResult setConfig(String userId, String name, Map config, String token) throws CatalogException { - ParamUtils.checkParameter(userId, "userId"); - ParamUtils.checkParameter(token, "sessionId"); - ParamUtils.checkParameter(name, "name"); - ParamUtils.checkObj(config, "ObjectMap"); + JwtPayload payload = validateToken(token); + String authenticatedUserId = payload.getUserId(); + String organizationId = payload.getOrganization(); ObjectMap auditParams = new ObjectMap() .append("userId", userId) @@ -1209,16 +1398,20 @@ public OpenCGAResult setConfig(String userId, String name, Map c .append("token", token); try { - userId = getCatalogUserId(userId, token); - userDBAdaptor.checkId(userId); + ParamUtils.checkParameter(userId, "userId"); + ParamUtils.checkParameter(name, "name"); + ParamUtils.checkObj(config, "ObjectMap"); + + userId = getValidUserId(userId, payload); + getUserDBAdaptor(organizationId).checkId(userId); - OpenCGAResult result = userDBAdaptor.setConfig(userId, name, config); - auditManager.auditUser(userId, Enums.Action.CHANGE_USER_CONFIG, userId, auditParams, + OpenCGAResult result = getUserDBAdaptor(organizationId).setConfig(userId, name, config); + auditManager.auditUser(organizationId, authenticatedUserId, Enums.Action.CHANGE_USER_CONFIG, userId, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return new OpenCGAResult(result.getTime(), Collections.emptyList(), 1, Collections.singletonList(config), 1); } catch (CatalogException e) { - auditManager.auditUser(userId, Enums.Action.CHANGE_USER_CONFIG, userId, auditParams, + auditManager.auditUser(organizationId, authenticatedUserId, Enums.Action.CHANGE_USER_CONFIG, userId, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -1236,20 +1429,23 @@ public OpenCGAResult setConfig(String userId, String name, Map c * not exist. */ public OpenCGAResult deleteConfig(String userId, String name, String token) throws CatalogException { - ParamUtils.checkParameter(userId, "userId"); - ParamUtils.checkParameter(token, "token"); - ParamUtils.checkParameter(name, "name"); + JwtPayload payload = validateToken(token); + String authenticatedUserId = payload.getUserId(); + String organizationId = payload.getOrganization(); ObjectMap auditParams = new ObjectMap() .append("userId", userId) .append("name", name) .append("token", token); try { - userId = getCatalogUserId(userId, token); - userDBAdaptor.checkId(userId); + ParamUtils.checkParameter(userId, "userId"); + ParamUtils.checkParameter(name, "name"); + + userId = getValidUserId(userId, payload); + getUserDBAdaptor(organizationId).checkId(userId); QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, UserDBAdaptor.QueryParams.CONFIGS.key()); - OpenCGAResult userDataResult = userDBAdaptor.get(userId, options); + OpenCGAResult userDataResult = getUserDBAdaptor(organizationId).get(userId, options); if (userDataResult.getNumResults() == 0) { throw new CatalogException("Internal error: Could not get user " + userId); } @@ -1263,12 +1459,12 @@ public OpenCGAResult deleteConfig(String userId, String name, String token) thro throw new CatalogException("Error: Cannot delete configuration with name " + name + ". Configuration name not found."); } - OpenCGAResult result = userDBAdaptor.deleteConfig(userId, name); - auditManager.auditUser(userId, Enums.Action.CHANGE_USER_CONFIG, userId, auditParams, + OpenCGAResult result = getUserDBAdaptor(organizationId).deleteConfig(userId, name); + auditManager.auditUser(organizationId, authenticatedUserId, Enums.Action.CHANGE_USER_CONFIG, userId, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return new OpenCGAResult(result.getTime(), Collections.emptyList(), 1, Collections.singletonList(configs.get(name)), 1); } catch (CatalogException e) { - auditManager.auditUser(userId, Enums.Action.CHANGE_USER_CONFIG, userId, auditParams, + auditManager.auditUser(organizationId, authenticatedUserId, Enums.Action.CHANGE_USER_CONFIG, userId, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -1286,19 +1482,22 @@ public OpenCGAResult deleteConfig(String userId, String name, String token) thro * does not exist. */ public OpenCGAResult getConfig(String userId, String name, String token) throws CatalogException { - ParamUtils.checkParameter(userId, "userId"); - ParamUtils.checkParameter(token, "sessionId"); + JwtPayload payload = validateToken(token); + String authenticatedUserId = payload.getUserId(); + String organizationId = payload.getOrganization(); ObjectMap auditParams = new ObjectMap() .append("userId", userId) .append("name", name) .append("token", token); try { - userId = getCatalogUserId(userId, token); - userDBAdaptor.checkId(userId); + ParamUtils.checkParameter(userId, "userId"); + + userId = getValidUserId(userId, payload); + getUserDBAdaptor(organizationId).checkId(userId); QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, UserDBAdaptor.QueryParams.CONFIGS.key()); - OpenCGAResult userDataResult = userDBAdaptor.get(userId, options); + OpenCGAResult userDataResult = getUserDBAdaptor(organizationId).get(userId, options); if (userDataResult.getNumResults() == 0) { throw new CatalogException("Internal error: Could not get user " + userId); } @@ -1312,77 +1511,57 @@ public OpenCGAResult getConfig(String userId, String name, String token) throws throw new CatalogException("Error: Cannot fetch configuration with name " + name + ". Configuration name not found."); } - auditManager.auditUser(userId, Enums.Action.FETCH_USER_CONFIG, userId, auditParams, + auditManager.auditUser(organizationId, authenticatedUserId, Enums.Action.FETCH_USER_CONFIG, userId, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return new OpenCGAResult(userDataResult.getTime(), userDataResult.getEvents(), 1, Collections.singletonList(configs.get(name)), 1); } catch (CatalogException e) { - auditManager.auditUser(userId, Enums.Action.FETCH_USER_CONFIG, userId, auditParams, + auditManager.auditUser(organizationId, authenticatedUserId, Enums.Action.FETCH_USER_CONFIG, userId, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } - private UserFilter getFilter(String userId, String name) throws CatalogException { - Query query = new Query() - .append(UserDBAdaptor.QueryParams.ID.key(), userId); - QueryOptions queryOptions = new QueryOptions(QueryOptions.INCLUDE, UserDBAdaptor.QueryParams.FILTERS.key()); - OpenCGAResult userDataResult = userDBAdaptor.get(query, queryOptions); - - if (userDataResult.getNumResults() != 1) { - throw new CatalogException("Internal error: User " + userId + " not found."); - } - - for (UserFilter filter : userDataResult.first().getFilters()) { - if (name.equals(filter.getId())) { - return filter; - } - } - - return null; - } - - private void checkUserExists(String userId) throws CatalogException { - if (userId.toLowerCase().equals(ANONYMOUS)) { + private void checkUserExists(String organizationId, String userId) throws CatalogException { + if (userId.equalsIgnoreCase(ANONYMOUS)) { throw new CatalogException("Permission denied: Cannot create users with special treatments in catalog."); } - if (userDBAdaptor.exists(userId)) { + if (getUserDBAdaptor(organizationId).exists(userId)) { throw new CatalogException("The user already exists in our database."); } } - private String getAuthenticationOriginId(String userId) throws CatalogException { - OpenCGAResult user = userDBAdaptor.get(userId, new QueryOptions()); + private String getAuthenticationOriginId(String organizationId, String userId) throws CatalogException { + OpenCGAResult user = getUserDBAdaptor(organizationId).get(userId, new QueryOptions()); if (user == null || user.getNumResults() == 0) { throw new CatalogException(userId + " user not found"); } - return user.first().getAccount().getAuthentication().getId(); + User user1 = user.first(); + return user1.getInternal().getAccount().getAuthentication().getId(); } /** - * Extracts the user id from the token. If it doesn't match the userId provided and the userId provided is actually an email, it will - * fetch the user of the token from Catalog and check whether the email matches. If it matches, it will return the corresponding user - * id. + * Compares the userId provided with the one from the token payload. If it doesn't match and the userId provided is actually an email, + * it will fetch the user from the token from Catalog and check whether the email matches. If it matches, it will return the + * corresponding user id or fail otherwise. * - * @param userId User id provided by the user. - * @param token Token. + * @param userId User id provided by the user. + * @param payload Token payload. * @return A valid user id for the user and token provided. * @throws CatalogException if the user cannot be retrieved for whatever reason. */ - private String getCatalogUserId(String userId, String token) throws CatalogException { - ParamUtils.checkParameter(userId, "User id"); - - String userFromToken = getUserId(token); - - if (userFromToken.equals(userId)) { + private String getValidUserId(String userId, JwtPayload payload) throws CatalogException { + if (payload.getUserId().equals(userId)) { return userId; } + String organizationId = payload.getOrganization(); + String userFromToken = payload.getUserId(); // User might be using the email as an id QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, UserDBAdaptor.QueryParams.EMAIL.key()); - OpenCGAResult userOpenCGAResult = userDBAdaptor.get(userFromToken, options); + OpenCGAResult userOpenCGAResult = getUserDBAdaptor(organizationId).get(userFromToken, options); if (userOpenCGAResult.getNumResults() == 0) { throw new CatalogException("User '" + userFromToken + "' not found. Please, call to login first or talk to your administrator"); } @@ -1394,38 +1573,22 @@ private String getCatalogUserId(String userId, String token) throws CatalogExcep } } - /** - * Get the userId from the sessionId. - * - * @param token Token - * @return UserId owner of the sessionId. Empty string if SessionId does not match. - * @throws CatalogException when the session id does not correspond to any user or the token has expired. - */ public String getUserId(String token) throws CatalogException { - for (Map.Entry entry : authenticationManagerMap.entrySet()) { - AuthenticationManager authenticationManager = entry.getValue(); - try { - String userId = authenticationManager.getUserId(token); - userDBAdaptor.checkId(userId); - return userId; - } catch (Exception e) { - logger.debug("Could not get user from token using {} authentication manager. {}", entry.getKey(), e.getMessage(), e); - } - } - // We make this call again to get the original exception - return authenticationManagerMap.get(INTERNAL_AUTHORIZATION).getUserId(token); + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + return tokenPayload.getUserId(); } - public Date getExpirationDate(String token) { - for (Map.Entry entry : authenticationManagerMap.entrySet()) { - AuthenticationManager authenticationManager = entry.getValue(); - try { - return authenticationManager.getExpirationDate(token); - } catch (Exception e) { - logger.debug("Could not get expiration date from token using {} authentication manager. {}", entry.getKey(), - e.getMessage(), e); - } - } - return null; + public String getUserIdContextProject(String projectStr, String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn catalogFqn = CatalogFqn.extractFqnFromProject(projectStr, tokenPayload); + String organizationId = catalogFqn.getOrganizationId(); + return tokenPayload.getUserId(organizationId); + } + + public String getUserIdContextStudy(String studyStr, String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn catalogFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = catalogFqn.getOrganizationId(); + return tokenPayload.getUserId(organizationId); } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/Migration.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/Migration.java index 7fb3b1dabf8..3c5ddea26d3 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/Migration.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/Migration.java @@ -54,6 +54,11 @@ */ boolean manual() default false; + /** + * @return the version when the migration was deprecated. + */ + String deprecatedSince() default ""; + enum MigrationDomain { CATALOG, STORAGE diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationException.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationException.java index 63f666eac14..000c46d57dc 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationException.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationException.java @@ -20,4 +20,10 @@ public static MigrationException offlineMigrationException(Migration migration) return new MigrationException("Migration '" + migration.id() + "' requires database to be offline. Please, ensure the database " + "cannot be accessed and run try again with '--offline' flag."); } + + public static MigrationException deprecatedMigration(Migration migration) { + return new MigrationException("Migration '" + migration.id() + "' can't be run since version '" + + migration.deprecatedSince() + "'. Please, run this migration from a previous OpenCGA version."); + } + } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationManager.java index 13f93e824b5..fa8fd7b9ee0 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationManager.java @@ -16,15 +16,15 @@ import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.utils.FileUtils; import org.opencb.opencga.catalog.db.DBAdaptorFactory; -import org.opencb.opencga.catalog.db.api.MigrationDBAdaptor; import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.managers.AbstractManager; import org.opencb.opencga.catalog.managers.CatalogManager; +import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.config.storage.StorageConfiguration; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.file.File; import org.opencb.opencga.core.models.file.FileLinkParams; @@ -32,6 +32,7 @@ import org.opencb.opencga.core.models.job.JobInternal; import org.opencb.opencga.core.models.job.JobReferenceParam; import org.opencb.opencga.core.models.job.ToolInfo; +import org.opencb.opencga.core.models.migration.MigrationRun; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.tools.annotations.Tool; import org.opencb.opencga.core.tools.result.ExecutionResult; @@ -57,8 +58,6 @@ public class MigrationManager { private final CatalogManager catalogManager; private final Configuration configuration; -// private final StorageConfiguration storageConfiguration; - private final MigrationDBAdaptor migrationDBAdaptor; private final Logger logger; private final MongoDBAdaptorFactory dbAdaptorFactory; @@ -66,23 +65,22 @@ public class MigrationManager { public MigrationManager(CatalogManager catalogManager, DBAdaptorFactory dbAdaptorFactory, Configuration configuration) { this.catalogManager = catalogManager; this.configuration = configuration; - this.migrationDBAdaptor = dbAdaptorFactory.getMigrationDBAdaptor(); this.dbAdaptorFactory = (MongoDBAdaptorFactory) dbAdaptorFactory; this.logger = LoggerFactory.getLogger(MigrationManager.class); } - public MigrationRun runManualMigration(String version, String id, Path appHome, ObjectMap params, String token) + public List runManualMigration(String version, String id, Path appHome, ObjectMap params, String token) throws CatalogException { return runManualMigration(version, id, appHome, false, false, params, token); } - public MigrationRun runManualMigration(String version, String id, Path appHome, boolean force, boolean offline, ObjectMap params, - String token) throws CatalogException { + public MigrationRun runManualMigration(String organizationId, String version, String id, Path appHome, boolean force, boolean offline, + ObjectMap params, String token) throws CatalogException { token = validateAdmin(token); for (Class c : getAvailableMigrations()) { Migration migration = getMigrationAnnotation(c); if (migration.id().equals(id) && migration.version().equals(version)) { - MigrationRun migrationRun = updateMigrationRun(migration, token); + MigrationRun migrationRun = updateMigrationRun(organizationId, migration, token); if (!offline && migration.offline()) { throw MigrationException.offlineMigrationException(migration); } @@ -98,12 +96,26 @@ public MigrationRun runManualMigration(String version, String id, Path appHome, break; } } - return run(c, appHome, params, token); + return run(organizationId, c, appHome, params, token); } } throw new MigrationException("Unable to find migration '" + id + "'"); } + public List runManualMigration(String version, String id, Path appHome, boolean force, boolean offline, ObjectMap params, + String token) throws CatalogException { + List migrationRunList = new LinkedList<>(); + // Migrate all organizations + for (String organizationId : dbAdaptorFactory.getOrganizationIds()) { + if (!ParamConstants.ADMIN_ORGANIZATION.equals(organizationId)) { + migrationRunList.add(runManualMigration(organizationId, version, id, appHome, force, offline, params, token)); + } + } + // Lastly, migrate the admin organization + migrationRunList.add(runManualMigration(ParamConstants.ADMIN_ORGANIZATION, version, id, appHome, force, offline, params, token)); + return migrationRunList; + } + public void runMigration(String version, Collection domainsFilter, Collection languageFilter, boolean offline, String appHome, String token) throws CatalogException, IOException { @@ -113,16 +125,35 @@ public void runMigration(String version, Collection d public void runMigration(String version, Collection domains, Collection languages, boolean offline, String appHome, ObjectMap params, String token) throws CatalogException, IOException { + runMigration(ParamConstants.ADMIN_ORGANIZATION, version, domains, languages, offline, appHome, params, token); + + // ***** Starts code to remove in future versions. Reload MongoDBAdaptorFactory to avoid Notes migration issue. *****/ + try (MongoDBAdaptorFactory mongoDBAdaptorFactory = new MongoDBAdaptorFactory(configuration, catalogManager.getIoManagerFactory())) { + for (String organizationId : mongoDBAdaptorFactory.getOrganizationIds()) { + // ***** Finish code to remove in future versions. Reload MongoDBAdaptorFactory to avoid Notes migration issue. *****/ + + // Migrate all organizations + // for (String organizationId : dbAdaptorFactory.getOrganizationIds()) { + if (!ParamConstants.ADMIN_ORGANIZATION.equals(organizationId)) { + runMigration(organizationId, version, domains, languages, offline, appHome, params, token); + } + } + } + } + + public void runMigration(String organizationId, String version, Collection domains, + Collection languages, boolean offline, String appHome, ObjectMap params, + String token) throws CatalogException, IOException { - logger.info("Run migrations"); + logger.info("Running migrations for organization '{}'", organizationId); if (StringUtils.isNotEmpty(version)) { - logger.info(" - Version : " + version); + logger.info(" - Version : {}", version); } if (CollectionUtils.isNotEmpty(domains)) { - logger.info(" - Domains : " + domains); + logger.info(" - Domains : {}", domains); } if (CollectionUtils.isNotEmpty(languages)) { - logger.info(" - Languages : " + languages); + logger.info(" - Languages : {}", languages); } Path appHomePath = Paths.get(appHome); @@ -131,15 +162,15 @@ public void runMigration(String version, Collection d token = validateAdmin(token); // 0. Fetch all migrations - updateMigrationRuns(token); + updateMigrationRuns(organizationId, token); Set> availableMigrations = getAvailableMigrations(); // 1. Fetch required migrations sorted by rank - List> runnableMigrations = filterRunnableMigrations(version, domains, languages, + List> runnableMigrations = filterRunnableMigrations(organizationId, version, domains, languages, availableMigrations); // 2. Get pending migrations - List> pendingMigrations = filterPendingMigrations(version, availableMigrations); + List> pendingMigrations = filterPendingMigrations(organizationId, version, availableMigrations); if (runnableMigrations.isEmpty() && pendingMigrations.isEmpty()) { logger.info("Nothing to run. OpenCGA is up to date"); @@ -172,33 +203,36 @@ public void runMigration(String version, Collection d // 3. Execute pending migrations for (Class migration : pendingMigrations) { - run(migration, appHomePath, new ObjectMap(), token); + run(organizationId, migration, appHomePath, new ObjectMap(), token); } // 4. Execute target migration for (Class migration : runnableMigrations) { - run(migration, appHomePath, params, token); + run(organizationId, migration, appHomePath, params, token); } } - public List> getPendingMigrations(String version, String token) throws CatalogException { - validateAdmin(token); - updateMigrationRuns(token); + public List> getPendingMigrations(String organizationId, String version, String token) + throws CatalogException { + token = validateAdmin(token); + updateMigrationRuns(organizationId, token); Set> availableMigrations = getAvailableMigrations(); - return filterPendingMigrations(version, availableMigrations); + return filterPendingMigrations(organizationId, version, availableMigrations); } - private List getMigrations() { - Set> availableMigrations = getAvailableMigrations(); - List migrations = new ArrayList<>(availableMigrations.size()); - for (Class migrationClass : availableMigrations) { - migrations.add(getMigrationAnnotation(migrationClass)); + public Map getMigrationSummary() throws CatalogException { + Map migrationSummaryMap = new HashMap<>(); + // Loop over organizations + for (String organizationId : dbAdaptorFactory.getOrganizationIds()) { + MigrationSummary migrationSummary = getMigrationSummary(organizationId); + migrationSummaryMap.put(organizationId, migrationSummary); } - return migrations; + return migrationSummaryMap; } - public MigrationSummary getMigrationSummary() throws CatalogException { - List> runs = getMigrationRuns(null, null, null); + private MigrationSummary getMigrationSummary(String organizationId) throws CatalogException { + logger.info("Fetching migration summary for organization '{}'", organizationId); + List> runs = getMigrationRuns(organizationId, null, null, null); MigrationSummary migrationSummary = new MigrationSummary() .setStatusCount(runs.stream().collect(Collectors.groupingBy( @@ -219,22 +253,77 @@ public MigrationSummary getMigrationSummary() throws CatalogException { return migrationSummary; } - public List> getMigrationRuns(String token) throws CatalogException { - return getMigrationRuns(null, null, null, token); + public List> getMigrationRuns(String organizationId, String token) throws CatalogException { + return getMigrationRuns(organizationId, null, null, null, token); } - public List> getMigrationRuns(String version, List domain, - List status, String token) throws CatalogException { - validateAdmin(token); + public List> getMigrationRuns(String organizationId, String version, + List domain, List status, String token) + throws CatalogException { + token = validateAdmin(token); // 0. Update migration runs - updateMigrationRuns(token); + updateMigrationRuns(organizationId, token); + + return getMigrationRuns(organizationId, version, domain, status); + } + + // This method should only be called when installing OpenCGA for the first time so it skips all available (and old) migrations. + public void skipPendingMigrations(String organizationId, String token) throws CatalogException { + validateAdmin(token); + + // 0. Fetch all migrations + Set> availableMigrations = getAvailableMigrations(); + + // 1. Skip all available migrations + for (Class runnableMigration : availableMigrations) { + Migration annotation = getMigrationAnnotation(runnableMigration); + + MigrationRun migrationRun = new MigrationRun(annotation.id(), annotation.description(), annotation.version(), + TimeUtils.getDate(), TimeUtils.getDate(), annotation.patch(), MigrationRun.MigrationStatus.REDUNDANT, ""); + try { + dbAdaptorFactory.getMigrationDBAdaptor(organizationId).upsert(migrationRun); + } catch (CatalogDBException e) { + throw new MigrationException("Could not register migration in OpenCGA", e); + } + } + } + + public void updateMigrationRuns(String token) throws CatalogException { + // Loop over all organizations + for (String organizationId : dbAdaptorFactory.getOrganizationIds()) { + if (!ParamConstants.ADMIN_ORGANIZATION.equals(organizationId)) { + updateMigrationRuns(organizationId, token); + } + } + // Lastly, migrate the admin organization + updateMigrationRuns(ParamConstants.ADMIN_ORGANIZATION, token); + } + + public void updateMigrationRuns(String organizationId, String token) throws CatalogException { + token = validateAdmin(token); + + // 0. Fetch all migrations + Set> availableMigrations = getAvailableMigrations(); - return getMigrationRuns(version, domain, status); + logger.info("Updating migration runs for organization '{}'", organizationId); + // 1. Update migration run status + for (Class runnableMigration : availableMigrations) { + updateMigrationRun(organizationId, getMigrationAnnotation(runnableMigration), token); + } + } + + private List getMigrations() { + Set> availableMigrations = getAvailableMigrations(); + List migrations = new ArrayList<>(availableMigrations.size()); + for (Class migrationClass : availableMigrations) { + migrations.add(getMigrationAnnotation(migrationClass)); + } + return migrations; } - private List> getMigrationRuns(String version, List domain, - List status) + private List> getMigrationRuns(String organizationId, String version, + List domain, List status) throws CatalogException { // 1. Get migrations and filter @@ -247,7 +336,8 @@ private List> getMigrationRuns(String version, Lis } // 2. Get migration runs and filter by status - List migrationRuns = migrationDBAdaptor.get(migrations.stream().map(Migration::id).collect(Collectors.toList())) + List migrationRuns = dbAdaptorFactory.getMigrationDBAdaptor(organizationId) + .get(migrations.stream().map(Migration::id).collect(Collectors.toList())) .getResults(); Map> map = new HashMap<>(migrations.size()); @@ -276,42 +366,8 @@ private List> getMigrationRuns(String version, Lis return pairs; } - // This method should only be called when installing OpenCGA for the first time so it skips all available (and old) migrations. - public void skipPendingMigrations(String token) throws CatalogException { - validateAdmin(token); - - // 0. Fetch all migrations - Set> availableMigrations = getAvailableMigrations(); - - // 1. Skip all available migrations - for (Class runnableMigration : availableMigrations) { - Migration annotation = getMigrationAnnotation(runnableMigration); - - MigrationRun migrationRun = new MigrationRun(annotation.id(), annotation.description(), annotation.version(), - TimeUtils.getDate(), TimeUtils.getDate(), annotation.patch(), MigrationRun.MigrationStatus.REDUNDANT, ""); - try { - migrationDBAdaptor.upsert(migrationRun); - } catch (CatalogDBException e) { - throw new MigrationException("Could not register migration in OpenCGA", e); - } - } - } - - public void updateMigrationRuns(String token) throws CatalogException { - validateAdmin(token); - - // 0. Fetch all migrations - Set> availableMigrations = getAvailableMigrations(); - - // 1. Update migration run status - for (Class runnableMigration : availableMigrations) { - updateMigrationRun(getMigrationAnnotation(runnableMigration), token); - } - - } - - private MigrationRun updateMigrationRun(Migration migration, String token) throws CatalogException { - MigrationRun migrationRun = migrationDBAdaptor.get(migration.id()).first(); + private MigrationRun updateMigrationRun(String organizationId, Migration migration, String token) throws CatalogException { + MigrationRun migrationRun = dbAdaptorFactory.getMigrationDBAdaptor(organizationId).get(migration.id()).first(); boolean updated = false; if (migrationRun == null) { @@ -337,7 +393,7 @@ private MigrationRun updateMigrationRun(Migration migration, String token) throw break; case ON_HOLD: // Check jobs - MigrationRun.MigrationStatus status = getOnHoldMigrationRunStatus(migration, migrationRun, token); + MigrationRun.MigrationStatus status = getOnHoldMigrationRunStatus(organizationId, migration, migrationRun, token); migrationRun.setStatus(status); if (status != MigrationRun.MigrationStatus.ON_HOLD) { updated = true; @@ -353,18 +409,19 @@ private MigrationRun updateMigrationRun(Migration migration, String token) throw } } if (updated) { - migrationDBAdaptor.upsert(migrationRun); + dbAdaptorFactory.getMigrationDBAdaptor(organizationId).upsert(migrationRun); } return migrationRun; } - private MigrationRun.MigrationStatus getOnHoldMigrationRunStatus(Migration migration, MigrationRun migrationRun, String token) - throws CatalogException { + private MigrationRun.MigrationStatus getOnHoldMigrationRunStatus(String organizationId, Migration migration, MigrationRun migrationRun, + String token) throws CatalogException { boolean allDone = true; boolean anyError = false; for (JobReferenceParam jobR : migrationRun.getJobs()) { Job job = catalogManager.getJobManager() - .get(jobR.getStudyId(), jobR.getId(), new QueryOptions(QueryOptions.INCLUDE, "id,internal"), token).first(); + .get(jobR.getStudyId(), jobR.getId(), new QueryOptions(QueryOptions.INCLUDE, "id,internal"), token) + .first(); String jobStatus = job.getInternal().getStatus().getId(); if (jobStatus.equals(Enums.ExecutionStatus.ERROR) || jobStatus.equals(Enums.ExecutionStatus.ABORTED)) { @@ -389,7 +446,7 @@ private MigrationRun.MigrationStatus getOnHoldMigrationRunStatus(Migration migra } } - private List> filterPendingMigrations(String version, + private List> filterPendingMigrations(String organizationId, String version, Set> availableMigrations) throws MigrationException { @@ -420,15 +477,16 @@ private List> filterPendingMigrations(String vers } // Exclude successfully executed migrations - filterOutExecutedMigrations(migrations); + filterOutExecutedMigrations(organizationId, migrations); return migrations; } private String validateAdmin(String token) throws CatalogException { - String userId = catalogManager.getUserManager().getUserId(token); - catalogManager.getAuthorizationManager().checkIsInstallationAdministrator(userId); + JwtPayload jwtPayload = catalogManager.getUserManager().validateToken(token); + catalogManager.getAuthorizationManager().checkIsOpencgaAdministrator(jwtPayload); // Extend token life - return catalogManager.getUserManager().getNonExpiringToken(AbstractManager.OPENCGA, Collections.emptyMap(), token); + return catalogManager.getUserManager().getNonExpiringToken(jwtPayload.getOrganization(), jwtPayload.getUserId(), + Collections.emptyMap(), token); } private Set> getAvailableMigrations() { @@ -474,7 +532,7 @@ private static Collection getUrls() { Collection urls = new LinkedList<>(); for (URL url : ClasspathHelper.forPackage("org.opencb.opencga")) { String name = url.getPath().substring(url.getPath().lastIndexOf('/') + 1); - if (name.isEmpty() || (name.contains("opencga") && !name.contains("opencga-storage-hadoop-deps"))) { + if (name.isEmpty() || (name.contains("opencga") && !name.contains("opencga-hadoop-shaded"))) { urls.add(url); } } @@ -540,7 +598,7 @@ protected static int compareVersion(String version1, String version2) { return 0; } - private List> filterRunnableMigrations(String version, + private List> filterRunnableMigrations(String organizationId, String version, Collection domain, Collection language, Set> allMigrations) @@ -562,14 +620,18 @@ private List> filterRunnableMigrations(String ver .sorted(this::compareTo) .collect(Collectors.toList()); - filterOutExecutedMigrations(filteredMigrations); + filterOutExecutedMigrations(organizationId, filteredMigrations); return filteredMigrations; } - private MigrationRun run(Class runnableMigration, Path appHome, ObjectMap params, String token) - throws MigrationException { + private MigrationRun run(String organizationId, Class runnableMigration, Path appHome, ObjectMap params, + String token) throws MigrationException { Migration annotation = getMigrationAnnotation(runnableMigration); + if (StringUtils.isNotEmpty(annotation.deprecatedSince())) { + throw MigrationException.deprecatedMigration(annotation); + } + MigrationTool migrationTool; try { migrationTool = runnableMigration.newInstance(); @@ -580,7 +642,7 @@ private MigrationRun run(Class runnableMigration, Path Date start = TimeUtils.getDate(); MigrationRun migrationRun; try { - migrationRun = migrationDBAdaptor.get(annotation.id()).first(); + migrationRun = dbAdaptorFactory.getMigrationDBAdaptor(organizationId).get(annotation.id()).first(); if (migrationRun == null) { migrationRun = new MigrationRun(); } @@ -593,7 +655,7 @@ private MigrationRun run(Class runnableMigration, Path } catch (CatalogDBException e) { throw new MigrationException("Error reading migration run from catalog", e); } - migrationTool.setup(configuration, catalogManager, dbAdaptorFactory, migrationRun, appHome, params, token); + migrationTool.setup(configuration, catalogManager, dbAdaptorFactory, migrationRun, organizationId, appHome, params, token); StopWatch stopWatch = StopWatch.createStarted(); String path = Paths.get("JOBS") @@ -625,7 +687,7 @@ private MigrationRun run(Class runnableMigration, Path if (migrationRun.getJobs().isEmpty()) { status = MigrationRun.MigrationStatus.DONE; } else { - status = getOnHoldMigrationRunStatus(migrationTool.getAnnotation(), migrationRun, token); + status = getOnHoldMigrationRunStatus(organizationId, migrationTool.getAnnotation(), migrationRun, token); } } // Clear exception @@ -663,10 +725,11 @@ private MigrationRun run(Class runnableMigration, Path migrationRun.setEnd(TimeUtils.getDate()); migrationRun.setPatch(annotation.patch()); try { - String adminStudy = "opencga@admin:admin"; - migrationDBAdaptor.upsert(migrationRun); + String adminStudy = ParamConstants.ADMIN_ORGANIZATION + "@admin:admin"; + dbAdaptorFactory.getMigrationDBAdaptor(organizationId).upsert(migrationRun); OpenCGAResult outdir = catalogManager.getFileManager() - .createFolder(adminStudy, path, true, "Migration job " + migrationRun.getId(), null, QueryOptions.empty(), token); + .createFolder(adminStudy, path, true, "Migration job " + migrationRun.getId(), null, + QueryOptions.empty(), token); OpenCGAResult stderr = catalogManager.getFileManager() .link(adminStudy, new FileLinkParams() .setPath(Paths.get(path, logFile).toString()) @@ -722,12 +785,13 @@ private MigrationRun run(Class runnableMigration, Path return migrationRun; } - private void filterOutExecutedMigrations(List> migrations) throws MigrationException { + private void filterOutExecutedMigrations(String organizationId, List> migrations) + throws MigrationException { // Remove migrations successfully executed from list List migrationIdList = migrations.stream().map(m -> getMigrationAnnotation(m).id()).collect(Collectors.toList()); OpenCGAResult migrationResult; try { - migrationResult = migrationDBAdaptor.get(migrationIdList); + migrationResult = dbAdaptorFactory.getMigrationDBAdaptor(organizationId).get(migrationIdList); } catch (CatalogDBException e) { throw new MigrationException(e.getMessage(), e); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationRuntimeException.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationRuntimeException.java new file mode 100644 index 00000000000..01d1d59d4a2 --- /dev/null +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationRuntimeException.java @@ -0,0 +1,13 @@ +package org.opencb.opencga.catalog.migration; + +public class MigrationRuntimeException extends IllegalArgumentException { + + public MigrationRuntimeException(String message) { + super(message); + } + + public MigrationRuntimeException(String message, Throwable cause) { + super(message, cause); + } + +} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationSummary.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationSummary.java index 62bb21d31fa..b86e03b22ca 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationSummary.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationSummary.java @@ -1,6 +1,7 @@ package org.opencb.opencga.catalog.migration; import org.opencb.commons.datastore.core.Event; +import org.opencb.opencga.core.models.migration.MigrationRun; import java.util.EnumMap; import java.util.LinkedList; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationTool.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationTool.java index 15adbb74e5f..1a57bb6a239 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationTool.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationTool.java @@ -3,6 +3,8 @@ import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoCursor; import com.mongodb.client.model.IndexOptions; +import com.mongodb.client.model.InsertOneModel; +import com.mongodb.client.model.Projections; import com.mongodb.client.model.WriteModel; import org.apache.commons.lang3.StringUtils; import org.bson.Document; @@ -10,19 +12,18 @@ import org.opencb.commons.ProgressLogger; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.mongodb.GenericDocumentComplexConverter; -import org.opencb.commons.datastore.mongodb.MongoDBConfiguration; import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; +import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.managers.CatalogManager; import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.config.storage.StorageConfiguration; +import org.opencb.opencga.core.models.migration.MigrationRun; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.BufferedReader; import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStreamReader; import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; @@ -34,6 +35,7 @@ public abstract class MigrationTool { protected CatalogManager catalogManager; protected MongoDBAdaptorFactory dbAdaptorFactory; protected MigrationRun migrationRun; + protected String organizationId; protected Path appHome; protected String token; @@ -66,11 +68,12 @@ public final String getId() { } public final void setup(Configuration configuration, CatalogManager catalogManager, MongoDBAdaptorFactory dbAdaptorFactory, - MigrationRun migrationRun, Path appHome, ObjectMap params, String token) { + MigrationRun migrationRun, String organizationId, Path appHome, ObjectMap params, String token) { this.configuration = configuration; this.catalogManager = catalogManager; this.dbAdaptorFactory = dbAdaptorFactory; this.migrationRun = migrationRun; + this.organizationId = organizationId; this.appHome = appHome; this.params = params; this.token = token; @@ -78,6 +81,10 @@ public final void setup(Configuration configuration, CatalogManager catalogManag public final void execute() throws MigrationException { try { + Migration annotation = getAnnotation(); + if (StringUtils.isNotEmpty(annotation.deprecatedSince())) { + throw MigrationException.deprecatedMigration(annotation); + } run(); } catch (MigrationException e) { throw e; @@ -103,65 +110,6 @@ protected final StorageConfiguration readStorageConfiguration() throws Migration return storageConfiguration; } - protected final void runJavascript(Path file) throws MigrationException { - String authentication = ""; - if (StringUtils.isNotEmpty(configuration.getCatalog().getDatabase().getUser()) - && StringUtils.isNotEmpty(configuration.getCatalog().getDatabase().getPassword())) { - authentication = "-u " + configuration.getCatalog().getDatabase().getUser() + " -p " - + configuration.getCatalog().getDatabase().getPassword() + " --authenticationDatabase " - + configuration.getCatalog().getDatabase().getOptions().getOrDefault("authenticationDatabase", "admin") + " "; - } - if (configuration.getCatalog().getDatabase().getOptions() != null - && configuration.getCatalog().getDatabase().getOptions().containsKey(MongoDBConfiguration.SSL_ENABLED) - && Boolean.parseBoolean(configuration.getCatalog().getDatabase().getOptions().get(MongoDBConfiguration.SSL_ENABLED))) { - authentication += "--ssl "; - } - if (configuration.getCatalog().getDatabase().getOptions() != null - && configuration.getCatalog().getDatabase().getOptions().containsKey(MongoDBConfiguration.SSL_INVALID_CERTIFICATES_ALLOWED) - && Boolean.parseBoolean(configuration.getCatalog().getDatabase().getOptions() - .get(MongoDBConfiguration.SSL_INVALID_CERTIFICATES_ALLOWED))) { - authentication += "--sslAllowInvalidCertificates "; - } - if (configuration.getCatalog().getDatabase().getOptions() != null - && configuration.getCatalog().getDatabase().getOptions().containsKey(MongoDBConfiguration.SSL_INVALID_HOSTNAME_ALLOWED) - && Boolean.parseBoolean(configuration.getCatalog().getDatabase().getOptions() - .get(MongoDBConfiguration.SSL_INVALID_HOSTNAME_ALLOWED))) { - authentication += "--sslAllowInvalidHostnames "; - } - if (configuration.getCatalog().getDatabase().getOptions() != null && StringUtils.isNotEmpty( - configuration.getCatalog().getDatabase().getOptions().get(MongoDBConfiguration.AUTHENTICATION_MECHANISM))) { - authentication += "--authenticationMechanism " - + configuration.getCatalog().getDatabase().getOptions().get(MongoDBConfiguration.AUTHENTICATION_MECHANISM) + " "; - } - - String catalogCli = "mongo " + authentication - + StringUtils.join(configuration.getCatalog().getDatabase().getHosts(), ",") + "/" - + catalogManager.getCatalogDatabase() + " " + file.getFileName(); - - privateLogger.info("Running Javascript cli {} from {}", catalogCli, file.getParent()); - ProcessBuilder processBuilder = new ProcessBuilder(catalogCli.split(" ")); - processBuilder.directory(file.getParent().toFile()); - Process p; - try { - p = processBuilder.start(); - BufferedReader input = new BufferedReader(new InputStreamReader(p.getInputStream())); - String line; - while ((line = input.readLine()) != null) { - privateLogger.info(line); - } - p.waitFor(); - input.close(); - } catch (IOException | InterruptedException e) { - throw new MigrationException("Error executing cli: " + e.getMessage(), e); - } - - if (p.exitValue() == 0) { - privateLogger.info("Finished Javascript catalog migration"); - } else { - throw new MigrationException("Error with Javascript catalog migrating!"); - } - } - @FunctionalInterface protected interface MigrateCollectionFunc { void accept(Document document, List> bulk); @@ -172,11 +120,13 @@ protected interface QueryCollectionFunc { void accept(Document document); } - protected final void migrateCollection(String collection, Bson query, Bson projection, MigrateCollectionFunc migrateFunc) { + protected final void migrateCollection(String collection, Bson query, Bson projection, MigrateCollectionFunc migrateFunc) + throws CatalogDBException { migrateCollection(collection, collection, query, projection, migrateFunc); } - protected final void migrateCollection(List collections, Bson query, Bson projection, MigrateCollectionFunc migrateFunc) { + protected final void migrateCollection(List collections, Bson query, Bson projection, MigrateCollectionFunc migrateFunc) + throws CatalogDBException { for (String collection : collections) { privateLogger.info("Starting migration in {}", collection); migrateCollection(collection, collection, query, projection, migrateFunc); @@ -184,8 +134,9 @@ protected final void migrateCollection(List collections, Bson query, Bso } protected final void migrateCollection(String inputCollection, String outputCollection, Bson query, Bson projection, - MigrateCollectionFunc migrateFunc) { - migrateCollection(getMongoCollection(inputCollection), getMongoCollection(outputCollection), query, projection, migrateFunc); + MigrateCollectionFunc migrateFunc) throws CatalogDBException { + migrateCollection(getMongoCollection(inputCollection), + getMongoCollection(outputCollection), query, projection, migrateFunc); } protected final void migrateCollection(MongoCollection inputCollection, MongoCollection outputCollection, @@ -225,15 +176,34 @@ protected final void migrateCollection(MongoCollection inputCollection } } - protected final void createIndex(String collection, Document index) { + protected void copyData(Bson query, String sourceCol, String targetCol) throws CatalogDBException { + MongoCollection sourceMongoCollection = getMongoCollection(sourceCol); + MongoCollection targetMongoCollection = getMongoCollection(targetCol); + copyData(query, sourceMongoCollection, targetMongoCollection); + } + + protected void copyData(Bson query, MongoCollection sourceCol, MongoCollection targetCol) { + // Move data to the new collection + logger.info("Copying data from {} to {}", sourceCol.getNamespace(), targetCol.getNamespace()); + migrateCollection(sourceCol, targetCol, query, Projections.exclude("_id"), + (document, bulk) -> bulk.add(new InsertOneModel<>(document))); + } + + protected void moveData(Bson query, MongoCollection sourceCol, MongoCollection targetCol) { + copyData(query, sourceCol, targetCol); + // Remove data from the source collection + sourceCol.deleteMany(query); + } + + protected final void createIndex(String collection, Document index) throws CatalogDBException { createIndex(getMongoCollection(collection), index, new IndexOptions().background(true)); } - protected final void createIndex(List collections, Document index) { + protected final void createIndex(List collections, Document index) throws CatalogDBException { createIndexes(collections, Collections.singletonList(index)); } - protected final void createIndexes(List collections, List indexes) { + protected final void createIndexes(List collections, List indexes) throws CatalogDBException { for (String collection : collections) { for (Document index : indexes) { createIndex(getMongoCollection(collection), index, new IndexOptions().background(true)); @@ -241,7 +211,8 @@ protected final void createIndexes(List collections, List inde } } - protected final void createIndex(String collection, Document index, IndexOptions options) { + protected final void createIndex(String collection, Document index, IndexOptions options) + throws CatalogDBException { createIndex(getMongoCollection(collection), index, options); } @@ -253,7 +224,7 @@ protected final void createIndex(MongoCollection collection, Document collection.createIndex(index, options); } - protected final void dropIndex(String collection, Document index) { + protected final void dropIndex(String collection, Document index) throws CatalogDBException { dropIndex(getMongoCollection(collection), index); } @@ -275,9 +246,14 @@ protected final void dropIndex(MongoCollection collection, Document in } } - protected final void queryMongo(String inputCollectionStr, Bson query, Bson projection, QueryCollectionFunc queryCollectionFunc) { + protected final void queryMongo(String inputCollectionStr, Bson query, Bson projection, QueryCollectionFunc queryCollectionFunc) + throws CatalogDBException { MongoCollection inputCollection = getMongoCollection(inputCollectionStr); + queryMongo(inputCollection, query, projection, queryCollectionFunc); + } + protected final void queryMongo(MongoCollection inputCollection, Bson query, Bson projection, + QueryCollectionFunc queryCollectionFunc) throws CatalogDBException { try (MongoCursor it = inputCollection .find(query) .batchSize(batchSize) @@ -291,8 +267,12 @@ protected final void queryMongo(String inputCollectionStr, Bson query, Bson proj } } - protected final MongoCollection getMongoCollection(String collectionName) { - return dbAdaptorFactory.getMongoDataStore().getDb().getCollection(collectionName); + protected final MongoCollection getMongoCollection(String collectionName) throws CatalogDBException { + return dbAdaptorFactory.getMongoDataStore(organizationId).getDb().getCollection(collectionName); + } + + protected final MongoCollection getMongoCollection(String organization, String collectionName) throws CatalogDBException { + return dbAdaptorFactory.getMongoDataStore(organization).getDb().getCollection(collectionName); } protected Document convertToDocument(T value) { diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/models/ClinicalAnalysisLoadResult.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/models/ClinicalAnalysisLoadResult.java new file mode 100644 index 00000000000..d154b56871e --- /dev/null +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/models/ClinicalAnalysisLoadResult.java @@ -0,0 +1,86 @@ +/* + * Copyright 2015-2020 OpenCB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opencb.opencga.catalog.models; + +import java.util.HashMap; +import java.util.Map; + +public class ClinicalAnalysisLoadResult { + private int numLoaded; + private Map failures; + private String filename; + private int time; + + public ClinicalAnalysisLoadResult() { + this.numLoaded = 0; + failures = new HashMap<>(); + } + + public ClinicalAnalysisLoadResult(int numLoaded, Map failures, String filename, int time) { + this.numLoaded = numLoaded; + this.failures = failures; + this.filename = filename; + this.time = time; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("ClinicalAnalysisLoadResult{"); + sb.append("numLoaded=").append(numLoaded); + sb.append(", failures=").append(failures); + sb.append(", filename='").append(filename).append('\''); + sb.append(", time='").append(time).append('\''); + sb.append('}'); + return sb.toString(); + } + + public int getNumLoaded() { + return numLoaded; + } + + public ClinicalAnalysisLoadResult setNumLoaded(int numLoaded) { + this.numLoaded = numLoaded; + return this; + } + + public Map getFailures() { + return failures; + } + + public ClinicalAnalysisLoadResult setFailures(Map failures) { + this.failures = failures; + return this; + } + + public String getFilename() { + return filename; + } + + public ClinicalAnalysisLoadResult setFilename(String filename) { + this.filename = filename; + return this; + } + + public int getTime() { + return time; + } + + public ClinicalAnalysisLoadResult setTime(int time) { + this.time = time; + return this; + } +} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/CatalogSolrManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/CatalogSolrManager.java deleted file mode 100644 index b1c2df87d1f..00000000000 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/CatalogSolrManager.java +++ /dev/null @@ -1,294 +0,0 @@ -/* - * Copyright 2015-2020 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.catalog.stats.solr; - -import org.apache.commons.lang3.StringUtils; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.response.UpdateResponse; -import org.opencb.commons.datastore.core.*; -import org.opencb.commons.datastore.solr.SolrCollection; -import org.opencb.commons.datastore.solr.SolrManager; -import org.opencb.commons.utils.CollectionUtils; -import org.opencb.opencga.catalog.db.api.DBIterator; -import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.managers.AbstractManager; -import org.opencb.opencga.catalog.managers.CatalogManager; -import org.opencb.opencga.core.common.GitRepositoryState; -import org.opencb.opencga.core.config.DatabaseCredentials; -import org.opencb.opencga.core.models.study.Study; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Created by wasim on 27/06/18. - */ -public class CatalogSolrManager implements AutoCloseable { - - private CatalogManager catalogManager; - private SolrManager solrManager; - private int insertBatchSize; - private String DATABASE_PREFIX; - - public static final int DEFAULT_INSERT_BATCH_SIZE = 2000; - public static final String COHORT_SOLR_COLLECTION = "catalog-cohort"; - public static final String FILE_SOLR_COLLECTION = "catalog-file"; - public static final String FAMILY_SOLR_COLLECTION = "catalog-family"; - public static final String INDIVIDUAL_SOLR_COLLECTION = "catalog-individual"; - public static final String SAMPLE_SOLR_COLLECTION = "catalog-sample"; - public static final String JOB_SOLR_COLLECTION = "catalog-job"; - - public static final String COHORT_CONF_SET = "opencga-cohort-configset"; - public static final String FILE_CONF_SET = "opencga-file-configset"; - public static final String FAMILY_CONF_SET = "opencga-family-configset"; - public static final String INDIVIDUAL_CONF_SET = "opencga-individual-configset"; - public static final String SAMPLE_CONF_SET = "opencga-sample-configset"; - public static final String JOB_CONF_SET = "opencga-job-configset"; - private final Map CONFIGS_COLLECTION = new HashMap<>(); - - private Logger logger; - - public CatalogSolrManager(CatalogManager catalogManager) { - this.catalogManager = catalogManager; - DatabaseCredentials searchConfiguration = catalogManager.getConfiguration().getCatalog().getSearchEngine(); - String mode = searchConfiguration.getOptions().getOrDefault("mode", "cloud"); - int timeout = Integer.parseInt(searchConfiguration.getOptions().getOrDefault("timeout", "30000")); - int tmpInsertBatchSize = Integer.parseInt(searchConfiguration.getOptions().getOrDefault("insertBatchSize", - String.valueOf(DEFAULT_INSERT_BATCH_SIZE))); - insertBatchSize = tmpInsertBatchSize > 0 ? tmpInsertBatchSize : DEFAULT_INSERT_BATCH_SIZE; - this.solrManager = new SolrManager(searchConfiguration.getHosts(), mode, timeout); - - DATABASE_PREFIX = catalogManager.getConfiguration().getDatabasePrefix() + "-"; - - populateConfigCollectionMap(); - - logger = LoggerFactory.getLogger(CatalogSolrManager.class); - } - - public boolean isAlive(String collection) { - return solrManager.isAlive(collection); - } - - public void create(String dbName, String configSet) { - solrManager.create(dbName, configSet); - } - - public void createCore(String coreName, String configSet) { - solrManager.createCore(coreName, configSet); - } - - public void createCollection(String collectionName, String configSet) { - solrManager.createCollection(collectionName, configSet); - } - - public boolean exists(String dbName) { - return solrManager.exists(dbName); - } - - public boolean existsCore(String coreName) { - return solrManager.existsCore(coreName); - } - - public boolean existsCollection(String collectionName) { - return solrManager.existsCollection(collectionName); - } - - public void createSolrCollections(String solrCollection) throws CatalogException { - if (!CONFIGS_COLLECTION.containsKey(DATABASE_PREFIX + solrCollection)) { - throw new CatalogException("Cannot create solr collection '" + solrCollection + "'. Solr collection does not exist."); - } - - if (catalogManager.getConfiguration().getCatalog().getSearchEngine().getOptions().getOrDefault("mode", "cloud").equals("cloud")) { - createCatalogSolrCollections(solrCollection); - } else { - createCatalogSolrCores(solrCollection); - } - } - - private void createCatalogSolrCollections(String solrCollection) { - String key = DATABASE_PREFIX + solrCollection; - if (!existsCollection(key)) { - createCollection(key, CONFIGS_COLLECTION.get(key)); - } - } - - private void createCatalogSolrCores(String solrCollection) { - String key = DATABASE_PREFIX + solrCollection; - if (!existsCore(key)) { - createCore(key, CONFIGS_COLLECTION.get(key)); - } - } - - public void insertCatalogCollection(DBIterator iterator, ComplexTypeConverter converter, String collectionName) - throws CatalogException { - - int count = 0; - List records = new ArrayList<>(insertBatchSize); - while (iterator.hasNext()) { - T record = iterator.next(); - records.add(record); - count++; - if (count % insertBatchSize == 0) { - insertCatalogCollection(records, converter, collectionName); - records.clear(); - } - } - - if (CollectionUtils.isNotEmpty(records)) { - insertCatalogCollection(records, converter, collectionName); - } - } - - public void insertCatalogCollection(List records, ComplexTypeConverter converter, String collectionName) - throws CatalogException { - List solrModels = new ArrayList<>(); - - for (T record : records) { - solrModels.add((M) converter.convertToStorageType(record)); - } - - UpdateResponse updateResponse; - try { - updateResponse = solrManager.getSolrClient().addBeans(DATABASE_PREFIX + collectionName, solrModels); - if (updateResponse.getStatus() == 0) { - solrManager.getSolrClient().commit(DATABASE_PREFIX + collectionName); - } else { - throw new CatalogException(updateResponse.getException()); - } - } catch (IOException | SolrServerException e) { - throw new CatalogException(e.getMessage(), e); - } - } - - /** - * Return faceted data from a Solr core/collection - * according a given query. - * - * @param collection Collection name - * @param query Query - * @param queryOptions Query options (contains the facet and facetRange options) - * @return List of Variant objects - * @throws IOException IOException - * @throws CatalogException CatalogException - */ - @Deprecated - public DataResult facetedQuery(String collection, Query query, QueryOptions queryOptions) - throws IOException, CatalogException { - CatalogSolrQueryParser catalogSolrQueryParser = new CatalogSolrQueryParser(); - SolrQuery solrQuery = catalogSolrQueryParser.parse(query, queryOptions, null); - - SolrCollection solrCollection = solrManager.getCollection(DATABASE_PREFIX + collection); - try { - return solrCollection.facet(solrQuery, catalogSolrQueryParser.getAliasMap()); - } catch (SolrServerException e) { - throw new CatalogException(e.getMessage(), e); - } - } - - /** - * Return faceted data from a Solr core/collection according to a given query. - * - * @param study Study - * @param collection Collection name - * @param query Query - * @param queryOptions Query options (contains the facet and facetRange options) - * @param userId User performing the facet query. - * @return Facet results - * @throws CatalogException CatalogException - */ - public DataResult facetedQuery(Study study, String collection, Query query, QueryOptions queryOptions, String userId) - throws CatalogException { - Query queryCopy = query == null ? new Query() : new Query(query); - QueryOptions queryOptionsCopy = queryOptions == null ? new QueryOptions() : new QueryOptions(queryOptions); - - queryCopy.put(CatalogSolrQueryParser.QueryParams.STUDY.key(), study.getFqn()); - - if (!catalogManager.getAuthorizationManager().isOwnerOrAdmin(study.getUid(), userId)) { - // We need to add an acl query to perform the facet query - List groups = new ArrayList<>(); - study.getGroups().forEach(group -> { - if (group.getUserIds().contains(userId) || group.getUserIds().contains(AbstractManager.ANONYMOUS)) { - groups.add(group.getId()); - } - }); - - final String suffixPermission; - if (queryCopy.containsKey(CatalogSolrQueryParser.QueryParams.ANNOTATIONS.key())) { - suffixPermission = "__VIEW_ANNOTATIONS"; - } else { - suffixPermission = "__VIEW"; - } - List aclList = new ArrayList<>(); - if (AbstractManager.ANONYMOUS.equals(userId)) { - // We need to escape that user ( * ) - aclList.add("\\" + userId + suffixPermission); - groups.forEach(group -> aclList.add("(*:* -\\" + userId + "__NONE AND " + group + suffixPermission + ")")); - } else { - aclList.add(userId + suffixPermission); - groups.forEach(group -> aclList.add("(*:* -" + userId + "__NONE AND " + group + suffixPermission + ")")); - // Add anonymous user as well. If anonymous user can see, userId should be able to see - aclList.add("\\" + AbstractManager.ANONYMOUS + suffixPermission); - } - - queryCopy.put(CatalogSolrQueryParser.QueryParams.ACL.key(), "(" + StringUtils.join(aclList, " OR ") + ")"); - } - - CatalogSolrQueryParser catalogSolrQueryParser = new CatalogSolrQueryParser(); - SolrQuery solrQuery = catalogSolrQueryParser.parse(queryCopy, queryOptionsCopy, study.getVariableSets()); - - SolrCollection solrCollection = solrManager.getCollection(DATABASE_PREFIX + collection); - try { - return solrCollection.facet(solrQuery, catalogSolrQueryParser.getAliasMap()); - } catch (SolrServerException | IOException e) { - throw new CatalogException(e.getMessage(), e); - } - } - - //***************** PRIVATE ****************/ - - private void populateConfigCollectionMap() { - String version = GitRepositoryState.getInstance().getBuildVersion(); - - CONFIGS_COLLECTION.put(DATABASE_PREFIX + COHORT_SOLR_COLLECTION, COHORT_CONF_SET + "-" + version); - CONFIGS_COLLECTION.put(DATABASE_PREFIX + FILE_SOLR_COLLECTION, FILE_CONF_SET + "-" + version); - CONFIGS_COLLECTION.put(DATABASE_PREFIX + FAMILY_SOLR_COLLECTION, FAMILY_CONF_SET + "-" + version); - CONFIGS_COLLECTION.put(DATABASE_PREFIX + INDIVIDUAL_SOLR_COLLECTION, INDIVIDUAL_CONF_SET + "-" + version); - CONFIGS_COLLECTION.put(DATABASE_PREFIX + SAMPLE_SOLR_COLLECTION, SAMPLE_CONF_SET + "-" + version); - CONFIGS_COLLECTION.put(DATABASE_PREFIX + JOB_SOLR_COLLECTION, JOB_CONF_SET + "-" + version); - } - - protected void setSolrClient(SolrClient solrClient) { - solrManager.setSolrClient(solrClient); - } - - @Override - public void close() throws CatalogException { - try { - solrManager.close(); - } catch (IOException e) { - throw new CatalogException(e); - } - } -} - diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/CatalogSolrModel.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/CatalogSolrModel.java deleted file mode 100644 index 321847fcece..00000000000 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/CatalogSolrModel.java +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Copyright 2015-2020 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.catalog.stats.solr; - -import org.apache.solr.client.solrj.beans.Field; - -import java.util.List; - -public abstract class CatalogSolrModel { - - @Field - protected String id; - - @Field - protected long uid; - - @Field - protected String studyId; - - @Field - protected int creationYear; - - @Field - protected String creationMonth; - - @Field - protected int creationDay; - - @Field - protected String creationDayOfWeek; - - @Field - protected String status; - - @Field - protected int release; - - @Field - protected List acl; - - public CatalogSolrModel() { - } - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder("CatalogSolrModel{"); - sb.append("id='").append(id).append('\''); - sb.append(", uid=").append(uid); - sb.append(", studyId='").append(studyId).append('\''); - sb.append(", creationYear=").append(creationYear); - sb.append(", creationMonth='").append(creationMonth).append('\''); - sb.append(", creationDay=").append(creationDay); - sb.append(", creationDayOfWeek='").append(creationDayOfWeek).append('\''); - sb.append(", status='").append(status).append('\''); - sb.append(", release=").append(release); - sb.append(", acl=").append(acl); - sb.append('}'); - return sb.toString(); - } - - public String getId() { - return id; - } - - public CatalogSolrModel setId(String id) { - this.id = id; - return this; - } - - public long getUid() { - return uid; - } - - public CatalogSolrModel setUid(long uid) { - this.uid = uid; - return this; - } - - public String getStudyId() { - return studyId; - } - - public CatalogSolrModel setStudyId(String studyId) { - this.studyId = studyId; - return this; - } - - public int getCreationYear() { - return creationYear; - } - - public CatalogSolrModel setCreationYear(int creationYear) { - this.creationYear = creationYear; - return this; - } - - public String getCreationMonth() { - return creationMonth; - } - - public CatalogSolrModel setCreationMonth(String creationMonth) { - this.creationMonth = creationMonth; - return this; - } - - public int getCreationDay() { - return creationDay; - } - - public CatalogSolrModel setCreationDay(int creationDay) { - this.creationDay = creationDay; - return this; - } - - public String getCreationDayOfWeek() { - return creationDayOfWeek; - } - - public CatalogSolrModel setCreationDayOfWeek(String creationDayOfWeek) { - this.creationDayOfWeek = creationDayOfWeek; - return this; - } - - public String getStatus() { - return status; - } - - public CatalogSolrModel setStatus(String status) { - this.status = status; - return this; - } - - public int getRelease() { - return release; - } - - public CatalogSolrModel setRelease(int release) { - this.release = release; - return this; - } - - public List getAcl() { - return acl; - } - - public CatalogSolrModel setAcl(List acl) { - this.acl = acl; - return this; - } -} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/CatalogSolrQueryParser.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/CatalogSolrQueryParser.java deleted file mode 100644 index 549311b8be3..00000000000 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/CatalogSolrQueryParser.java +++ /dev/null @@ -1,575 +0,0 @@ -/* - * Copyright 2015-2020 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.catalog.stats.solr; - -import org.apache.commons.collections4.map.LinkedMap; -import org.apache.commons.lang3.StringUtils; -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.common.params.CommonParams; -import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.commons.datastore.core.Query; -import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.commons.datastore.core.QueryParam; -import org.opencb.commons.datastore.solr.FacetQueryParser; -import org.opencb.opencga.catalog.exceptions.CatalogDBException; -import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.utils.AnnotationUtils; -import org.opencb.opencga.catalog.utils.Constants; -import org.opencb.opencga.core.models.study.Variable; -import org.opencb.opencga.core.models.study.VariableSet; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.*; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import static org.opencb.commons.datastore.core.QueryParam.Type.*; - -/** - * Created by wasim on 09/07/18. - */ - -public class CatalogSolrQueryParser { - - protected static Logger logger = LoggerFactory.getLogger(CatalogSolrQueryParser.class); - - public static final Pattern OPERATION_PATTERN = Pattern.compile("^(<=?|>=?|!==?|!?=?~|==?=?)([^=<>~!]+.*)$"); - - private Map aliasMap = new HashMap<>(); - - enum QueryParams { - - // Common - STUDY("study", TEXT), - RELEASE("release", INTEGER), - CREATION_YEAR("creationYear", INTEGER), - CREATION_MONTH("creationMonth", TEXT), - CREATION_DAY("creationDay", INTEGER), - CREATION_DAY_OF_WEEK("creationDayOfWeek", TEXT), - STATUS("status", TEXT), - ANNOTATIONS(Constants.ANNOTATION, TEXT), - ACL("acl", TEXT), - - // Shared - VERSION("version", INTEGER), - TYPE("type", TEXT), - PHENOTYPES("phenotypes", TEXT_ARRAY), - NUM_SAMPLES("numSamples", INTEGER), - - // Sample - SOURCE("source", TEXT), - SOMATIC("somatic", BOOLEAN), - - // Family - NUM_MEMBERS("numMembers", INTEGER), - EXPECTED_SIZE("expectedSize", INTEGER), - - // File - NAME("name", TEXT), - FORMAT("format", TEXT), - BIOFORMAT("bioformat", TEXT), - EXTERNAL("external", BOOLEAN), - SIZE("size", LONG), - SOFTWARE("software", TEXT), - EXPERIMENT("experiment", TEXT), - NUM_RELATED_FILES("numRelatedFiles", INTEGER), - - // Individual - HAS_FATHER("hasFather", BOOLEAN), - HAS_MOTHER("hasMother", BOOLEAN), - SEX("sex", TEXT), - KARYOTYPIC_SEX("karyotypicSex", TEXT), - ETHNICITY("ethnicity", TEXT), - POPULATION("population", TEXT), - LIFE_STATUS("lifeStatus", TEXT), - AFFECTATION_STATUS("affectationStatus", TEXT), - PARENTAL_CONSANGUINITY("parentalConsanguinity", BOOLEAN); - - private static Map map; - static { - map = new LinkedMap(); - for (QueryParams params : QueryParams.values()) { - map.put(params.key(), params); - } - } - - private final String key; - private QueryParam.Type type; - - QueryParams(String key, QueryParam.Type type) { - this.key = key; - this.type = type; - } - - public String key() { - return key; - } - - public QueryParam.Type type() { - return type; - } - - public static Map getMap() { - return map; - } - - public static QueryParams getParam(String key) { - return map.get(key); - } - } - - public CatalogSolrQueryParser() { - } - - /** - * Create a SolrQuery object from Query and QueryOptions. - * - * @param query Query - * @param queryOptions Query Options - * @param variableSetList List of variable sets available for the study whose query is going to be parsed. - * @throws CatalogException CatalogException - * @return SolrQuery - */ - public SolrQuery parse(Query query, QueryOptions queryOptions, List variableSetList) throws CatalogException { - - Map filterList = new HashMap<>(); - SolrQuery solrQuery = new SolrQuery(); - - ObjectMap variableMap = generateVariableMap(variableSetList); - - //------------------------------------- - // Facet processing - //------------------------------------- - if (queryOptions.containsKey(QueryOptions.FACET) && StringUtils.isNotEmpty(queryOptions.getString(QueryOptions.FACET))) { - replaceAnnotationFormat(queryOptions, QueryOptions.FACET, variableMap); - FacetQueryParser facetQueryParser = new FacetQueryParser(); - String facetQuery; - try { - facetQuery = facetQueryParser.parse(queryOptions.getString(QueryOptions.FACET)); - } catch (Exception e) { - throw new CatalogException("Solr parse exception: " + e.getMessage(), e); - } - if (StringUtils.isNotEmpty(facetQuery)) { - solrQuery.set("json.facet", facetQuery); - } - } - - query.entrySet().forEach(entry -> { - QueryParams queryParam = QueryParams.getParam(entry.getKey()); - if (queryParam != null) { - if (queryParam == QueryParams.STUDY) { - filterList.put("studyId", query.getString(queryParam.key()).replace(":", "__")); - } else if (queryParam == QueryParams.ACL) { - filterList.put(entry.getKey(), (String) entry.getValue()); - } else if (queryParam == QueryParams.ANNOTATIONS) { - try { - filterList.putAll(parseAnnotationQueryField(query.getString(Constants.ANNOTATION), - query.get(Constants.PRIVATE_ANNOTATION_PARAM_TYPES, ObjectMap.class), variableSetList)); - } catch (CatalogException e) { - logger.warn("Error parsing annotation info: {}", e.getMessage(), e); - } - } else { - try { - filterList.put(entry.getKey(), getValues(Arrays.asList(String.valueOf(entry.getValue()).split(",")), - queryParam.type())); - } catch (CatalogException e) { - logger.warn("Error parsing parameter {}: {}", entry.getKey(), e.getMessage(), e); - } - } - } - }); - - solrQuery.setQuery("*:*"); - // We only want stats, so we avoid retrieving the first 10 results - solrQuery.add(CommonParams.ROWS, "0"); - filterList.forEach((queryParamter, filter) -> { - solrQuery.addFilterQuery(queryParamter.concat(":").concat(filter)); - logger.debug("Solr fq: {}\n", filter); - }); - - return solrQuery; - - } - - public Map getAliasMap() { - return aliasMap; - } - - private void replaceAnnotationFormat(QueryOptions queryOptions, String facetType, ObjectMap variableMap) { - // variableMap format: Map of full variable path -> Map - // type - // variableSetId - String facet = queryOptions.getString(facetType); - final String prefix = "annotation."; - - // Look for annotation. in facet string - int index = facet.indexOf(prefix); - String copy = facet; - while (index != -1) { - int lastIndex = getMin(copy.indexOf(",", index), Integer.MAX_VALUE); - lastIndex = getMin(copy.indexOf(";", index), lastIndex); - lastIndex = getMin(copy.indexOf("<", index), lastIndex); - lastIndex = getMin(copy.indexOf(">", index), lastIndex); - lastIndex = getMin(copy.indexOf("=", index), lastIndex); - - String annotation; - if (lastIndex == Integer.MAX_VALUE) { - // + 11 because we exclude annotation. prefix - annotation = copy.substring(index + 11); - - // We replace the user input for the annotation key format stored in solr - String annotationReplacement = getInternalAnnotationKey(annotation, variableMap); - facet = facet.replace(prefix + annotation, annotationReplacement); - - // We add the field we have replaced to the map - aliasMap.put(annotationReplacement, "annotation." + annotation); - - index = -1; - } else { - // + 11 because we exclude annotation. prefix - annotation = copy.substring(index + 11, lastIndex); - - // We replace the user input for the annotation key format stored in solr - String annotationReplacement = getInternalAnnotationKey(annotation, variableMap); - facet = facet.replace(prefix + annotation, annotationReplacement); - - // We add the field we have replaced to the map - aliasMap.put(annotationReplacement, "annotation." + annotation); - - copy = copy.substring(lastIndex); - index = copy.indexOf(prefix); - } - } - - queryOptions.put(facetType, facet); - } - - private String getInternalAnnotationKey(String annotation, ObjectMap variableMap) { - ObjectMap annotationMap = (ObjectMap) variableMap.get(annotation); - if (annotationMap == null || annotationMap.isEmpty() && annotation.contains(".")) { - String dynamicAnnotation = annotation.substring(0, annotation.lastIndexOf(".")) + ".*"; - annotationMap = (ObjectMap) variableMap.get(dynamicAnnotation); - } - if (annotationMap == null || annotationMap.isEmpty()) { - logger.error("Cannot parse " + annotation + " string to internal annotation format"); - return ""; - } - - try { - return "annotations" + getAnnotationType((QueryParam.Type) annotationMap.get("type")) - + annotationMap.getString("variableSetId") + "." + annotation; - } catch (CatalogException e) { - logger.error("Cannot parse " + annotation + " string to internal annotation format"); - return ""; - } - } - - private int getMin(int value, int currentValue) { - if (value == -1) { - return currentValue; - } - return value < currentValue ? value : currentValue; - } - - private ObjectMap generateVariableMap(List variableSetList) { - if (variableSetList == null || variableSetList.isEmpty()) { - return new ObjectMap(); - } - // Full variable path -> Map - // type: - // variableSetId: - ObjectMap variableMap = new ObjectMap(); - for (VariableSet variableSet : variableSetList) { - Queue queue = new LinkedList<>(); - for (Variable variable : variableSet.getVariables()) { - // We add the current variable to the queue - ObjectMap auxiliarVariable = new ObjectMap() - .append("variable", variable) - .append("fullVariablePath", "") - .append("isParentArray", false); - queue.add(auxiliarVariable); - } - - while (!queue.isEmpty()) { - ObjectMap auxiliarVariable = queue.remove(); - Variable variable = auxiliarVariable.get("variable", Variable.class); - boolean isParentArray = auxiliarVariable.getBoolean("isParentArray"); - String fullVariablePath = auxiliarVariable.getString("fullVariablePath"); - if (StringUtils.isEmpty(fullVariablePath)) { - fullVariablePath = variable.getId(); - } else { - fullVariablePath = fullVariablePath + "." + variable.getId(); - } - - ObjectMap auxVariableMap = new ObjectMap("variableSetId", variableSet.getId()); - switch (variable.getType()) { - case BOOLEAN: - auxVariableMap.put("type", isParentArray || variable.isMultiValue() - ? QueryParam.Type.BOOLEAN_ARRAY - : QueryParam.Type.BOOLEAN); - variableMap.put(fullVariablePath, auxVariableMap); - break; - case CATEGORICAL: - case STRING: - auxVariableMap.put("type", isParentArray || variable.isMultiValue() - ? QueryParam.Type.TEXT_ARRAY - : QueryParam.Type.TEXT); - variableMap.put(fullVariablePath, auxVariableMap); - break; - case INTEGER: - auxVariableMap.put("type", isParentArray || variable.isMultiValue() - ? LONG_ARRAY - : LONG); - variableMap.put(fullVariablePath, auxVariableMap); - break; - case DOUBLE: - auxVariableMap.put("type", isParentArray || variable.isMultiValue() - ? QueryParam.Type.DECIMAL_ARRAY - : QueryParam.Type.DECIMAL); - variableMap.put(fullVariablePath, auxVariableMap); - break; - case OBJECT: - if (variable.getVariables() != null && !variable.getVariables().isEmpty()) { - for (Variable nestedVariable : variable.getVariables()) { - ObjectMap nestedAuxiliarVariable = new ObjectMap() - .append("variable", nestedVariable) - .append("fullVariablePath", fullVariablePath) - .append("isParentArray", isParentArray || variable.isMultiValue()); - queue.add(nestedAuxiliarVariable); - } - } - break; - case MAP_BOOLEAN: - auxVariableMap.put("type", isParentArray || variable.isMultiValue() ? BOOLEAN_ARRAY : BOOLEAN); - variableMap.put(fullVariablePath + ".*", auxVariableMap); - break; - case MAP_INTEGER: - auxVariableMap.put("type", isParentArray || variable.isMultiValue() ? LONG_ARRAY : LONG); - variableMap.put(fullVariablePath + ".*", auxVariableMap); - break; - case MAP_DOUBLE: - auxVariableMap.put("type", isParentArray || variable.isMultiValue() ? DECIMAL_ARRAY : DECIMAL); - variableMap.put(fullVariablePath + ".*", auxVariableMap); - break; - case MAP_STRING: - auxVariableMap.put("type", isParentArray || variable.isMultiValue() ? TEXT_ARRAY : TEXT); - variableMap.put(fullVariablePath + ".*", auxVariableMap); - break; - default: - break; - } - } - } - - return variableMap; - } - - private Map parseAnnotationQueryField(String annotations, ObjectMap variableTypeMap, List variableSetList) - throws CatalogException { - -// Map variableSetUidIdMap = new HashMap<>(); -// variableSetList.forEach(variableSet -> variableSetUidIdMap.put(variableSet.getUid(), variableSet.getId())); - - Map annotationMap = new HashMap<>(); - - if (StringUtils.isNotEmpty(annotations)) { - // Annotation Filter - final String sepAnd = ";"; - String[] annotationArray = StringUtils.split(annotations, sepAnd); - - for (String annotation : annotationArray) { - Matcher matcher = AnnotationUtils.ANNOTATION_PATTERN.matcher(annotation); - - if (matcher.find()) { - String valueString = matcher.group(4); - - if (annotation.startsWith(Constants.ANNOTATION_SET_NAME)) { - annotationMap.put("annotationSets", getValues(Arrays.asList(valueString.split(",")), QueryParam.Type.TEXT)); - } else { // annotation - // Split the annotation by key - value - // Remove the : at the end of the variableSet - String variableSet = matcher.group(2).replace(":", ""); - // long variableSetUid = Long.valueOf(matcher.group(1).replace(":", "")); - String key = matcher.group(3); - - if (variableTypeMap == null || variableTypeMap.isEmpty()) { - logger.error("Internal error: The variableTypeMap is null or empty {}", variableTypeMap); - throw new CatalogException("Internal error. Could not build the annotation query"); - } - QueryParam.Type type = variableTypeMap.get(variableSet + ":" + key, QueryParam.Type.class); - if (type == null) { - logger.error("Internal error: Could not find the type of the variable {}:{}", variableSet, key); - throw new CatalogException("Internal error. Could not find the type of the variable " + variableSet + ":" - + key); - } - - annotationMap.put("annotations" + getAnnotationType(type) + variableSet + "." + key, - getValues(Arrays.asList(valueString.split(",")), type)); - } - } else { - throw new CatalogDBException("Annotation " + annotation + " could not be parsed to a query."); - } - } - } - - return annotationMap; - } - - private String getAnnotationType(QueryParam.Type type) throws CatalogException { - switch (type) { - case TEXT: - return "__s__"; - case TEXT_ARRAY: - return "__sm__"; - case INTEGER: - return "__i__"; - case INTEGER_ARRAY: - return "__im__"; - case DECIMAL: - return "__d__"; - case DECIMAL_ARRAY: - return "__dm__"; - case BOOLEAN: - return "__b__"; - case BOOLEAN_ARRAY: - return "__bm__"; - default: - throw new CatalogException("Unexpected variable type " + type); - - } - } - - private String getValues(List values, QueryParam.Type type) throws CatalogException { - ArrayList or = new ArrayList<>(values.size()); - for (String option : values) { - Matcher matcher = OPERATION_PATTERN.matcher(option); - String operator; - String filter; - if (!matcher.find()) { - operator = ""; - filter = option; - } else { - operator = matcher.group(1); - filter = matcher.group(2); - } - switch (type) { - case INTEGER: - case INTEGER_ARRAY: - try { - int intValue = Integer.parseInt(filter); - or.add(addNumberOperationQueryFilter(operator, intValue, intValue - 1, intValue + 1)); - } catch (NumberFormatException e) { - throw new CatalogDBException("Expected an integer value - " + e.getMessage(), e); - } - break; - case DECIMAL: - case DECIMAL_ARRAY: - try { - double doubleValue = Double.parseDouble(filter); - or.add(addNumberOperationQueryFilter(operator, doubleValue, doubleValue - 1, doubleValue + 1)); - } catch (NumberFormatException e) { - throw new CatalogDBException("Expected a double value - " + e.getMessage(), e); - } - break; - case TEXT: - case TEXT_ARRAY: - or.add(addStringOperationQueryFilter(operator, filter)); - break; - case BOOLEAN: - case BOOLEAN_ARRAY: - or.add(addBooleanOperationQueryFilter(operator, Boolean.parseBoolean(filter))); - break; - default: - break; - } - } - if (or.isEmpty()) { - return ""; - } else if (or.size() == 1) { - return or.get(0); - } else { - return "(" + StringUtils.join(or, " OR ") + ")"; - } - } - - private String addBooleanOperationQueryFilter(String operator, boolean value) throws CatalogException { - String query; - switch (operator) { - case "!=": - query = "(" + !value + ")"; - break; - case "": - case "=": - case "==": - query = "(" + value + ")"; - break; - default: - throw new CatalogException("Unknown boolean query operation " + operator); - } - return query; - } - - private String addStringOperationQueryFilter(String operator, String filter) throws CatalogException { - String query; - switch (operator) { - case "!=": - query = "(*:* -" + filter + ")"; - break; - case "": - case "=": - case "==": - query = "(" + filter + ")"; - break; - default: - throw new CatalogException("Unknown string query operation " + operator); - } - return query; - } - - private String addNumberOperationQueryFilter(String operator, Number value, Number valueMinus1, Number valuePlus1) - throws CatalogException { - String query; - switch (operator) { - case "<": - query = "[* TO " + valueMinus1 + "]"; - break; - case "<=": - query = "[* TO " + value + "]"; - break; - case ">": - query = "[" + valuePlus1 + " TO *]"; - break; - case ">=": - query = "[" + value + " TO *]"; - break; - case "!=": - query = "-[" + value + " TO " + value + "]"; - break; - case "": - case "=": - case "==": - query = "[" + value + " TO " + value + "]"; - break; - default: - throw new CatalogException("Unknown string query operation " + operator); - } - return query; - } - - -} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/CohortSolrModel.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/CohortSolrModel.java deleted file mode 100644 index a37648ebce4..00000000000 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/CohortSolrModel.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright 2015-2020 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.catalog.stats.solr; - -import org.apache.solr.client.solrj.beans.Field; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Created by wasim on 27/06/18. - */ -public class CohortSolrModel extends CatalogSolrModel { - - @Field - private String type; - - @Field - private int numSamples; - - @Field - private List annotationSets; - - @Field("annotations__*") - private Map annotations; - - public CohortSolrModel() { - this.annotationSets = new ArrayList<>(); - this.annotations = new HashMap<>(); - } - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder("CohortSolrModel{"); - sb.append("id='").append(id).append('\''); - sb.append(", uid=").append(uid); - sb.append(", studyId='").append(studyId).append('\''); - sb.append(", type='").append(type).append('\''); - sb.append(", creationYear=").append(creationYear); - sb.append(", creationMonth='").append(creationMonth).append('\''); - sb.append(", creationDay=").append(creationDay); - sb.append(", creationDayOfWeek='").append(creationDayOfWeek).append('\''); - sb.append(", status='").append(status).append('\''); - sb.append(", release=").append(release); - sb.append(", numSamples=").append(numSamples); - sb.append(", acl=").append(acl); - sb.append(", annotationSets=").append(annotationSets); - sb.append(", annotations=").append(annotations); - sb.append('}'); - return sb.toString(); - } - - public String getType() { - return type; - } - - public CohortSolrModel setType(String type) { - this.type = type; - return this; - } - - public int getNumSamples() { - return numSamples; - } - - public CohortSolrModel setNumSamples(int numSamples) { - this.numSamples = numSamples; - return this; - } - - public List getAnnotationSets() { - return annotationSets; - } - - public CohortSolrModel setAnnotationSets(List annotationSets) { - this.annotationSets = annotationSets; - return this; - } - - public Map getAnnotations() { - return annotations; - } - - public CohortSolrModel setAnnotations(Map annotations) { - this.annotations = annotations; - return this; - } -} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/FamilySolrModel.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/FamilySolrModel.java deleted file mode 100644 index 566b3daf059..00000000000 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/FamilySolrModel.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Copyright 2015-2020 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.catalog.stats.solr; - -import org.apache.solr.client.solrj.beans.Field; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Created by wasim on 27/06/18. - */ -public class FamilySolrModel extends CatalogSolrModel { - - @Field - private List phenotypes; - - @Field - private List disorders; - - @Field - private int numMembers; - - @Field - private int expectedSize; - - @Field - private int version; - - @Field - private List annotationSets; - - @Field("annotations__*") - private Map annotations; - - public FamilySolrModel() { - this.annotationSets = new ArrayList<>(); - this.annotations = new HashMap<>(); - this.phenotypes = new ArrayList<>(); - this.disorders = new ArrayList<>(); - } - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder("FamilySolrModel{"); - sb.append("id='").append(id).append('\''); - sb.append(", uid=").append(uid); - sb.append(", studyId='").append(studyId).append('\''); - sb.append(", creationYear=").append(creationYear); - sb.append(", creationMonth='").append(creationMonth).append('\''); - sb.append(", creationDay=").append(creationDay); - sb.append(", creationDayOfWeek='").append(creationDayOfWeek).append('\''); - sb.append(", status='").append(status).append('\''); - sb.append(", phenotypes=").append(phenotypes); - sb.append(", disorders=").append(disorders); - sb.append(", numMembers=").append(numMembers); - sb.append(", expectedSize=").append(expectedSize); - sb.append(", release=").append(release); - sb.append(", version=").append(version); - sb.append(", acl=").append(acl); - sb.append(", annotationSets=").append(annotationSets); - sb.append(", annotations=").append(annotations); - sb.append('}'); - return sb.toString(); - } - - public List getPhenotypes() { - return phenotypes; - } - - public FamilySolrModel setPhenotypes(List phenotypes) { - this.phenotypes = phenotypes; - return this; - } - - public List getDisorders() { - return disorders; - } - - public FamilySolrModel setDisorders(List disorders) { - this.disorders = disorders; - return this; - } - - public int getNumMembers() { - return numMembers; - } - - public FamilySolrModel setNumMembers(int numMembers) { - this.numMembers = numMembers; - return this; - } - - public int getExpectedSize() { - return expectedSize; - } - - public FamilySolrModel setExpectedSize(int expectedSize) { - this.expectedSize = expectedSize; - return this; - } - - public int getVersion() { - return version; - } - - public FamilySolrModel setVersion(int version) { - this.version = version; - return this; - } - - public List getAnnotationSets() { - return annotationSets; - } - - public FamilySolrModel setAnnotationSets(List annotationSets) { - this.annotationSets = annotationSets; - return this; - } - - public Map getAnnotations() { - return annotations; - } - - public FamilySolrModel setAnnotations(Map annotations) { - this.annotations = annotations; - return this; - } -} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/FileSolrModel.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/FileSolrModel.java deleted file mode 100644 index fc9c1d39176..00000000000 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/FileSolrModel.java +++ /dev/null @@ -1,341 +0,0 @@ -/* - * Copyright 2015-2020 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.catalog.stats.solr; - -import org.apache.solr.client.solrj.beans.Field; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Created by wasim on 27/06/18. - */ -public class FileSolrModel extends CatalogSolrModel { - - @Field - private String name; - - @Field - private String type; - - @Field - private String format; - - @Field - private String bioformat; - - @Field - private boolean external; - - @Field - private long size; - - @Field - private String softwareName; - - @Field - private String softwareVersion; - - @Field - private String experimentTechnology; - - @Field - private String experimentMethod; - - @Field - private String experimentNucleicAcidType; - - @Field - private String experimentManufacturer; - - @Field - private String experimentPlatform; - - @Field - private String experimentLibrary; - - @Field - private String experimentCenter; - - @Field - private String experimentLab; - - @Field - private String experimentResponsible; - - @Field - private List tags; - - @Field - private int numSamples; - - @Field - private int numRelatedFiles; - - @Field - private List annotationSets; - - @Field("annotations__*") - private Map annotations; - - public FileSolrModel() { - this.annotationSets = new ArrayList<>(); - this.annotations = new HashMap<>(); - this.tags = new ArrayList<>(); - } - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder("FileSolrModel{"); - sb.append("id='").append(id).append('\''); - sb.append(", uid=").append(uid); - sb.append(", studyId='").append(studyId).append('\''); - sb.append(", name='").append(name).append('\''); - sb.append(", type='").append(type).append('\''); - sb.append(", format='").append(format).append('\''); - sb.append(", bioformat='").append(bioformat).append('\''); - sb.append(", release=").append(release); - sb.append(", creationYear=").append(creationYear); - sb.append(", creationMonth='").append(creationMonth).append('\''); - sb.append(", creationDay=").append(creationDay); - sb.append(", creationDayOfWeek='").append(creationDayOfWeek).append('\''); - sb.append(", status='").append(status).append('\''); - sb.append(", external=").append(external); - sb.append(", size=").append(size); - sb.append(", softwareName='").append(softwareName).append('\''); - sb.append(", softwareVersion='").append(softwareVersion).append('\''); - sb.append(", experimentTechnology='").append(experimentTechnology).append('\''); - sb.append(", experimentMethod='").append(experimentMethod).append('\''); - sb.append(", experimentNucleicAcidType='").append(experimentNucleicAcidType).append('\''); - sb.append(", experimentManufacturer='").append(experimentManufacturer).append('\''); - sb.append(", experimentPlatform='").append(experimentPlatform).append('\''); - sb.append(", experimentLibrary='").append(experimentLibrary).append('\''); - sb.append(", experimentCenter='").append(experimentCenter).append('\''); - sb.append(", experimentLab='").append(experimentLab).append('\''); - sb.append(", experimentResponsible='").append(experimentResponsible).append('\''); - sb.append(", tags=").append(tags); - sb.append(", numSamples=").append(numSamples); - sb.append(", numRelatedFiles=").append(numRelatedFiles); - sb.append(", acl=").append(acl); - sb.append(", annotationSets=").append(annotationSets); - sb.append(", annotations=").append(annotations); - sb.append('}'); - return sb.toString(); - } - - public String getName() { - return name; - } - - public FileSolrModel setName(String name) { - this.name = name; - return this; - } - - public String getType() { - return type; - } - - public FileSolrModel setType(String type) { - this.type = type; - return this; - } - - public String getFormat() { - return format; - } - - public FileSolrModel setFormat(String format) { - this.format = format; - return this; - } - - public String getBioformat() { - return bioformat; - } - - public FileSolrModel setBioformat(String bioformat) { - this.bioformat = bioformat; - return this; - } - - public boolean isExternal() { - return external; - } - - public FileSolrModel setExternal(boolean external) { - this.external = external; - return this; - } - - public long getSize() { - return size; - } - - public FileSolrModel setSize(long size) { - this.size = size; - return this; - } - - public String getSoftwareName() { - return softwareName; - } - - public FileSolrModel setSoftwareName(String softwareName) { - this.softwareName = softwareName; - return this; - } - - public String getSoftwareVersion() { - return softwareVersion; - } - - public FileSolrModel setSoftwareVersion(String softwareVersion) { - this.softwareVersion = softwareVersion; - return this; - } - - public String getExperimentTechnology() { - return experimentTechnology; - } - - public FileSolrModel setExperimentTechnology(String experimentTechnology) { - this.experimentTechnology = experimentTechnology; - return this; - } - - public String getExperimentMethod() { - return experimentMethod; - } - - public FileSolrModel setExperimentMethod(String experimentMethod) { - this.experimentMethod = experimentMethod; - return this; - } - - public String getExperimentNucleicAcidType() { - return experimentNucleicAcidType; - } - - public FileSolrModel setExperimentNucleicAcidType(String experimentNucleicAcidType) { - this.experimentNucleicAcidType = experimentNucleicAcidType; - return this; - } - - public String getExperimentManufacturer() { - return experimentManufacturer; - } - - public FileSolrModel setExperimentManufacturer(String experimentManufacturer) { - this.experimentManufacturer = experimentManufacturer; - return this; - } - - public String getExperimentPlatform() { - return experimentPlatform; - } - - public FileSolrModel setExperimentPlatform(String experimentPlatform) { - this.experimentPlatform = experimentPlatform; - return this; - } - - public String getExperimentLibrary() { - return experimentLibrary; - } - - public FileSolrModel setExperimentLibrary(String experimentLibrary) { - this.experimentLibrary = experimentLibrary; - return this; - } - - public String getExperimentCenter() { - return experimentCenter; - } - - public FileSolrModel setExperimentCenter(String experimentCenter) { - this.experimentCenter = experimentCenter; - return this; - } - - public String getExperimentLab() { - return experimentLab; - } - - public FileSolrModel setExperimentLab(String experimentLab) { - this.experimentLab = experimentLab; - return this; - } - - public String getExperimentResponsible() { - return experimentResponsible; - } - - public FileSolrModel setExperimentResponsible(String experimentResponsible) { - this.experimentResponsible = experimentResponsible; - return this; - } - - public List getTags() { - return tags; - } - - public FileSolrModel setTags(List tags) { - this.tags = tags; - return this; - } - - public int getNumSamples() { - return numSamples; - } - - public FileSolrModel setNumSamples(int numSamples) { - this.numSamples = numSamples; - return this; - } - - public int getNumRelatedFiles() { - return numRelatedFiles; - } - - public FileSolrModel setNumRelatedFiles(int numRelatedFiles) { - this.numRelatedFiles = numRelatedFiles; - return this; - } - - public List getAnnotationSets() { - return annotationSets; - } - - public FileSolrModel setAnnotationSets(List annotationSets) { - this.annotationSets = annotationSets; - return this; - } - - public Map getAnnotations() { - return annotations; - } - - public FileSolrModel setAnnotations(Map annotations) { - this.annotations = annotations; - return this; - } -} - - diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/IndividualSolrModel.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/IndividualSolrModel.java deleted file mode 100644 index d5b8b63876b..00000000000 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/IndividualSolrModel.java +++ /dev/null @@ -1,314 +0,0 @@ -/* - * Copyright 2015-2020 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.catalog.stats.solr; - -import org.apache.solr.client.solrj.beans.Field; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Created by wasim on 27/06/18. - */ -public class IndividualSolrModel extends CatalogSolrModel { - - @Field - private boolean hasFather; - - @Field - private boolean hasMother; - - @Field - private String locationCity; - - @Field - private String locationState; - - @Field - private String locationCountry; - - @Field - private int yearOfBirth; - - @Field - private String monthOfBirth; - - @Field - private int dayOfBirth; - - @Field - private String sex; - - @Field - private String karyotypicSex; - - @Field - private String ethnicity; - - @Field - private String population; - - @Field - private int version; - - @Field - private String lifeStatus; - - @Field - private List phenotypes; - - @Field - private List disorders; - - @Field - private int numSamples; - - @Field - private boolean parentalConsanguinity; - - @Field - private List annotationSets; - - @Field("annotations__*") - private Map annotations; - - public IndividualSolrModel() { - this.annotationSets = new ArrayList<>(); - this.phenotypes = new ArrayList<>(); - this.disorders = new ArrayList<>(); - this.annotations = new HashMap<>(); - } - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder("IndividualSolrModel{"); - sb.append("id='").append(id).append('\''); - sb.append(", uid=").append(uid); - sb.append(", studyId='").append(studyId).append('\''); - sb.append(", hasFather=").append(hasFather); - sb.append(", hasMother=").append(hasMother); - sb.append(", locationCity='").append(locationCity).append('\''); - sb.append(", locationState='").append(locationState).append('\''); - sb.append(", locationCountry='").append(locationCountry).append('\''); - sb.append(", yearOfBirth=").append(yearOfBirth); - sb.append(", monthOfBirth='").append(monthOfBirth).append('\''); - sb.append(", dayOfBirth=").append(dayOfBirth); - sb.append(", sex='").append(sex).append('\''); - sb.append(", karyotypicSex='").append(karyotypicSex).append('\''); - sb.append(", ethnicity='").append(ethnicity).append('\''); - sb.append(", population='").append(population).append('\''); - sb.append(", release=").append(release); - sb.append(", version=").append(version); - sb.append(", creationYear=").append(creationYear); - sb.append(", creationMonth='").append(creationMonth).append('\''); - sb.append(", creationDay=").append(creationDay); - sb.append(", creationDayOfWeek='").append(creationDayOfWeek).append('\''); - sb.append(", status='").append(status).append('\''); - sb.append(", lifeStatus='").append(lifeStatus).append('\''); - sb.append(", phenotypes=").append(phenotypes); - sb.append(", disorders=").append(disorders); - sb.append(", numSamples=").append(numSamples); - sb.append(", parentalConsanguinity=").append(parentalConsanguinity); - sb.append(", acl=").append(acl); - sb.append(", annotationSets=").append(annotationSets); - sb.append(", annotations=").append(annotations); - sb.append('}'); - return sb.toString(); - } - - public boolean isHasFather() { - return hasFather; - } - - public IndividualSolrModel setHasFather(boolean hasFather) { - this.hasFather = hasFather; - return this; - } - - public boolean isHasMother() { - return hasMother; - } - - public IndividualSolrModel setHasMother(boolean hasMother) { - this.hasMother = hasMother; - return this; - } - - public String getLocationCity() { - return locationCity; - } - - public IndividualSolrModel setLocationCity(String locationCity) { - this.locationCity = locationCity; - return this; - } - - public String getLocationState() { - return locationState; - } - - public IndividualSolrModel setLocationState(String locationState) { - this.locationState = locationState; - return this; - } - - public String getLocationCountry() { - return locationCountry; - } - - public IndividualSolrModel setLocationCountry(String locationCountry) { - this.locationCountry = locationCountry; - return this; - } - - public int getYearOfBirth() { - return yearOfBirth; - } - - public IndividualSolrModel setYearOfBirth(int yearOfBirth) { - this.yearOfBirth = yearOfBirth; - return this; - } - - public String getMonthOfBirth() { - return monthOfBirth; - } - - public IndividualSolrModel setMonthOfBirth(String monthOfBirth) { - this.monthOfBirth = monthOfBirth; - return this; - } - - public int getDayOfBirth() { - return dayOfBirth; - } - - public IndividualSolrModel setDayOfBirth(int dayOfBirth) { - this.dayOfBirth = dayOfBirth; - return this; - } - - public String getSex() { - return sex; - } - - public IndividualSolrModel setSex(String sex) { - this.sex = sex; - return this; - } - - public String getKaryotypicSex() { - return karyotypicSex; - } - - public IndividualSolrModel setKaryotypicSex(String karyotypicSex) { - this.karyotypicSex = karyotypicSex; - return this; - } - - public String getEthnicity() { - return ethnicity; - } - - public IndividualSolrModel setEthnicity(String ethnicity) { - this.ethnicity = ethnicity; - return this; - } - - public String getPopulation() { - return population; - } - - public IndividualSolrModel setPopulation(String population) { - this.population = population; - return this; - } - - public int getVersion() { - return version; - } - - public IndividualSolrModel setVersion(int version) { - this.version = version; - return this; - } - - public String getLifeStatus() { - return lifeStatus; - } - - public IndividualSolrModel setLifeStatus(String lifeStatus) { - this.lifeStatus = lifeStatus; - return this; - } - - public List getPhenotypes() { - return phenotypes; - } - - public IndividualSolrModel setPhenotypes(List phenotypes) { - this.phenotypes = phenotypes; - return this; - } - - public List getDisorders() { - return disorders; - } - - public IndividualSolrModel setDisorders(List disorders) { - this.disorders = disorders; - return this; - } - - public int getNumSamples() { - return numSamples; - } - - public IndividualSolrModel setNumSamples(int numSamples) { - this.numSamples = numSamples; - return this; - } - - public boolean isParentalConsanguinity() { - return parentalConsanguinity; - } - - public IndividualSolrModel setParentalConsanguinity(boolean parentalConsanguinity) { - this.parentalConsanguinity = parentalConsanguinity; - return this; - } - - public List getAnnotationSets() { - return annotationSets; - } - - public IndividualSolrModel setAnnotationSets(List annotationSets) { - this.annotationSets = annotationSets; - return this; - } - - public Map getAnnotations() { - return annotations; - } - - public IndividualSolrModel setAnnotations(Map annotations) { - this.annotations = annotations; - return this; - } -} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/JobSolrModel.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/JobSolrModel.java deleted file mode 100644 index 0f61ea3b385..00000000000 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/JobSolrModel.java +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Copyright 2015-2020 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.catalog.stats.solr; - -import org.apache.solr.client.solrj.beans.Field; - -import java.util.ArrayList; -import java.util.List; - -public class JobSolrModel extends CatalogSolrModel { - - @Field - private String toolId; - - @Field - private String toolScope; - - @Field - private String toolType; - - @Field - private String toolResource; - - @Field - private String userId; - - @Field - private String priority; - - @Field - private List tags; - - @Field - private String executorId; - - @Field - private String executorFramework; - - public JobSolrModel() { - this.tags = new ArrayList<>(); - } - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder("JobSolrModel{"); - sb.append("toolId='").append(toolId).append('\''); - sb.append(", toolScope='").append(toolScope).append('\''); - sb.append(", toolType='").append(toolType).append('\''); - sb.append(", toolResource='").append(toolResource).append('\''); - sb.append(", userId='").append(userId).append('\''); - sb.append(", priority='").append(priority).append('\''); - sb.append(", tags=").append(tags); - sb.append(", executorId='").append(executorId).append('\''); - sb.append(", executorFramework='").append(executorFramework).append('\''); - sb.append(", id='").append(id).append('\''); - sb.append(", uid=").append(uid); - sb.append(", studyId='").append(studyId).append('\''); - sb.append(", creationYear=").append(creationYear); - sb.append(", creationMonth='").append(creationMonth).append('\''); - sb.append(", creationDay=").append(creationDay); - sb.append(", creationDayOfWeek='").append(creationDayOfWeek).append('\''); - sb.append(", status='").append(status).append('\''); - sb.append(", release=").append(release); - sb.append(", acl=").append(acl); - sb.append('}'); - return sb.toString(); - } - - public String getToolId() { - return toolId; - } - - public JobSolrModel setToolId(String toolId) { - this.toolId = toolId; - return this; - } - - public String getToolScope() { - return toolScope; - } - - public JobSolrModel setToolScope(String toolScope) { - this.toolScope = toolScope; - return this; - } - - public String getToolType() { - return toolType; - } - - public JobSolrModel setToolType(String toolType) { - this.toolType = toolType; - return this; - } - - public String getToolResource() { - return toolResource; - } - - public JobSolrModel setToolResource(String toolResource) { - this.toolResource = toolResource; - return this; - } - - public String getUserId() { - return userId; - } - - public JobSolrModel setUserId(String userId) { - this.userId = userId; - return this; - } - - public String getPriority() { - return priority; - } - - public JobSolrModel setPriority(String priority) { - this.priority = priority; - return this; - } - - public List getTags() { - return tags; - } - - public JobSolrModel setTags(List tags) { - this.tags = tags; - return this; - } - - public String getExecutorId() { - return executorId; - } - - public JobSolrModel setExecutorId(String executorId) { - this.executorId = executorId; - return this; - } - - public String getExecutorFramework() { - return executorFramework; - } - - public JobSolrModel setExecutorFramework(String executorFramework) { - this.executorFramework = executorFramework; - return this; - } -} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/SampleSolrModel.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/SampleSolrModel.java deleted file mode 100644 index a13acee8de9..00000000000 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/SampleSolrModel.java +++ /dev/null @@ -1,213 +0,0 @@ -/* - * Copyright 2015-2020 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.catalog.stats.solr; - -import org.apache.solr.client.solrj.beans.Field; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Created by wasim on 27/06/18. - */ -public class SampleSolrModel extends CatalogSolrModel { - - @Field - private int version; - - @Field - private boolean somatic; - - @Field - @Deprecated - private String product; - - @Field - private String preparationMethod; - - @Field - private String extractionMethod; - - @Field - private String labSampleId; - - @Field - @Deprecated - private String tissue; - - @Field - @Deprecated - private String organ; - - @Field - private String method; - - @Field - private List phenotypes; - - @Field - private List annotationSets; - - @Field("annotations__*") - private Map annotations; - - public SampleSolrModel() { - this.annotationSets = new ArrayList<>(); - this.phenotypes = new ArrayList<>(); - this.annotations = new HashMap<>(); - } - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder("SampleSolrModel{"); - sb.append("id='").append(id).append('\''); - sb.append(", uid=").append(uid); - sb.append(", studyId='").append(studyId).append('\''); - sb.append(", release=").append(release); - sb.append(", version=").append(version); - sb.append(", creationYear=").append(creationYear); - sb.append(", creationMonth='").append(creationMonth).append('\''); - sb.append(", creationDay=").append(creationDay); - sb.append(", creationDayOfWeek='").append(creationDayOfWeek).append('\''); - sb.append(", status='").append(status).append('\''); - sb.append(", somatic=").append(somatic); - sb.append(", product='").append(product).append('\''); - sb.append(", preparationMethod='").append(preparationMethod).append('\''); - sb.append(", extractionMethod='").append(extractionMethod).append('\''); - sb.append(", labSampleId='").append(labSampleId).append('\''); - sb.append(", tissue='").append(tissue).append('\''); - sb.append(", organ='").append(organ).append('\''); - sb.append(", method='").append(method).append('\''); - sb.append(", phenotypes=").append(phenotypes); - sb.append(", acl=").append(acl); - sb.append(", annotationSets=").append(annotationSets); - sb.append(", annotations=").append(annotations); - sb.append('}'); - return sb.toString(); - } - - public int getVersion() { - return version; - } - - public SampleSolrModel setVersion(int version) { - this.version = version; - return this; - } - - public boolean isSomatic() { - return somatic; - } - - public SampleSolrModel setSomatic(boolean somatic) { - this.somatic = somatic; - return this; - } - - public String getProduct() { - return product; - } - - public SampleSolrModel setProduct(String product) { - this.product = product; - return this; - } - - public String getPreparationMethod() { - return preparationMethod; - } - - public SampleSolrModel setPreparationMethod(String preparationMethod) { - this.preparationMethod = preparationMethod; - return this; - } - - public String getExtractionMethod() { - return extractionMethod; - } - - public SampleSolrModel setExtractionMethod(String extractionMethod) { - this.extractionMethod = extractionMethod; - return this; - } - - public String getLabSampleId() { - return labSampleId; - } - - public SampleSolrModel setLabSampleId(String labSampleId) { - this.labSampleId = labSampleId; - return this; - } - - public String getTissue() { - return tissue; - } - - public SampleSolrModel setTissue(String tissue) { - this.tissue = tissue; - return this; - } - - public String getOrgan() { - return organ; - } - - public SampleSolrModel setOrgan(String organ) { - this.organ = organ; - return this; - } - - public String getMethod() { - return method; - } - - public SampleSolrModel setMethod(String method) { - this.method = method; - return this; - } - - public List getPhenotypes() { - return phenotypes; - } - - public SampleSolrModel setPhenotypes(List phenotypes) { - this.phenotypes = phenotypes; - return this; - } - - public List getAnnotationSets() { - return annotationSets; - } - - public SampleSolrModel setAnnotationSets(List annotationSets) { - this.annotationSets = annotationSets; - return this; - } - - public Map getAnnotations() { - return annotations; - } - - public SampleSolrModel setAnnotations(Map annotations) { - this.annotations = annotations; - return this; - } -} - diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/converters/CatalogCohortToSolrCohortConverter.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/converters/CatalogCohortToSolrCohortConverter.java deleted file mode 100644 index 133a28edb91..00000000000 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/converters/CatalogCohortToSolrCohortConverter.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright 2015-2020 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.catalog.stats.solr.converters; - -import org.apache.commons.lang3.NotImplementedException; -import org.opencb.commons.datastore.core.ComplexTypeConverter; -import org.opencb.commons.datastore.core.QueryParam; -import org.opencb.opencga.catalog.exceptions.CatalogDBException; -import org.opencb.opencga.catalog.stats.solr.CohortSolrModel; -import org.opencb.opencga.catalog.utils.AnnotationUtils; -import org.opencb.opencga.core.common.TimeUtils; -import org.opencb.opencga.core.models.cohort.Cohort; -import org.opencb.opencga.core.models.common.AnnotationSet; -import org.opencb.opencga.core.models.study.Study; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.time.LocalDate; -import java.time.ZoneId; -import java.util.*; -import java.util.stream.Collectors; - -/** - * Created by wasim on 03/07/18. - */ -public class CatalogCohortToSolrCohortConverter implements ComplexTypeConverter { - - private Study study; - private Map> variableMap; - - protected static Logger logger = LoggerFactory.getLogger(CatalogCohortToSolrCohortConverter.class); - - public CatalogCohortToSolrCohortConverter(Study study) { - this.study = study; - this.variableMap = new HashMap<>(); - if (this.study.getVariableSets() != null) { - this.study.getVariableSets().forEach(variableSet -> { - try { - this.variableMap.put(variableSet.getId(), AnnotationUtils.getVariableMap(variableSet)); - } catch (CatalogDBException e) { - logger.warn("Could not parse variableSet {}: {}", variableSet.getId(), e.getMessage(), e); - } - }); - } - } - - @Override - public Cohort convertToDataModelType(CohortSolrModel object) { - throw new NotImplementedException("Operation not supported"); - } - - @Override - public CohortSolrModel convertToStorageType(Cohort cohort) { - CohortSolrModel cohortSolrModel = new CohortSolrModel(); - - cohortSolrModel.setId(cohort.getUuid()); - cohortSolrModel.setUid(cohort.getUid()); - cohortSolrModel.setStudyId(study.getFqn().replace(":", "__")); - cohortSolrModel.setType(cohort.getType().name()); - - Date date = TimeUtils.toDate(cohort.getCreationDate()); - LocalDate localDate = date.toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); - - cohortSolrModel.setCreationYear(localDate.getYear()); - cohortSolrModel.setCreationMonth(localDate.getMonth().toString()); - cohortSolrModel.setCreationDay(localDate.getDayOfMonth()); - cohortSolrModel.setCreationDayOfWeek(localDate.getDayOfWeek().toString()); - cohortSolrModel.setStatus(cohort.getInternal().getStatus().getId()); - - if (cohort.getSamples() != null) { - cohortSolrModel.setNumSamples(cohort.getSamples().size()); - } else { - cohortSolrModel.setNumSamples(0); - } - - cohortSolrModel.setRelease(cohort.getRelease()); - cohortSolrModel.setAnnotations(SolrConverterUtil.populateAnnotations(variableMap, cohort.getAnnotationSets())); - - if (cohort.getAnnotationSets() != null) { - cohortSolrModel.setAnnotationSets(cohort.getAnnotationSets().stream().map(AnnotationSet::getId).collect(Collectors.toList())); - } else { - cohortSolrModel.setAnnotationSets(Collections.emptyList()); - } - - // Extract the permissions - Map> cohortAcl = - SolrConverterUtil.parseInternalOpenCGAAcls((List>) cohort.getAttributes().get("OPENCGA_ACL")); - List effectivePermissions = - SolrConverterUtil.getEffectivePermissions((Map>) study.getAttributes().get("OPENCGA_ACL"), cohortAcl, - "COHORT"); - cohortSolrModel.setAcl(effectivePermissions); - - return cohortSolrModel; - } -} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/converters/CatalogFamilyToSolrFamilyConverter.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/converters/CatalogFamilyToSolrFamilyConverter.java deleted file mode 100644 index 3159a8332c7..00000000000 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/converters/CatalogFamilyToSolrFamilyConverter.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright 2015-2020 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.catalog.stats.solr.converters; - -import org.apache.commons.lang3.NotImplementedException; -import org.opencb.commons.datastore.core.ComplexTypeConverter; -import org.opencb.commons.datastore.core.QueryParam; -import org.opencb.opencga.catalog.exceptions.CatalogDBException; -import org.opencb.opencga.catalog.stats.solr.FamilySolrModel; -import org.opencb.opencga.catalog.utils.AnnotationUtils; -import org.opencb.opencga.core.common.TimeUtils; -import org.opencb.opencga.core.models.common.AnnotationSet; -import org.opencb.opencga.core.models.family.Family; -import org.opencb.opencga.core.models.study.Study; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.time.LocalDate; -import java.time.ZoneId; -import java.util.*; -import java.util.stream.Collectors; - -/** - * Created by wasim on 04/07/18. - */ -public class CatalogFamilyToSolrFamilyConverter implements ComplexTypeConverter { - - private Study study; - private Map> variableMap; - - protected static Logger logger = LoggerFactory.getLogger(CatalogFamilyToSolrFamilyConverter.class); - - public CatalogFamilyToSolrFamilyConverter(Study study) { - this.study = study; - this.variableMap = new HashMap<>(); - if (this.study.getVariableSets() != null) { - this.study.getVariableSets().forEach(variableSet -> { - try { - this.variableMap.put(variableSet.getId(), AnnotationUtils.getVariableMap(variableSet)); - } catch (CatalogDBException e) { - logger.warn("Could not parse variableSet {}: {}", variableSet.getId(), e.getMessage(), e); - } - }); - } - } - - @Override - public Family convertToDataModelType(FamilySolrModel object) { - throw new NotImplementedException("Operation not supported"); - } - - @Override - public FamilySolrModel convertToStorageType(Family family) { - FamilySolrModel familySolrModel = new FamilySolrModel(); - - familySolrModel.setId(family.getUuid()); - familySolrModel.setUid(family.getUid()); - familySolrModel.setStudyId(study.getFqn().replace(":", "__")); - - Date date = TimeUtils.toDate(family.getCreationDate()); - LocalDate localDate = date.toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); - - familySolrModel.setCreationYear(localDate.getYear()); - familySolrModel.setCreationMonth(localDate.getMonth().toString()); - familySolrModel.setCreationDay(localDate.getDayOfMonth()); - familySolrModel.setCreationDayOfWeek(localDate.getDayOfWeek().toString()); - familySolrModel.setStatus(family.getInternal().getStatus().getId()); - - if (family.getInternal().getStatus() != null) { - familySolrModel.setStatus(family.getInternal().getStatus().getId()); - } - familySolrModel.setPhenotypes(SolrConverterUtil.populatePhenotypes(family.getPhenotypes())); - familySolrModel.setPhenotypes(SolrConverterUtil.populateDisorders(family.getDisorders())); - - familySolrModel.setNumMembers(family.getMembers() != null ? family.getMembers().size() : 0); - familySolrModel.setExpectedSize(family.getExpectedSize()); - - familySolrModel.setRelease(family.getRelease()); - familySolrModel.setVersion(family.getVersion()); - familySolrModel.setAnnotations(SolrConverterUtil.populateAnnotations(variableMap, family.getAnnotationSets())); - - if (family.getAnnotationSets() != null) { - familySolrModel.setAnnotationSets(family.getAnnotationSets().stream().map(AnnotationSet::getId).collect(Collectors.toList())); - } else { - familySolrModel.setAnnotationSets(Collections.emptyList()); - } - - // Extract the permissions - Map> familyAcl = - SolrConverterUtil.parseInternalOpenCGAAcls((List>) family.getAttributes().get("OPENCGA_ACL")); - List effectivePermissions = - SolrConverterUtil.getEffectivePermissions((Map>) study.getAttributes().get("OPENCGA_ACL"), familyAcl, - "FAMILY"); - familySolrModel.setAcl(effectivePermissions); - - return familySolrModel; - } -} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/converters/CatalogFileToSolrFileConverter.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/converters/CatalogFileToSolrFileConverter.java deleted file mode 100644 index 0e55064b97b..00000000000 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/converters/CatalogFileToSolrFileConverter.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Copyright 2015-2020 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.catalog.stats.solr.converters; - -import org.apache.commons.lang3.NotImplementedException; -import org.opencb.commons.datastore.core.ComplexTypeConverter; -import org.opencb.commons.datastore.core.QueryParam; -import org.opencb.opencga.catalog.exceptions.CatalogDBException; -import org.opencb.opencga.catalog.stats.solr.FileSolrModel; -import org.opencb.opencga.catalog.utils.AnnotationUtils; -import org.opencb.opencga.core.common.TimeUtils; -import org.opencb.opencga.core.models.common.AnnotationSet; -import org.opencb.opencga.core.models.file.File; -import org.opencb.opencga.core.models.study.Study; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.time.LocalDate; -import java.time.ZoneId; -import java.util.*; -import java.util.stream.Collectors; - -/** - * Created by wasim on 04/07/18. - */ -public class CatalogFileToSolrFileConverter implements ComplexTypeConverter { - - private Study study; - private Map> variableMap; - - protected static Logger logger = LoggerFactory.getLogger(CatalogFileToSolrFileConverter.class); - - public CatalogFileToSolrFileConverter(Study study) { - this.study = study; - this.variableMap = new HashMap<>(); - if (this.study.getVariableSets() != null) { - this.study.getVariableSets().forEach(variableSet -> { - try { - this.variableMap.put(variableSet.getId(), AnnotationUtils.getVariableMap(variableSet)); - } catch (CatalogDBException e) { - logger.warn("Could not parse variableSet {}: {}", variableSet.getId(), e.getMessage(), e); - } - }); - } - } - - @Override - public File convertToDataModelType(FileSolrModel object) { - throw new NotImplementedException("Operation not supported"); - } - - @Override - public FileSolrModel convertToStorageType(File file) { - FileSolrModel fileSolrModel = new FileSolrModel(); - - fileSolrModel.setId(file.getUuid()); - fileSolrModel.setUid(file.getUid()); - fileSolrModel.setName(file.getName()); - fileSolrModel.setStudyId(study.getFqn().replace(":", "__")); - fileSolrModel.setType(file.getType().name()); - if (file.getFormat() != null) { - fileSolrModel.setFormat(file.getFormat().name()); - } - if (file.getBioformat() != null) { - fileSolrModel.setBioformat(file.getBioformat().name()); - } - fileSolrModel.setRelease(file.getRelease()); - - Date date = TimeUtils.toDate(file.getCreationDate()); - LocalDate localDate = date.toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); - - fileSolrModel.setCreationYear(localDate.getYear()); - fileSolrModel.setCreationMonth(localDate.getMonth().toString()); - fileSolrModel.setCreationDay(localDate.getDayOfMonth()); - fileSolrModel.setCreationDayOfWeek(localDate.getDayOfWeek().toString()); - fileSolrModel.setStatus(file.getInternal().getStatus().getId()); - - fileSolrModel.setStatus(file.getInternal().getStatus().getId()); - fileSolrModel.setExternal(file.isExternal()); - fileSolrModel.setSize(file.getSize()); - if (file.getSoftware() != null) { - fileSolrModel.setSoftwareName(file.getSoftware().getName()); - fileSolrModel.setSoftwareVersion(file.getSoftware().getVersion()); - } - - fileSolrModel.setTags(file.getTags()); - - if (file.getExperiment() != null) { - fileSolrModel.setExperimentTechnology(file.getExperiment().getTechnology() != null - ? file.getExperiment().getTechnology().name() : null); - fileSolrModel.setExperimentMethod(file.getExperiment().getMethod() != null - ? file.getExperiment().getMethod().name() : null); - fileSolrModel.setExperimentNucleicAcidType(file.getExperiment().getNucleicAcidType() != null - ? file.getExperiment().getNucleicAcidType().name() : null); - fileSolrModel.setExperimentManufacturer(file.getExperiment().getManufacturer()); - fileSolrModel.setExperimentPlatform(file.getExperiment().getPlatform()); - fileSolrModel.setExperimentLibrary(file.getExperiment().getLibrary()); - fileSolrModel.setExperimentCenter(file.getExperiment().getCenter()); - fileSolrModel.setExperimentLab(file.getExperiment().getLab()); - fileSolrModel.setExperimentResponsible(file.getExperiment().getResponsible()); - } - - - fileSolrModel.setNumSamples(file.getSampleIds() != null ? file.getSampleIds().size() : 0); - fileSolrModel.setNumRelatedFiles(file.getRelatedFiles() != null ? file.getRelatedFiles().size() : 0); - - fileSolrModel.setAnnotations(SolrConverterUtil.populateAnnotations(variableMap, file.getAnnotationSets())); - if (file.getAnnotationSets() != null) { - fileSolrModel.setAnnotationSets(file.getAnnotationSets().stream().map(AnnotationSet::getId).collect(Collectors.toList())); - } else { - fileSolrModel.setAnnotationSets(Collections.emptyList()); - } - - // Extract the permissions - Map> fileAcl = - SolrConverterUtil.parseInternalOpenCGAAcls((List>) file.getAttributes().get("OPENCGA_ACL")); - List effectivePermissions = - SolrConverterUtil.getEffectivePermissions((Map>) study.getAttributes().get("OPENCGA_ACL"), fileAcl, - "FILE"); - fileSolrModel.setAcl(effectivePermissions); - - return fileSolrModel; - } -} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/converters/CatalogIndividualToSolrIndividualConverter.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/converters/CatalogIndividualToSolrIndividualConverter.java deleted file mode 100644 index b5dd29ffa3e..00000000000 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/converters/CatalogIndividualToSolrIndividualConverter.java +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Copyright 2015-2020 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.catalog.stats.solr.converters; - -import org.apache.commons.lang3.NotImplementedException; -import org.apache.commons.lang3.StringUtils; -import org.opencb.commons.datastore.core.ComplexTypeConverter; -import org.opencb.commons.datastore.core.QueryParam; -import org.opencb.opencga.catalog.exceptions.CatalogDBException; -import org.opencb.opencga.catalog.stats.solr.IndividualSolrModel; -import org.opencb.opencga.catalog.utils.AnnotationUtils; -import org.opencb.opencga.core.common.TimeUtils; -import org.opencb.opencga.core.models.common.AnnotationSet; -import org.opencb.opencga.core.models.individual.Individual; -import org.opencb.opencga.core.models.study.Study; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.time.LocalDate; -import java.time.ZoneId; -import java.util.*; -import java.util.stream.Collectors; - -/** - * Created by wasim on 04/07/18. - */ -public class CatalogIndividualToSolrIndividualConverter implements ComplexTypeConverter { - - private Study study; - private Map> variableMap; - - protected static Logger logger = LoggerFactory.getLogger(CatalogIndividualToSolrIndividualConverter.class); - - public CatalogIndividualToSolrIndividualConverter(Study study) { - this.study = study; - this.variableMap = new HashMap<>(); - if (this.study.getVariableSets() != null) { - this.study.getVariableSets().forEach(variableSet -> { - try { - this.variableMap.put(variableSet.getId(), AnnotationUtils.getVariableMap(variableSet)); - } catch (CatalogDBException e) { - logger.warn("Could not parse variableSet {}: {}", variableSet.getId(), e.getMessage(), e); - } - }); - } - } - - @Override - public Individual convertToDataModelType(IndividualSolrModel object) { - throw new NotImplementedException("Operation not supported"); - } - - @Override - public IndividualSolrModel convertToStorageType(Individual individual) { - IndividualSolrModel individualSolrModel = new IndividualSolrModel(); - - individualSolrModel.setId(individual.getUuid()); - individualSolrModel.setUid(individual.getUid()); - individualSolrModel.setStudyId(study.getFqn().replace(":", "__")); - - individualSolrModel.setHasFather(individual.getFather() != null && individual.getFather().getUid() > 0); - individualSolrModel.setHasMother(individual.getMother() != null && individual.getMother().getUid() > 0); - - if (individual.getSex() != null) { - individualSolrModel.setSex(individual.getSex().getSex().name()); - } - - if (individual.getKaryotypicSex() != null) { - individualSolrModel.setKaryotypicSex(individual.getKaryotypicSex().name()); - } - - individualSolrModel.setEthnicity(individual.getEthnicity().getId()); - - if (individual.getPopulation() != null) { - individualSolrModel.setPopulation(individual.getPopulation().getName()); - } - individualSolrModel.setRelease(individual.getRelease()); - individualSolrModel.setVersion(individual.getVersion()); - - Date date = TimeUtils.toDate(individual.getCreationDate()); - LocalDate localDate = date.toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); - - individualSolrModel.setCreationYear(localDate.getYear()); - individualSolrModel.setCreationMonth(localDate.getMonth().toString()); - individualSolrModel.setCreationDay(localDate.getDayOfMonth()); - individualSolrModel.setCreationDayOfWeek(localDate.getDayOfWeek().toString()); - individualSolrModel.setStatus(individual.getInternal().getStatus().getId()); - - if (individual.getInternal().getStatus() != null) { - individualSolrModel.setStatus(individual.getInternal().getStatus().getId()); - } - if (individual.getLifeStatus() != null) { - individualSolrModel.setLifeStatus(individual.getLifeStatus().name()); - } - if (individual.getLocation() != null) { - individualSolrModel.setLocationCity(individual.getLocation().getCity()); - individualSolrModel.setLocationState(individual.getLocation().getState()); - individualSolrModel.setLocationCountry(individual.getLocation().getCountry()); - } - if (StringUtils.isNotEmpty(individual.getDateOfBirth())) { - date = TimeUtils.toDate(individual.getDateOfBirth()); - localDate = date.toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); - - individualSolrModel.setDayOfBirth(localDate.getDayOfMonth()); - individualSolrModel.setMonthOfBirth(localDate.getMonth().toString()); - individualSolrModel.setYearOfBirth(localDate.getYear()); - } - individualSolrModel.setPhenotypes(SolrConverterUtil.populatePhenotypes(individual.getPhenotypes())); - individualSolrModel.setDisorders(SolrConverterUtil.populateDisorders(individual.getDisorders())); - - individualSolrModel.setNumSamples(individual.getSamples() != null ? individual.getSamples().size() : 0); - - individualSolrModel.setParentalConsanguinity(individual.isParentalConsanguinity()); - individualSolrModel.setAnnotations(SolrConverterUtil.populateAnnotations(variableMap, individual.getAnnotationSets())); - - if (individual.getAnnotationSets() != null) { - individualSolrModel.setAnnotationSets( - individual.getAnnotationSets().stream().map(AnnotationSet::getId).collect(Collectors.toList())); - } else { - individualSolrModel.setAnnotationSets(Collections.emptyList()); - } - - // Extract the permissions - Map> individualAcl = - SolrConverterUtil.parseInternalOpenCGAAcls((List>) individual.getAttributes().get("OPENCGA_ACL")); - List effectivePermissions = - SolrConverterUtil.getEffectivePermissions((Map>) study.getAttributes().get("OPENCGA_ACL"), - individualAcl, "INDIVIDUAL"); - individualSolrModel.setAcl(effectivePermissions); - - return individualSolrModel; - } -} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/converters/CatalogSampleToSolrSampleConverter.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/converters/CatalogSampleToSolrSampleConverter.java deleted file mode 100644 index f2ff34f900a..00000000000 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/converters/CatalogSampleToSolrSampleConverter.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright 2015-2020 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.catalog.stats.solr.converters; - -import org.apache.commons.lang3.NotImplementedException; -import org.apache.commons.lang3.StringUtils; -import org.opencb.commons.datastore.core.ComplexTypeConverter; -import org.opencb.commons.datastore.core.QueryParam; -import org.opencb.opencga.catalog.exceptions.CatalogDBException; -import org.opencb.opencga.catalog.stats.solr.SampleSolrModel; -import org.opencb.opencga.catalog.utils.AnnotationUtils; -import org.opencb.opencga.core.common.TimeUtils; -import org.opencb.opencga.core.models.common.AnnotationSet; -import org.opencb.opencga.core.models.sample.Sample; -import org.opencb.opencga.core.models.study.Study; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.time.LocalDate; -import java.time.ZoneId; -import java.util.*; -import java.util.stream.Collectors; - -/** - * Created by wasim on 27/06/18. - */ -public class CatalogSampleToSolrSampleConverter implements ComplexTypeConverter { - - private Study study; - private Map> variableMap; - - protected static Logger logger = LoggerFactory.getLogger(CatalogSampleToSolrSampleConverter.class); - - public CatalogSampleToSolrSampleConverter(Study study) { - this.study = study; - this.variableMap = new HashMap<>(); - if (this.study.getVariableSets() != null) { - this.study.getVariableSets().forEach(variableSet -> { - try { - this.variableMap.put(variableSet.getId(), AnnotationUtils.getVariableMap(variableSet)); - } catch (CatalogDBException e) { - logger.warn("Could not parse variableSet {}: {}", variableSet.getId(), e.getMessage(), e); - } - }); - } - } - - @Override - public Sample convertToDataModelType(SampleSolrModel sampleSolrModel) { - throw new NotImplementedException("Operation not supported"); - } - - @Override - public SampleSolrModel convertToStorageType(Sample sample) { - SampleSolrModel sampleSolrModel = new SampleSolrModel(); - - sampleSolrModel.setId(sample.getUuid()); - sampleSolrModel.setUid(sample.getUid()); - sampleSolrModel.setStudyId(study.getFqn().replace(":", "__")); - - sampleSolrModel.setRelease(sample.getRelease()); - sampleSolrModel.setVersion(sample.getVersion()); - - Date date = TimeUtils.toDate(sample.getCreationDate()); - LocalDate localDate = date.toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); - - sampleSolrModel.setCreationYear(localDate.getYear()); - sampleSolrModel.setCreationMonth(localDate.getMonth().toString()); - sampleSolrModel.setCreationDay(localDate.getDayOfMonth()); - sampleSolrModel.setCreationDayOfWeek(localDate.getDayOfWeek().toString()); - sampleSolrModel.setStatus(sample.getInternal().getStatus().getId()); - sampleSolrModel.setSomatic(sample.isSomatic()); - - if (sample.getPhenotypes() != null) { - sampleSolrModel.setPhenotypes(SolrConverterUtil.populatePhenotypes(sample.getPhenotypes())); - } - - sampleSolrModel.setAnnotations(SolrConverterUtil.populateAnnotations(variableMap, sample.getAnnotationSets())); - if (sample.getAnnotationSets() != null) { - sampleSolrModel.setAnnotationSets(sample.getAnnotationSets().stream().map(AnnotationSet::getId).collect(Collectors.toList())); - } else { - sampleSolrModel.setAnnotationSets(Collections.emptyList()); - } - - if (sample.getCollection() != null) { -// sampleSolrModel.setTissue(StringUtils.defaultIfEmpty(sample.getCollection().getTissue(), "")); -// sampleSolrModel.setOrgan(StringUtils.defaultIfEmpty(sample.getCollection().getOrgan(), "")); - sampleSolrModel.setMethod(StringUtils.defaultIfEmpty(sample.getCollection().getMethod(), "")); - } - - if (sample.getProcessing() != null) { - sampleSolrModel.setProduct(sample.getProcessing().getProduct() != null - ? sample.getProcessing().getProduct().getId() - : ""); - sampleSolrModel.setPreparationMethod(StringUtils.defaultIfEmpty(sample.getProcessing().getPreparationMethod(), "")); - sampleSolrModel.setExtractionMethod(StringUtils.defaultIfEmpty(sample.getProcessing().getExtractionMethod(), "")); - sampleSolrModel.setLabSampleId(StringUtils.defaultIfEmpty(sample.getProcessing().getLabSampleId(), "")); - } - - // Extract the permissions - Map> sampleAcl = - SolrConverterUtil.parseInternalOpenCGAAcls((List>) sample.getAttributes().get("OPENCGA_ACL")); - List effectivePermissions = - SolrConverterUtil.getEffectivePermissions((Map>) study.getAttributes().get("OPENCGA_ACL"), sampleAcl, - "SAMPLE"); - sampleSolrModel.setAcl(effectivePermissions); - - return sampleSolrModel; - } - -} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/converters/JobSolrConverter.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/converters/JobSolrConverter.java deleted file mode 100644 index d05b8c1c06d..00000000000 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/converters/JobSolrConverter.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright 2015-2020 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.catalog.stats.solr.converters; - -import org.apache.commons.lang3.NotImplementedException; -import org.opencb.commons.datastore.core.ComplexTypeConverter; -import org.opencb.opencga.catalog.stats.solr.JobSolrModel; -import org.opencb.opencga.core.common.TimeUtils; -import org.opencb.opencga.core.models.common.Enums; -import org.opencb.opencga.core.models.job.Job; -import org.opencb.opencga.core.models.study.Study; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.time.LocalDate; -import java.time.ZoneId; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.Set; - -public class JobSolrConverter implements ComplexTypeConverter { - - private Study study; - - protected static Logger logger= LoggerFactory.getLogger(JobSolrConverter.class); - - public JobSolrConverter(Study study) { - this.study=study; - } - - @Override - public Job convertToDataModelType(JobSolrModel jobSolrModel) { - throw new NotImplementedException("Operation not supported"); - } - - @Override - public JobSolrModel convertToStorageType(Job job) { - JobSolrModel jobSolrModel =new JobSolrModel(); - - jobSolrModel.setId(job.getUuid()); - jobSolrModel.setUid(job.getUid()); - jobSolrModel.setStudyId(study.getFqn().replace(":", "__")); - - jobSolrModel.setRelease(job.getRelease()); - - Date date= TimeUtils.toDate(job.getCreationDate()); - LocalDate localDate=date.toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); - - jobSolrModel.setCreationYear(localDate.getYear()); - jobSolrModel.setCreationMonth(localDate.getMonth().toString()); - jobSolrModel.setCreationDay(localDate.getDayOfMonth()); - jobSolrModel.setCreationDayOfWeek(localDate.getDayOfWeek().toString()); - jobSolrModel.setStatus(job.getInternal().getStatus().getId()); - - if (job.getTool() != null) { - jobSolrModel.setToolId(job.getTool().getId()); - jobSolrModel.setToolScope(job.getTool().getScope() != null ? job.getTool().getScope().name() : null); - jobSolrModel.setToolType(job.getTool().getType() != null ? job.getTool().getType().name() : null); - jobSolrModel.setToolResource(job.getTool().getResource() != null ? job.getTool().getResource().name() : null); - } - - jobSolrModel.setUserId(job.getUserId()); - jobSolrModel.setPriority(job.getPriority() != null ? job.getPriority().name() : null); - jobSolrModel.setTags(job.getTags()); - - if (job.getExecution() != null && job.getExecution().getExecutor() != null) { - jobSolrModel.setExecutorId(job.getExecution().getExecutor().getId()); - jobSolrModel.setExecutorFramework(job.getExecution().getExecutor().getFramework() != null - ? job.getExecution().getExecutor().getFramework().name() : null); - } - - // Extract the permissions - Map> jobAcl = - SolrConverterUtil.parseInternalOpenCGAAcls((List>)job.getAttributes().get("OPENCGA_ACL")); - List effectivePermissions = - SolrConverterUtil.getEffectivePermissions((Map>)study.getAttributes().get("OPENCGA_ACL"), jobAcl, - Enums.Resource.JOB.name()); - jobSolrModel.setAcl(effectivePermissions); - - return jobSolrModel; - } -} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/converters/SolrConverterUtil.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/converters/SolrConverterUtil.java deleted file mode 100644 index 3eda19dc793..00000000000 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/converters/SolrConverterUtil.java +++ /dev/null @@ -1,185 +0,0 @@ -/* - * Copyright 2015-2020 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.catalog.stats.solr.converters; - -import org.apache.commons.collections4.map.HashedMap; -import org.opencb.biodata.models.clinical.Disorder; -import org.opencb.biodata.models.clinical.Phenotype; -import org.opencb.commons.datastore.core.QueryParam; -import org.opencb.opencga.core.models.common.AnnotationSet; - -import java.util.*; - -/** - * Created by wasim on 03/07/18. - */ -public class SolrConverterUtil { - - - public static Map populateAnnotations(Map> variableTypeMap, - List annotationSets) { - Map result = new HashedMap(); - if (annotationSets != null) { - for (AnnotationSet annotationSet : annotationSets) { - Map typeMap = variableTypeMap.get(annotationSet.getVariableSetId()); - - for (String annotationKey : annotationSet.getAnnotations().keySet()) { - Object value = annotationSet.getAnnotations().get(annotationKey); - if (typeMap.containsKey(annotationKey)) { - result.put("annotations" + type(typeMap.get(annotationKey)) + annotationSet.getVariableSetId() + "." - + annotationKey, value); - } else { - // Dynamic annotation - String dynamicKey = annotationKey.substring(0, annotationKey.lastIndexOf(".") + 1) + "*"; - if (!typeMap.containsKey(dynamicKey)) { - // TODO: This condition should be removed in 3.0. This is here because of a bug that happened annotating - // TODO: This should be completely fixed and removed in future releases - continue; - } - result.put("annotations" + type(typeMap.get(dynamicKey)) + annotationSet.getVariableSetId() + "." + annotationKey, - value); - } - } - } - } - return result; - } - - public static List populatePhenotypes(List phenotypes) { - Set phenotypesIds = new HashSet<>(); - if (phenotypes != null) { - for (Phenotype phenotype : phenotypes) { - phenotypesIds.add(phenotype.getId()); - phenotypesIds.add(phenotype.getName()); - } - } - return new ArrayList(phenotypesIds); - } - - public static List populateDisorders(List disorders) { - Set disorderIds = new HashSet<>(); - if (disorders != null) { - for (Disorder disorder : disorders) { - disorderIds.add(disorder.getId()); - disorderIds.add(disorder.getName()); - } - } - return new ArrayList(disorderIds); - } - - public static String type(QueryParam.Type type) { - switch (type) { - case TEXT: - return "__s__"; - case TEXT_ARRAY: - return "__sm__"; - case INTEGER: - case LONG: - return "__i__"; - case INTEGER_ARRAY: - case LONG_ARRAY: - return "__im__"; - case DECIMAL: - return "__d__"; - case DECIMAL_ARRAY: - return "__dm__"; - case BOOLEAN: - return "__b__"; - case BOOLEAN_ARRAY: - return "__bm__"; - default: - return "__o__"; - } - } - - /** - * Parse internal OpenCGA ACLs. - * - * Given a List> in the form [{member: user1, permissions: [VIEW, UPDATE]}, {member: user2, permissions: [DELETE]}], - * return a Map> containing {user1: [VIEW, UPDATE], user2: [DELETE]} - * - * @param internalPermissions List containing the permissions for each member. - * @return a map of permissions per member. - */ - public static Map> parseInternalOpenCGAAcls(List> internalPermissions) { - if (internalPermissions == null) { - return new HashMap<>(); - } - Map> retPermissions = new HashMap<>(internalPermissions.size()); - - internalPermissions.forEach(aclEntry -> - retPermissions.put((String) aclEntry.get("member"), new HashSet<>((List) aclEntry.get("permissions"))) - ); - - return retPermissions; - } - - /** - * Get effective OpenCGA VIEW permissions. - * - * Given the map of permissions in the form {user1: [VIEW, UPDATE], user2: [DELETE]} for both the study and an entity, return the - * effective read only permissions in the way [user1_VIEW, user2_NONE] - * - * @param studyPermissions Map of permissions. - * @param entityPermissions Map of permissions. - * @param entity Entity name. - * @return a flattened list of effective VIEW permissions. - */ - public static List getEffectivePermissions(Map> studyPermissions, - Map> entityPermissions, String entity) { - if (studyPermissions == null) { - studyPermissions = new HashMap<>(); - } - // entityPermissions are already fine, but we should increase that list with the ones contained in the studyPermissions if they are - // not overrided by the entry permissions - Map> additionalPermissions = new HashMap<>(); - studyPermissions.forEach((key, value) -> { - if (!entityPermissions.containsKey(key)) { - additionalPermissions.put(key, value); - } - }); - - // FAMILY will need to become VIEW_FAMILIES, SAMPLE will be VIEW_SAMPLES - String viewEntry = "VIEW_" + (entity.endsWith("Y") ? entity.replace("Y", "IE") : entity) + "S"; - String viewAnnotation = "VIEW_" + entity + "_ANNOTATIONS"; - - List permissions = new ArrayList<>((additionalPermissions.size() + entityPermissions.size()) * 3); - addEffectivePermissions(entityPermissions, permissions, "VIEW", "VIEW_ANNOTATIONS"); - addEffectivePermissions(additionalPermissions, permissions, viewEntry, viewAnnotation); - - return permissions; - } - - private static void addEffectivePermissions(Map> allPermissions, List permissions, String viewPermission, - String viewAnnotationPermission) { - allPermissions.entrySet().forEach(aclEntry -> { - List currentPermissions = new ArrayList<>(2); - if (aclEntry.getValue().contains(viewPermission)) { - currentPermissions.add(aclEntry.getKey() + "__VIEW"); - } - if (aclEntry.getValue().contains(viewAnnotationPermission)) { - currentPermissions.add(aclEntry.getKey() + "__VIEW_ANNOTATIONS"); - } - - if (!currentPermissions.isEmpty()) { - permissions.addAll(currentPermissions); - } else { - permissions.add(aclEntry.getKey() + "__NONE"); - } - }); - } -} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/templates/TemplateManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/templates/TemplateManager.java index bb59584363b..d740f703ed7 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/templates/TemplateManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/templates/TemplateManager.java @@ -29,6 +29,7 @@ import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.common.GitRepositoryState; import org.opencb.opencga.core.models.AclEntry; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.clinical.ClinicalAnalysisUpdateParams; import org.opencb.opencga.core.models.cohort.CohortUpdateParams; import org.opencb.opencga.core.models.common.Enums; @@ -79,23 +80,24 @@ public void execute(TemplateManifest manifest, Path path) throws CatalogExceptio List projectIndexVcfJobIds = new ArrayList<>(); List statsJobIds = new ArrayList<>(); + String organizationId = manifest.getConfiguration().getOrganizationId(); TemplateStudy study = manifest.getStudy(); // NOTE: Do not change the order of the following resource creation. - String studyFqn = addStudyMetadata(manifest.getConfiguration().getProjectId(), study); + String studyFqn = addStudyMetadata(organizationId, manifest.getConfiguration().getProjectId(), study); - createIndividuals(studyFqn, path); - createSamples(studyFqn, path); - createCohorts(studyFqn, path); - createFamilies(studyFqn, path); - createPanels(studyFqn, path); - createClinicalAnalyses(studyFqn, path); + createIndividuals(organizationId, studyFqn, path); + createSamples(organizationId, studyFqn, path); + createCohorts(organizationId, studyFqn, path); + createFamilies(organizationId, studyFqn, path); + createPanels(organizationId, studyFqn, path); + createClinicalAnalyses(organizationId, studyFqn, path); // TODO: What is this? // if (study.getVariantEngineConfiguration() != null) { // configureVariantEngine(studyFqn, study); // } - createFiles(studyFqn, path); + createFiles(organizationId, studyFqn, path); // if (CollectionUtils.isNotEmpty(study.getFiles())) { // List studyIndexVcfJobIds = fetchFiles(template, studyFqn, study); // projectIndexVcfJobIds.addAll(studyIndexVcfJobIds); @@ -114,6 +116,9 @@ public void validate(TemplateManifest manifest) throws CatalogException { if (manifest.getConfiguration() == null) { throw new IllegalStateException("Missing 'configuration' section from the Manifest file"); } + if (StringUtils.isEmpty(manifest.getConfiguration().getOrganizationId())) { + throw new IllegalStateException("Missing 'configuration.organizationId' from the Manifest file"); + } if (StringUtils.isEmpty(manifest.getConfiguration().getProjectId())) { throw new IllegalStateException("Missing 'configuration.projectId' from the Manifest file"); } @@ -129,7 +134,7 @@ public void validate(TemplateManifest manifest) throws CatalogException { String version = gitRepositoryState.getBuildVersion(); String versionShort; if (version.contains("-")) { - logger.warn("Using development OpenCGA version: " + version); + logger.warn("Using development OpenCGA version: {}", version); versionShort = version.split("-")[0]; } else { versionShort = version; @@ -137,10 +142,12 @@ public void validate(TemplateManifest manifest) throws CatalogException { String templateVersion = manifest.getConfiguration().getVersion(); checkVersion(versionShort, templateVersion); + String organizationId = manifest.getConfiguration().getOrganizationId(); // Study should already exist - Study study = getStudy(manifest.getConfiguration().getProjectId(), manifest.getStudy().getId()); - String userId = catalogManager.getUserManager().getUserId(token); - catalogManager.getAuthorizationManager().checkIsOwnerOrAdmin(study.getUid(), userId); + Study study = getStudy(organizationId, manifest.getConfiguration().getProjectId(), manifest.getStudy().getId()); + JwtPayload payload = catalogManager.getUserManager().validateToken(token); + String userId = payload.getUserId(organizationId); + catalogManager.getAuthorizationManager().checkIsAtLeastStudyAdministrator(organizationId, study.getUid(), userId); // // Check if any study exists before we start, if a study exists we should fail. Projects are allowed to exist. // if (!resume && !overwrite) { @@ -166,17 +173,17 @@ private void checkVersion(String opencgaVersion, String templateVersion) { } } logger.warn("Using a template version lower than the OpenCGA installation version. Some things may not work properly. " - + "Template version: " + templateVersion + ", OpenCGA version: " + opencgaVersion); + + "Template version: {}, OpenCGA version: {}", templateVersion, opencgaVersion); } - private Study getStudy(String projectId, String studyId) throws CatalogException { + private Study getStudy(String organizationId, String projectId, String studyId) throws CatalogException { OpenCGAResult studyOpenCGAResult = catalogManager.getStudyManager().get(projectId + ":" + studyId, QueryOptions.empty(), token); return studyOpenCGAResult.first(); } - private String addStudyMetadata(String projectId, TemplateStudy tmplStudy) throws CatalogException { - Study origStudy = getStudy(projectId, tmplStudy.getId()); + private String addStudyMetadata(String organizationId, String projectId, TemplateStudy tmplStudy) throws CatalogException { + Study origStudy = getStudy(organizationId, projectId, tmplStudy.getId()); String fqn; // if (origStudy == null) { // Study study = new Study() @@ -261,7 +268,7 @@ private String addStudyMetadata(String projectId, TemplateStudy tmplStudy) throw // return openCGAClient.getUserId() + "@" + project.getId() + ":" + study.getId(); // } - private void createIndividuals(String studyFqn, Path path) throws CatalogException { + private void createIndividuals(String organizationId, String studyFqn, Path path) throws CatalogException { boolean hasParents = false; // Process/Create individuals without parents try (TemplateEntryIterator iterator = @@ -293,7 +300,8 @@ private void createIndividuals(String studyFqn, Path path) throws CatalogExcepti // Create individual logger.info("Create individual '{}'", individual.getId()); - catalogManager.getIndividualManager().create(studyFqn, individual.toIndividual(), QueryOptions.empty(), token); + catalogManager.getIndividualManager().create(studyFqn, individual.toIndividual(), QueryOptions.empty(), + token); count++; } else if (overwrite) { @@ -302,7 +310,8 @@ private void createIndividuals(String studyFqn, Path path) throws CatalogExcepti individual.setId(null); logger.info("Update individual '{}'", individual.getId()); - catalogManager.getIndividualManager().update(studyFqn, individualId, individual, QueryOptions.empty(), token); + catalogManager.getIndividualManager().update(studyFqn, individualId, individual, QueryOptions.empty(), + token); count++; } @@ -329,8 +338,8 @@ private void createIndividuals(String studyFqn, Path path) throws CatalogExcepti .setMother(individual.getMother()); logger.info("Updating individual '{}' parents", individual.getId()); - catalogManager.getIndividualManager().update(studyFqn, individual.getId(), updateParams, QueryOptions.empty(), - token); + catalogManager.getIndividualManager().update(studyFqn, individual.getId(), updateParams, + QueryOptions.empty(), token); count++; } @@ -423,7 +432,7 @@ private boolean hasParents(IndividualUpdateParams individual) { // } // } - private void createSamples(String studyFqn, Path path) throws CatalogException { + private void createSamples(String organizationId, String studyFqn, Path path) throws CatalogException { // Process/Create samples try (TemplateEntryIterator iterator = new TemplateEntryIterator<>(path, "samples", SampleUpdateParams.class)) { @@ -464,7 +473,7 @@ private void createSamples(String studyFqn, Path path) throws CatalogException { } } - private void createCohorts(String studyFqn, Path path) throws CatalogException { + private void createCohorts(String organizationId, String studyFqn, Path path) throws CatalogException { // Process/Create cohorts try (TemplateEntryIterator iterator = new TemplateEntryIterator<>(path, "cohorts", CohortUpdateParams.class)) { @@ -506,7 +515,7 @@ private void createCohorts(String studyFqn, Path path) throws CatalogException { } } - private void createFamilies(String studyFqn, Path path) throws CatalogException { + private void createFamilies(String organizationId, String studyFqn, Path path) throws CatalogException { // Process/Create families try (TemplateEntryIterator iterator = new TemplateEntryIterator<>(path, "families", FamilyUpdateParams.class)) { @@ -532,7 +541,8 @@ private void createFamilies(String studyFqn, Path path) throws CatalogException if (CollectionUtils.isNotEmpty(completeFamily.getMembers())) { List memberIds = completeFamily.getMembers().stream().map(Individual::getId).collect(Collectors.toList()); completeFamily.setMembers(null); - catalogManager.getFamilyManager().create(studyFqn, completeFamily, memberIds, QueryOptions.empty(), token); + catalogManager.getFamilyManager().create(studyFqn, completeFamily, memberIds, QueryOptions.empty(), + token); } else { catalogManager.getFamilyManager().create(studyFqn, family.toFamily(), QueryOptions.empty(), token); } @@ -555,7 +565,7 @@ private void createFamilies(String studyFqn, Path path) throws CatalogException } } - private void createPanels(String studyFqn, Path path) throws CatalogException { + private void createPanels(String organizationId, String studyFqn, Path path) throws CatalogException { // Process/Create panels try (TemplateEntryIterator iterator = new TemplateEntryIterator<>(path, "panels", PanelUpdateParams.class)) { @@ -597,7 +607,7 @@ private void createPanels(String studyFqn, Path path) throws CatalogException { } } - private void createClinicalAnalyses(String studyFqn, Path path) throws CatalogException { + private void createClinicalAnalyses(String organizationId, String studyFqn, Path path) throws CatalogException { // Process/Create Clinical Anlyses try (TemplateEntryIterator iterator = new TemplateEntryIterator<>(path, "clinical", ClinicalAnalysisUpdateParams.class)) { @@ -606,7 +616,8 @@ private void createClinicalAnalyses(String studyFqn, Path path) throws CatalogEx ClinicalAnalysisUpdateParams clinical = iterator.next(); Query query = new Query(ClinicalAnalysisDBAdaptor.QueryParams.ID.key(), clinical.getId()); - boolean exists = catalogManager.getClinicalAnalysisManager().count(studyFqn, query, token).getNumMatches() > 0; + boolean exists = catalogManager.getClinicalAnalysisManager().count(studyFqn, query, token) + .getNumMatches() > 0; if (exists && !resume) { throw new CatalogException("Clinical Analysis '" + clinical.getId() @@ -620,8 +631,8 @@ private void createClinicalAnalyses(String studyFqn, Path path) throws CatalogEx // Create Clinical Analysis logger.info("Create Clinical Analysis '{}'", clinical.getId()); - catalogManager.getClinicalAnalysisManager().create(studyFqn, clinical.toClinicalAnalysis(), QueryOptions.empty(), - token); + catalogManager.getClinicalAnalysisManager().create(studyFqn, clinical.toClinicalAnalysis(), + QueryOptions.empty(), token); count++; } else if (overwrite) { @@ -630,7 +641,8 @@ private void createClinicalAnalyses(String studyFqn, Path path) throws CatalogEx clinical.setId(null); logger.info("Update Clinical Analysis '{}'", clinical.getId()); - catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalId, clinical, QueryOptions.empty(), token); + catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalId, clinical, QueryOptions.empty(), + token); count++; } @@ -641,7 +653,7 @@ private void createClinicalAnalyses(String studyFqn, Path path) throws CatalogEx } } - private void createFiles(String studyFqn, Path path) throws CatalogException { + private void createFiles(String organizationId, String studyFqn, Path path) throws CatalogException { // Process/Create Files try (TemplateEntryIterator iterator = new TemplateEntryIterator<>(path, "files", TemplateFile.class)) { diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/templates/config/TemplateConfiguration.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/templates/config/TemplateConfiguration.java index 62de832f445..20e7b293eae 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/templates/config/TemplateConfiguration.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/templates/config/TemplateConfiguration.java @@ -21,6 +21,7 @@ public class TemplateConfiguration { private String version; private String baseUrl; private boolean index; + private String organizationId; private String projectId; @Override @@ -29,6 +30,7 @@ public String toString() { sb.append("version='").append(version).append('\''); sb.append(", baseUrl='").append(baseUrl).append('\''); sb.append(", index=").append(index); + sb.append(", organizationId='").append(organizationId).append('\''); sb.append(", projectId='").append(projectId).append('\''); sb.append('}'); return sb.toString(); @@ -58,6 +60,15 @@ public void setIndex(boolean index) { this.index = index; } + public String getOrganizationId() { + return organizationId; + } + + public TemplateConfiguration setOrganizationId(String organizationId) { + this.organizationId = organizationId; + return this; + } + public String getProjectId() { return projectId; } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/AnnotationUtils.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/AnnotationUtils.java index 30494174dfa..c636ae714ca 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/AnnotationUtils.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/AnnotationUtils.java @@ -79,10 +79,6 @@ public static void checkVariable(Variable variable) throws CatalogException { throw new CatalogException("Only variables with type \"OBJECT\" can define an internal variableSet"); } - if (variable.getType() == Variable.VariableType.TEXT) { - variable.setType(Variable.VariableType.STRING); - } - //Check default values switch (variable.getType()) { case BOOLEAN: @@ -198,6 +194,11 @@ public static void checkAnnotationSet(VariableSet variableSet, AnnotationSet ann } } + if (variableSet.isUnique() && StringUtils.isEmpty(annotationSet.getId())) { + // If no annotation set id is provided, replicate the variable set id + annotationSet.setId(variableSet.getId()); + } + //Get annotationSetName set and variableId map Set annotatedVariables = annotationSet.getAnnotations().entrySet() .stream() @@ -431,7 +432,7 @@ private static void checkAllowedValue(Variable variable, Object value, String me Map objectMap = (Map) object; checkAnnotationSet(new VariableSet(variable.getId(), variable.getId(), false, false, false, variable.getDescription(), variable.getVariables(), null, 1, null), - new AnnotationSet("", variable.getId(), objectMap, null, 1, null), null, true); + new AnnotationSet("", variable.getId(), objectMap), null, true); } } break; @@ -731,25 +732,28 @@ private static String fixQueryOptionAnnotation(List projectionList) { /** * Fixes any field that might be missing from the annotation built by the user so it is perfectly ready for the dbAdaptors to be parsed. * - * @param study study corresponding to the entry that is being queried. Study should contain the variableSets field filled in. - * @param query query object containing the annotation. + * @param organizationId Organization id. + * @param study study corresponding to the entry that is being queried. Study should contain the variableSets field filled in. + * @param query query object containing the annotation. * @throws CatalogException if there are unknown variables being queried, non-existing variable sets... */ - public static void fixQueryAnnotationSearch(Study study, Query query) throws CatalogException { - fixQueryAnnotationSearch(study, null, query, null); + public static void fixQueryAnnotationSearch(String organizationId, Study study, Query query) throws CatalogException { + fixQueryAnnotationSearch(organizationId, study, null, query, null); } /** * Fixes any field that might be missing from the annotation built by the user so it is perfectly ready for the dbAdaptors to be parsed. * - * @param study study corresponding to the entry that is being queried. Study should contain the variableSets field filled in. - * @param user for which the confidential permission should be checked. - * @param query query object containing the annotation. + * @param organizationId Organization id. + * @param study study corresponding to the entry that is being queried. Study should contain the variableSets field + * filled in. + * @param user for which the confidential permission should be checked. + * @param query query object containing the annotation. * @param authorizationManager Authorization manager to check for confidential permissions. If null, permissions won't be checked. * @throws CatalogException if there are unknown variables being queried, non-existing variable sets... */ - public static void fixQueryAnnotationSearch(Study study, String user, Query query, AuthorizationManager authorizationManager) - throws CatalogException { + public static void fixQueryAnnotationSearch(String organizationId, Study study, String user, Query query, + AuthorizationManager authorizationManager) throws CatalogException { if (query == null || query.isEmpty() || !query.containsKey(Constants.ANNOTATION)) { return; } @@ -796,7 +800,7 @@ public static void fixQueryAnnotationSearch(Study study, String user, Query quer if (authorizationManager != null && !confidentialPermissionChecked && variableSet.isConfidential()) { // We only check the confidential permission if needed once - authorizationManager.checkStudyPermission(study.getUid(), user, + authorizationManager.checkStudyPermission(organizationId, study.getUid(), user, StudyPermissions.Permissions.CONFIDENTIAL_VARIABLE_SET_ACCESS); confidentialPermissionChecked = true; } @@ -858,7 +862,7 @@ public static void fixQueryAnnotationSearch(Study study, String user, Query quer if (authorizationManager != null && !confidentialPermissionChecked && variableSetMap.get(variableSetString) .isConfidential()) { // We only check the confidential permission if needed once - authorizationManager.checkStudyPermission(study.getUid(), user, + authorizationManager.checkStudyPermission(organizationId, study.getUid(), user, StudyPermissions.Permissions.CONFIDENTIAL_VARIABLE_SET_ACCESS); confidentialPermissionChecked = true; } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/CatalogDemo.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/CatalogDemo.java index 91708265ca4..d56f598dabb 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/CatalogDemo.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/CatalogDemo.java @@ -17,12 +17,15 @@ package org.opencb.opencga.catalog.utils; import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.catalog.auth.authentication.JwtManager; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.CatalogManager; +import org.opencb.opencga.core.common.PasswordUtils; +import org.opencb.opencga.core.models.organizations.OrganizationCreateParams; +import org.opencb.opencga.core.models.organizations.OrganizationUpdateParams; import org.opencb.opencga.core.models.study.Group; import org.opencb.opencga.core.models.study.GroupUpdateParams; import org.opencb.opencga.core.models.study.StudyAclParams; -import org.opencb.opencga.core.models.user.Account; import java.io.IOException; import java.util.*; @@ -40,22 +43,33 @@ private CatalogDemo() { * Populates the database with dummy data. * * @param catalogManager Catalog manager instance. - * @param adminPassword Administrator password. - * @param force Used in the case where a database already exists with the same name. When force = true, it will override it. + * @param organizationId Organization id for the demo. + * @param adminPassword Administrator password. + * @param force Used in the case where a database already exists with the same name. When force = true, it will override it. * @throws CatalogException when there is already a database with the same name and force is false. */ - public static void createDemoDatabase(CatalogManager catalogManager, String adminPassword, boolean force) + public static void createDemoDatabase(CatalogManager catalogManager, String organizationId, String adminPassword, boolean force) throws CatalogException { - catalogManager.installCatalogDB(catalogManager.getConfiguration().getAdmin().getSecretKey(), adminPassword, "opencga@admin.com", - "", force); + catalogManager.installCatalogDB("HS256", PasswordUtils.getStrongRandomPassword(JwtManager.SECRET_KEY_MIN_LENGTH), adminPassword, + "opencga@admin.com", force); + String token = catalogManager.getUserManager().loginAsAdmin(adminPassword).getToken(); try { - populateDatabase(catalogManager); + populateDatabase(catalogManager, organizationId, token); } catch (IOException e) { throw new CatalogException(e.getMessage()); } } - private static void populateDatabase(CatalogManager catalogManager) throws CatalogException, IOException { + private static void populateDatabase(CatalogManager catalogManager, String organizationId, String opencgaToken) + throws CatalogException, IOException { + catalogManager.getOrganizationManager().create(new OrganizationCreateParams().setId(organizationId), QueryOptions.empty(), + opencgaToken); + catalogManager.getUserManager().create("owner", "owner", "owner@mail.com", "owner_pass", organizationId, 2000L, + null); + catalogManager.getOrganizationManager().update(organizationId, new OrganizationUpdateParams().setOwner("owner"), + QueryOptions.empty(), opencgaToken); + String ownerToken = catalogManager.getUserManager().login(organizationId, "owner", "owner_pass").getToken(); + // Create users Map userSessions = new HashMap<>(5); for (int i = 1; i <= 5; i++) { @@ -63,15 +77,17 @@ private static void populateDatabase(CatalogManager catalogManager) throws Catal String name = "User" + i; String password = id + "_pass"; String email = id + "@gmail.com"; - catalogManager.getUserManager().create(id, name, email, password, "organization", 2000L, Account.AccountType.FULL, null); - userSessions.put(id, catalogManager.getUserManager().login(id, password).getToken()); + catalogManager.getUserManager().create(id, name, email, password, organizationId, 2000L, + ownerToken); + userSessions.put(id, catalogManager.getUserManager().login(organizationId, id, password).getToken()); } // Create one project per user Map projects = new HashMap<>(5); for (Map.Entry userSession : userSessions.entrySet()) { - projects.put(userSession.getKey(), catalogManager.getProjectManager().create("default", "DefaultProject", "Description", - "Homo sapiens", null, "GrCh38", new QueryOptions(), userSession.getValue()).first().getFqn()); + projects.put(userSession.getKey(), catalogManager.getProjectManager() + .create("default", "DefaultProject", "Description", "Homo sapiens", null, "GrCh38", new QueryOptions(), + userSession.getValue()).first().getFqn()); } // Create two studies per user @@ -82,8 +98,8 @@ private static void populateDatabase(CatalogManager catalogManager) throws Catal for (int i = 1; i <= 2; i++) { String name = "Name of study" + i; String id = "study" + i; - studiesTmp.add(catalogManager.getStudyManager().create(projectId, id, id, name, "Description of " + id, null, - null, null, null, null, userSession.getValue()).first().getFqn()); + studiesTmp.add(catalogManager.getStudyManager().create(projectId, id, id, name, "Description of " + id, + null, null, null, null, null, userSession.getValue()).first().getFqn()); } studies.put(userSession.getKey(), studiesTmp); } @@ -98,14 +114,16 @@ private static void populateDatabase(CatalogManager catalogManager) throws Catal catalogManager.getStudyManager().updateGroup(studyId, "@admins", ParamUtils.BasicUpdateAction.ADD, new GroupUpdateParams(Collections.singletonList("user5")), userSessions.get("user1")); // user5 will add the rest of users. user2, user3 and user4 go to group "members" - catalogManager.getStudyManager().createGroup(studyId, new Group("analyst", Arrays.asList("user2", "user3", "user4")), - sessionId); + catalogManager.getStudyManager().createGroup(studyId, new Group("analyst", + Arrays.asList("user2", "user3", "user4")), sessionId); // // @members will have the role "analyst" StudyAclParams aclParams1 = new StudyAclParams("", "analyst"); - catalogManager.getStudyManager().updateAcl(Arrays.asList(studyId), "@analyst", aclParams1, ParamUtils.AclAction.ADD, sessionId); + catalogManager.getStudyManager().updateAcl(studyId, "@analyst", aclParams1, ParamUtils.AclAction.ADD, + sessionId); // // Add anonymous user to the role "denyAll". Later we will give it permissions to see some concrete samples. StudyAclParams aclParams = new StudyAclParams("", "locked"); - catalogManager.getStudyManager().updateAcl(Arrays.asList(studyId), "*", aclParams, ParamUtils.AclAction.ADD, sessionId); + catalogManager.getStudyManager().updateAcl(studyId, "*", aclParams, ParamUtils.AclAction.ADD, + sessionId); } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/CatalogFqn.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/CatalogFqn.java new file mode 100644 index 00000000000..4bd98981a83 --- /dev/null +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/CatalogFqn.java @@ -0,0 +1,188 @@ +package org.opencb.opencga.catalog.utils; + +import org.apache.commons.lang3.StringUtils; +import org.opencb.opencga.core.models.JwtPayload; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public final class CatalogFqn { + + public static final String PROJECT_FQN_FORMAT = "projectId:studyId"; + public static final String STUDY_FQN_FORMAT = "organizationId@projectId:studyId"; + + private String organizationId; + private String projectId; + private String studyId; + + private String projectUuid; + private String studyUuid; + + private String providedId; + + private static final String USER_PATTERN = "[A-Za-z][[-_.]?[A-Za-z0-9]?]*"; + private static final String PROJECT_PATTERN = "[A-Za-z0-9][[-_.]?[A-Za-z0-9]?]*"; + private static final String STUDY_PATTERN = "[A-Za-z0-9\\-_.]+|\\*"; + public static final Pattern ORGANIZATION_PROJECT_STUDY_PATTERN = Pattern.compile("^(" + USER_PATTERN + ")@(" + PROJECT_PATTERN + + "):(" + STUDY_PATTERN + ")$"); + public static final Pattern PROJECT_STUDY_PATTERN = Pattern.compile("^(" + PROJECT_PATTERN + "):(" + STUDY_PATTERN + ")$"); + + private CatalogFqn(String organizationId, String id) { + this.organizationId = organizationId; + this.providedId = id; + } + + public static CatalogFqn fromProjectFqn(String projectFqn) { + if (StringUtils.isEmpty(projectFqn)) { + throw new IllegalArgumentException("Missing project fqn"); + } + + if (UuidUtils.isOpenCgaUuid(projectFqn)) { + throw new IllegalArgumentException("Project fqn cannot be an OpenCGA uuid"); + } + + String[] split = projectFqn.split("@", 2); + if (split.length == 2) { + return new CatalogFqn(split[0], projectFqn).setProjectId(split[1]); + } else { + throw new IllegalArgumentException("Provided string '" + projectFqn + "' is not a valid project fqn."); + } + } + + public static CatalogFqn extractFqnFromProject(String projectStr, JwtPayload payload) { + if (StringUtils.isEmpty(projectStr)) { + return new CatalogFqn(payload.getOrganization(), projectStr); + } + + if (UuidUtils.isOpenCgaUuid(projectStr)) { + return new CatalogFqn(payload.getOrganization(), projectStr).setProjectUuid(projectStr); + } + + String[] split = projectStr.split("@"); + if (split.length == 2) { + return new CatalogFqn(split[0], projectStr) + .setProjectId(split[1]); + } else { + return new CatalogFqn(payload.getOrganization(), projectStr).setProjectId(projectStr); + } + } + + public static CatalogFqn extractFqnFromProjectFqn(String projectFqn) { + if (StringUtils.isEmpty(projectFqn)) { + throw new IllegalArgumentException("Missing project fqn"); + } + + String[] split = projectFqn.split("@"); + if (split.length == 2) { + return new CatalogFqn(split[0], projectFqn) + .setProjectId(split[1]); + } else { + throw new IllegalArgumentException("Provided string '" + projectFqn + "' is not a valid project fqn."); + } + } + + public static CatalogFqn extractFqnFromStudyFqn(String studyFqn) { + if (StringUtils.isEmpty(studyFqn)) { + throw new IllegalArgumentException("Missing study fqn"); + } + Matcher matcher = ORGANIZATION_PROJECT_STUDY_PATTERN.matcher(studyFqn); + if (matcher.find()) { + // studyStr contains the full path (organization@project:study) + String organizationId = matcher.group(1); + String projectId = matcher.group(2); + String studyId = matcher.group(3); + return new CatalogFqn(organizationId, studyFqn) + .setProjectId(projectId) + .setStudyId(studyId); + } else { + throw new IllegalArgumentException("Provided string '" + studyFqn + "' is not a valid study FQN. " + + "The accepted pattern is [" + STUDY_FQN_FORMAT + "]"); + } + } + + public static CatalogFqn extractFqnFromStudy(String studyStr, JwtPayload payload) { + if (StringUtils.isEmpty(studyStr)) { + return new CatalogFqn(payload.getOrganization(), studyStr); + } + + if (UuidUtils.isOpenCgaUuid(studyStr)) { + return new CatalogFqn(payload.getOrganization(), studyStr).setStudyUuid(studyStr); + } + + Matcher matcher = ORGANIZATION_PROJECT_STUDY_PATTERN.matcher(studyStr); + if (matcher.find()) { + // studyStr contains the full path (organization@project:study) + String organizationId = matcher.group(1); + String projectId = matcher.group(2); + String studyId = matcher.group(3); + return new CatalogFqn(organizationId, studyStr) + .setProjectId(projectId) + .setStudyId(studyId); + } else { + matcher = PROJECT_STUDY_PATTERN.matcher(studyStr); + if (matcher.find()) { + // studyStr contains the path (project:study) + String projectId = matcher.group(1); + String studyId = matcher.group(2); + return new CatalogFqn(payload.getOrganization(), studyStr) + .setProjectId(projectId) + .setStudyId(studyId); + } else { + // studyStr only contains the study information + return new CatalogFqn(payload.getOrganization(), studyStr).setStudyId(studyStr); + } + } + } + + public String getProvidedId() { + return providedId; + } + + public String getOrganizationId() { + return organizationId; + } + + public String getProjectId() { + return projectId; + } + + public CatalogFqn setProjectId(String projectId) { + this.projectId = projectId; + return this; + } + + public String getStudyId() { + return studyId; + } + + public CatalogFqn setStudyId(String studyId) { + this.studyId = studyId; + return this; + } + + public String getProjectUuid() { + return projectUuid; + } + + public CatalogFqn setProjectUuid(String projectUuid) { + this.projectUuid = projectUuid; + return this; + } + + public String getStudyUuid() { + return studyUuid; + } + + public CatalogFqn setStudyUuid(String studyUuid) { + this.studyUuid = studyUuid; + return this; + } + + public String toProjectFqn() { + // check if any of the fields is null + if (StringUtils.isEmpty(organizationId) || StringUtils.isEmpty(projectId)) { + throw new IllegalArgumentException("Missing organization or project"); + } + return organizationId + "@" + projectId; + } +} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/CatalogSampleAnnotationsLoader.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/CatalogSampleAnnotationsLoader.java index 25e618a4e68..b2c46c74dbe 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/CatalogSampleAnnotationsLoader.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/CatalogSampleAnnotationsLoader.java @@ -28,12 +28,12 @@ import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.AnnotationSetManager; import org.opencb.opencga.catalog.managers.CatalogManager; -import org.opencb.opencga.catalog.managers.FileUtils; import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.common.AnnotationSet; -import org.opencb.opencga.core.models.sample.SampleUpdateParams; import org.opencb.opencga.core.models.file.File; import org.opencb.opencga.core.models.sample.Sample; +import org.opencb.opencga.core.models.sample.SampleUpdateParams; import org.opencb.opencga.core.models.study.Study; import org.opencb.opencga.core.models.study.Variable; import org.opencb.opencga.core.models.study.VariableSet; @@ -48,7 +48,7 @@ */ public class CatalogSampleAnnotationsLoader { - private static Logger logger = LoggerFactory.getLogger(FileUtils.class); + private static Logger logger = LoggerFactory.getLogger(CatalogSampleAnnotationsLoader.class); private final CatalogManager catalogManager; public CatalogSampleAnnotationsLoader(CatalogManager catalogManager) { @@ -59,13 +59,17 @@ protected CatalogSampleAnnotationsLoader() { this.catalogManager = null; } - public DataResult loadSampleAnnotations(File pedFile, String variableSetId, String sessionId) throws CatalogException { + public DataResult loadSampleAnnotations(String studyStr, File pedFile, String variableSetId, String sessionId) + throws CatalogException { if (!pedFile.getFormat().equals(File.Format.PED)) { throw new CatalogException(pedFile.getUid() + " is not a pedigree file"); } + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(sessionId); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); - URI fileUri = catalogManager.getFileManager().getUri(pedFile); - Study study = catalogManager.getFileManager().getStudy(pedFile, sessionId); + URI fileUri = catalogManager.getFileManager().getUri(organizationId, pedFile); + Study study = catalogManager.getFileManager().getStudy(organizationId, pedFile, sessionId); long auxTime; long startTime = System.currentTimeMillis(); @@ -76,7 +80,8 @@ public DataResult loadSampleAnnotations(File pedFile, String variableSet //Take or infer the VariableSet VariableSet variableSet; if (variableSetId != null) { - variableSet = catalogManager.getStudyManager().getVariableSet(study.getFqn(), variableSetId, null, sessionId).first(); + variableSet = catalogManager.getStudyManager().getVariableSet(study.getFqn(), variableSetId, null, sessionId) + .first(); } else { variableSet = getVariableSetFromPedFile(ped); AnnotationUtils.checkVariableSet(variableSet); @@ -86,8 +91,7 @@ public DataResult loadSampleAnnotations(File pedFile, String variableSet for (Individual individual : ped.getIndividuals().values()) { Map annotation = getAnnotation(individual, sampleMap, variableSet, ped.getFields()); try { - AnnotationUtils.checkAnnotationSet(variableSet, new AnnotationSet("", variableSet.getId(), annotation, "", 1, - null), null, true); + AnnotationUtils.checkAnnotationSet(variableSet, new AnnotationSet("", variableSet.getId(), annotation), null, true); } catch (CatalogException e) { String message = "Validation with the variableSet {id: " + variableSetId + "} over ped File = {path: " + pedFile.getPath() + ", name: \"" + pedFile.getName() + "\"} failed"; @@ -105,8 +109,8 @@ public DataResult loadSampleAnnotations(File pedFile, String variableSet List variableList = new ArrayList<>(); variableList.addAll(variableSet.getVariables()); String name = pedFile.getName(); - variableSet = catalogManager.getStudyManager().createVariableSet(study.getFqn(), name, name, true, false, "Auto-generated " - + "VariableSet from File = {path: " + pedFile.getPath() + ", name: \"" + pedFile.getName() + "\"}", null, + variableSet = catalogManager.getStudyManager().createVariableSet(study.getFqn(), name, name, true, false, + "Auto-generated VariableSet from File = {path: " + pedFile.getPath() + ", name: \"" + pedFile.getName() + "\"}", null, variableList, Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), sessionId).getResults().get(0); variableSetId = variableSet.getId(); logger.debug("Added VariableSet = {id: {}} in {}ms", variableSetId, System.currentTimeMillis() - auxTime); @@ -115,7 +119,8 @@ public DataResult loadSampleAnnotations(File pedFile, String variableSet //Add Samples Query samplesQuery = new Query(SampleDBAdaptor.QueryParams.ID.key(), new LinkedList<>(ped.getIndividuals().keySet())); Map loadedSamples = new HashMap<>(); - for (Sample sample : catalogManager.getSampleManager().search(study.getFqn(), samplesQuery, null, sessionId).getResults()) { + for (Sample sample : catalogManager.getSampleManager().search(study.getFqn(), samplesQuery, null, sessionId) + .getResults()) { loadedSamples.put(sample.getId(), sample); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/Constants.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/Constants.java index 543502a6f87..8b1f4ac747e 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/Constants.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/Constants.java @@ -111,4 +111,6 @@ public class Constants { */ public static final String JOB_DELETED_OUTPUT_DIRECTORY = "deletedOutputFiles"; + public static final String DEFAULT_USER_EXPIRATION_DATE = "21000101000000"; + } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/FileMetadataReader.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/FileMetadataReader.java index 82bebabb5aa..afd8575662a 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/FileMetadataReader.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/FileMetadataReader.java @@ -106,7 +106,8 @@ public File updateMetadataInformation(String studyId, File file, String token) t }); if (!missingSamples.isEmpty()) { for (String missingSample : missingSamples) { - catalogManager.getSampleManager().create(studyId, new Sample().setId(missingSample), new QueryOptions(), token); + catalogManager.getSampleManager().create(studyId, new Sample().setId(missingSample), + new QueryOptions(), token); } } } @@ -133,7 +134,8 @@ public File updateMetadataInformation(String studyId, File file, String token) t progressLogger.increment(samplesBatch.size()); } - catalogManager.getFileManager().update(studyId, file.getUuid(), updateParams, QueryOptions.empty(), token); + catalogManager.getFileManager().update(studyId, file.getUuid(), updateParams, QueryOptions.empty(), + token); } else { catalogManager.getFileManager().update(studyId, file.getUuid(), updateParams, new QueryOptions(Constants.ACTIONS, diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/FileScanner.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/FileScanner.java index a9f81ecd312..831fb8a0a95 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/FileScanner.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/FileScanner.java @@ -73,18 +73,21 @@ public FileScanner(CatalogManager catalogManager) { * Set file status {@link FileStatus#MISSING} if the file (fileUri) is unreachable * Set file status to {@link FileStatus#READY} if was {@link FileStatus#MISSING} and file (fileUri) is reachable * + * @param organizationId Organization id. * @param study The study to check - * @param sessionId User sessionId * @param calculateChecksum Calculate checksum for "found files" + * @param sessionId User sessionId * @return found and lost files * @throws CatalogException if a Catalog error occurs */ - public List checkStudyFiles(Study study, boolean calculateChecksum, String sessionId) throws CatalogException { + public List checkStudyFiles(String organizationId, Study study, boolean calculateChecksum, String sessionId) + throws CatalogException { Query query = new Query(); query.put(FileDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), Arrays.asList( FileStatus.READY, FileStatus.MISSING, FileStatus.TRASHED)); - DBIterator iterator = catalogManager.getFileManager().iterator(study.getFqn(), query, new QueryOptions(), sessionId); + DBIterator iterator = catalogManager.getFileManager().iterator(study.getFqn(), query, new QueryOptions(), + sessionId); List modifiedFiles = new LinkedList<>(); while (iterator.hasNext()) { @@ -101,6 +104,7 @@ public List checkStudyFiles(Study study, boolean calculateChecksum, String /** * Scan the study folder, add all untracked files and check tracking. * + * @param organizationId Organization id. * @param study Study to resync * @param calculateChecksum Calculates checksum of all the files in the directory to scan * @param sessionId User sessionId @@ -108,19 +112,20 @@ public List checkStudyFiles(Study study, boolean calculateChecksum, String * @throws CatalogException if a Catalog error occurs * @throws IOException if an I/O error occurs */ - public List reSync(Study study, boolean calculateChecksum, String sessionId) throws CatalogException, IOException { + public List reSync(String organizationId, Study study, boolean calculateChecksum, String sessionId) + throws CatalogException, IOException { Query query = new Query(FileDBAdaptor.QueryParams.TYPE.key(), File.Type.DIRECTORY); DBIterator iterator = catalogManager.getFileManager().iterator(study.getFqn(), query, null, sessionId); List scan = new LinkedList<>(); while (iterator.hasNext()) { File folder = iterator.next(); - scan.addAll(scan(folder, catalogManager.getFileManager().getUri(folder), FileScannerPolicy.REPLACE, calculateChecksum, - false, sessionId)); + scan.addAll(scan(organizationId, folder, catalogManager.getFileManager().getUri(organizationId, folder), + FileScannerPolicy.REPLACE, calculateChecksum, false, sessionId)); } // TODO: Scan per file - scan.addAll(checkStudyFiles(study, calculateChecksum, sessionId)); + scan.addAll(checkStudyFiles(organizationId, study, calculateChecksum, sessionId)); return scan; } @@ -128,12 +133,13 @@ public List reSync(Study study, boolean calculateChecksum, String sessionI /** * Return all untracked files in a study folder. * - * @param study Study to scan - * @param sessionId User sessionId + * @param organizationId Organization id. + * @param study Study to scan + * @param sessionId User sessionId * @return Untracked files * @throws CatalogException if a Catalog error occurs */ - public Map untrackedFiles(Study study, String sessionId) throws CatalogException { + public Map untrackedFiles(String organizationId, Study study, String sessionId) throws CatalogException { long studyId = study.getUid(); URI studyUri = study.getUri(); @@ -181,6 +187,7 @@ public Map untrackedFiles(Study study, String sessionId) throws Cat /** * Scans the files inside the specified URI and adds to the provided directory. * + * @param organizationId Organization id. * @param directory Directory where add found files * @param directoryToScan Directory to scan * @param policy What to do when there is a file in the target path. See {@link FileScannerPolicy} @@ -191,15 +198,15 @@ public Map untrackedFiles(Study study, String sessionId) throws Cat * @throws IOException if an I/O error occurs * @throws CatalogException if a Catalog error occurs */ - public List scan(File directory, URI directoryToScan, FileScannerPolicy policy, - boolean calculateChecksum, boolean deleteSource, String sessionId) - throws IOException, CatalogException { - return scan(directory, directoryToScan, policy, calculateChecksum, deleteSource, uri -> true, sessionId); + public List scan(String organizationId, File directory, URI directoryToScan, FileScannerPolicy policy, + boolean calculateChecksum, boolean deleteSource, String sessionId) throws IOException, CatalogException { + return scan(organizationId, directory, directoryToScan, policy, calculateChecksum, deleteSource, uri -> true, sessionId); } /** * Scans the files inside the specified URI and adds to the provided directory. * + * @param organizationId Organization id. * @param directory Directory where add found files * @param directoryToScan Directory to scan * @param policy What to do when there is a file in the target path. See {@link FileScannerPolicy} @@ -211,13 +218,13 @@ public List scan(File directory, URI directoryToScan, FileScannerPolicy po * @throws IOException if an I/O error occurs * @throws CatalogException if a Catalog error occurs */ - public List scan(File directory, URI directoryToScan, FileScannerPolicy policy, boolean calculateChecksum, boolean deleteSource, - Predicate filter, String sessionId) throws CatalogException, IOException { + public List scan(String organizationId, File directory, URI directoryToScan, FileScannerPolicy policy, boolean calculateChecksum, + boolean deleteSource, Predicate filter, String sessionId) throws CatalogException, IOException { if (filter == null) { filter = uri -> true; } if (directoryToScan == null) { - directoryToScan = catalogManager.getFileManager().getUri(directory); + directoryToScan = catalogManager.getFileManager().getUri(organizationId, directory); } if (!directoryToScan.getPath().endsWith("/")) { directoryToScan = URI.create(directoryToScan.toString() + "/"); @@ -225,7 +232,7 @@ public List scan(File directory, URI directoryToScan, FileScannerPolicy po if (!directory.getType().equals(File.Type.DIRECTORY)) { throw new CatalogException("Provided folder " + directory.getPath() + " is actually a file."); } - Study study = catalogManager.getFileManager().getStudy(directory, sessionId); + Study study = catalogManager.getFileManager().getStudy(organizationId, directory, sessionId); long createFilesTime = 0, uploadFilesTime = 0, metadataReadTime = 0; IOManager ioManager = catalogManager.getIoManagerFactory().get(directoryToScan); @@ -248,7 +255,8 @@ public List scan(File directory, URI directoryToScan, FileScannerPolicy po } Query query = new Query(FileDBAdaptor.QueryParams.PATH.key(), filePath); - DataResult searchFile = catalogManager.getFileManager().search(study.getFqn(), query, null, sessionId); + DataResult searchFile = catalogManager.getFileManager().search(study.getFqn(), query, null, + sessionId); File file = null; boolean overwrite = true; boolean returnFile = false; @@ -263,12 +271,12 @@ public List scan(File directory, URI directoryToScan, FileScannerPolicy po // Set the status of the file to PENDING DELETE FileUpdateParams updateParams = new FileUpdateParams() .setInternal(new SmallFileInternal(new FileStatus(FileStatus.PENDING_DELETE))); - catalogManager.getFileManager().update(study.getFqn(), tmpQuery, updateParams, QueryOptions.empty(), - sessionId); + catalogManager.getFileManager().update(study.getFqn(), tmpQuery, updateParams, + QueryOptions.empty(), sessionId); // Delete completely the file/folder ! - catalogManager.getFileManager().delete(study.getFqn(), tmpQuery, new QueryOptions(Constants.SKIP_TRASH, - true), sessionId); + catalogManager.getFileManager().delete(study.getFqn(), tmpQuery, + new QueryOptions(Constants.SKIP_TRASH, true), sessionId); overwrite = false; break; case REPLACE: @@ -287,14 +295,14 @@ public List scan(File directory, URI directoryToScan, FileScannerPolicy po if (file == null) { long start, end; if (uri.getPath().endsWith("/")) { - file = catalogManager.getFileManager().createFolder(study.getFqn(), Paths.get(filePath).toString(), true, - null, QueryOptions.empty(), sessionId).first(); + file = catalogManager.getFileManager().createFolder(study.getFqn(), + Paths.get(filePath).toString(), true, null, QueryOptions.empty(), sessionId).first(); } else { start = System.currentTimeMillis(); InputStream inputStream = new BufferedInputStream(new FileInputStream(new java.io.File(uri))); - file = catalogManager.getFileManager().upload(study.getFqn(), inputStream, new File().setPath(filePath), - overwrite, true, calculateChecksum, sessionId).first(); + file = catalogManager.getFileManager().upload(study.getFqn(), inputStream, + new File().setPath(filePath), overwrite, true, calculateChecksum, sessionId).first(); if (deleteSource) { ioManager.deleteFile(uri); } @@ -317,8 +325,8 @@ public List scan(File directory, URI directoryToScan, FileScannerPolicy po long start = System.currentTimeMillis(); InputStream inputStream = new FileInputStream(new java.io.File(uri)); - file = catalogManager.getFileManager().upload(study.getFqn(), inputStream, file, overwrite, true, - calculateChecksum, sessionId).first(); + file = catalogManager.getFileManager().upload(study.getFqn(), inputStream, file, overwrite, + true, calculateChecksum, sessionId).first(); long end = System.currentTimeMillis(); uploadFilesTime += end - start; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/FqnUtils.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/FqnUtils.java index d1976076184..0096eed8a9f 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/FqnUtils.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/FqnUtils.java @@ -7,7 +7,7 @@ public class FqnUtils { public static class FQN { - private final String user; + private final String organization; private final String project; private final String study; private final String projectFqn; @@ -16,12 +16,12 @@ public FQN(String fqn) { if (fqn == null || fqn.isEmpty()) { throw new IllegalArgumentException("Empty FQN"); } - int idx1 = fqn.indexOf(ParamConstants.USER_PROJECT_SEPARATOR); + int idx1 = fqn.indexOf(ParamConstants.ORGANIZATION_PROJECT_SEPARATOR); int idx2 = fqn.indexOf(ParamConstants.PROJECT_STUDY_SEPARATOR, idx1 + 1); if (idx1 < 0) { - throw new IllegalArgumentException("Invalid FQN. Expected @[:]. Got '" + fqn + "'"); + throw new IllegalArgumentException("Invalid FQN. Expected @[:]. Got '" + fqn + "'"); } - user = fqn.substring(0, idx1); + organization = fqn.substring(0, idx1); if (idx2 < 0) { project = fqn.substring(idx1 + 1); projectFqn = fqn; @@ -33,19 +33,19 @@ public FQN(String fqn) { } } - public FQN(String user, String project) { - this(user, project, null); + public FQN(String organization, String project) { + this(organization, project, null); } - public FQN(String user, String project, String study) { - this.user = Objects.requireNonNull(user); + public FQN(String organization, String project, String study) { + this.organization = Objects.requireNonNull(organization); this.project = Objects.requireNonNull(project); - projectFqn = user + ParamConstants.USER_PROJECT_SEPARATOR + project; + projectFqn = organization + ParamConstants.ORGANIZATION_PROJECT_SEPARATOR + project; this.study = study; } - public String getUser() { - return user; + public String getOrganization() { + return organization; } public String getProject() { @@ -62,8 +62,8 @@ public String getStudy() { @Override public String toString() { - final StringBuilder sb = new StringBuilder(user) - .append(ParamConstants.USER_PROJECT_SEPARATOR) + final StringBuilder sb = new StringBuilder(organization) + .append(ParamConstants.ORGANIZATION_PROJECT_SEPARATOR) .append(project); if (study != null) { sb.append(ParamConstants.PROJECT_STUDY_SEPARATOR) @@ -77,8 +77,8 @@ public static FQN parse(String fqn) { return new FQN(fqn); } - public static String getUser(String fqn) { - return new FQN(fqn).getUser(); + public static String getOrganization(String fqn) { + return new FQN(fqn).getOrganization(); } public static String getProject(String fqn) { @@ -93,12 +93,12 @@ public static String toProjectFqn(String fqn) { return new FQN(fqn).getProjectFqn(); } - public static String buildFqn(String user, String project) { - return new FQN(user, project).toString(); + public static String buildFqn(String organizationId, String project) { + return new FQN(organizationId, project).toString(); } - public static String buildFqn(String user, String project, String study) { - return new FQN(user, project, study).toString(); + public static String buildFqn(String organizationId, String project, String study) { + return new FQN(organizationId, project, study).toString(); } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/ParamUtils.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/ParamUtils.java index c39d111d8c7..5e93414f08d 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/ParamUtils.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/ParamUtils.java @@ -174,6 +174,22 @@ public static void checkDateFormat(String creationDate, String param) throws Cat } } + public static void checkDateIsNotExpired(String dateStr, String param) throws CatalogParameterException { + if (StringUtils.isEmpty(dateStr)) { + throw CatalogParameterException.isNull(param); + } else { + // Validate date can be parsed and has the proper format + Date date = TimeUtils.toDate(dateStr); + if (date == null || dateStr.length() != 14) { + throw new CatalogParameterException("Unexpected '" + param + "' format. Expected format is 'yyyyMMddHHmmss'"); + } + if (date.before(TimeUtils.getDate())) { + throw new CatalogParameterException("The date for '" + param + "' introduced is already expired. Please, introduce a valid" + + " date in the future."); + } + } + } + public static String checkDateOrGetCurrentDate(String date, String param) throws CatalogParameterException { if (StringUtils.isEmpty(date)) { return TimeUtils.getTime(); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/PedigreeGraphUtils.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/PedigreeGraphUtils.java index b56966a44e8..480781b0e21 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/PedigreeGraphUtils.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/PedigreeGraphUtils.java @@ -11,8 +11,12 @@ import org.opencb.opencga.core.models.family.PedigreeGraph; import org.opencb.opencga.core.models.individual.Individual; -import java.io.*; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.PrintWriter; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; @@ -85,11 +89,8 @@ public static String getB64Image(Path outDir) throws IOException { if (!imgFile.exists()) { throw new IOException("Pedigree graph image file (" + PEDIGREE_IMAGE_FILENAME + ") not found at " + outDir); } - - FileInputStream fileInputStreamReader = new FileInputStream(imgFile); - byte[] bytes = new byte[(int) imgFile.length()]; - fileInputStreamReader.read(bytes); - return new String(Base64.getEncoder().encode(bytes), StandardCharsets.UTF_8); + byte[] bytes = Files.readAllBytes(imgFile.toPath()); + return Base64.getEncoder().encodeToString(bytes); } public static Object getJsonPedigreeGraph(Path outDir) throws IOException { @@ -107,9 +108,7 @@ public static String getTsvPedigreeGraph(Path outDir) throws IOException { throw new IOException("Pedigree graph TSV file (" + PEDIGREE_TSV_FILENAME + ") not found at " + outDir); } - FileInputStream fileInputStreamReader = new FileInputStream(tsvFile); - byte[] bytes = new byte[(int) tsvFile.length()]; - fileInputStreamReader.read(bytes); + byte[] bytes = Files.readAllBytes(tsvFile.toPath()); return new String(bytes, StandardCharsets.UTF_8); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/UuidUtils.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/UuidUtils.java index fe30716cadd..8fcab95c4d2 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/UuidUtils.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/UuidUtils.java @@ -48,7 +48,9 @@ public enum Entity { JOB(8), CLINICAL(9), PANEL(10), - INTERPRETATION(11); + INTERPRETATION(11), + ORGANIZATION(12), + NOTES(13); private final int mask; diff --git a/opencga-catalog/src/main/resources/catalog-indexes.txt b/opencga-catalog/src/main/resources/catalog-indexes.txt index 4eeafe1f865..fe7f43506d5 100644 --- a/opencga-catalog/src/main/resources/catalog-indexes.txt +++ b/opencga-catalog/src/main/resources/catalog-indexes.txt @@ -1,8 +1,20 @@ +{"collections": ["note", "note_archive"], "fields": {"id": 1, "studyUid": 1, "version": 1}, "options": {"unique": true}} +{"collections": ["note", "note_archive"], "fields": {"uid": 1, "version": 1}, "options": {"unique": true}} +{"collections": ["note", "note_archive"], "fields": {"uuid": 1, "version": 1}, "options": {"unique": true}} +{"collections": ["note", "note_archive"], "fields": {"scope": 1, "studyUid": 1}, "options": {}} +{"collections": ["note", "note_archive"], "fields": {"studyUid": 1}, "options": {}} +{"collections": ["note", "note_archive"], "fields": {"visibility": 1, "studyUid": 1}, "options": {}} +{"collections": ["note", "note_archive"], "fields": {"userId": 1, "studyUid": 1}, "options": {}} +{"collections": ["note", "note_archive"], "fields": {"tags": 1, "studyUid": 1}, "options": {}} +{"collections": ["note", "note_archive"], "fields": {"_creationDate": 1, "studyUid": 1}, "options": {}} +{"collections": ["note", "note_archive"], "fields": {"_modificationDate": 1, "studyUid": 1}, "options": {}} + +{"collections": ["project"], "fields": {"id": 1}, "options": {"unique": true}} +{"collections": ["project"], "fields": {"uid": 1}, "options": {"unique": true}} +{"collections": ["project"], "fields": {"uuid": 1}, "options": {"unique": true}} +{"collections": ["project"], "fields": {"fqn": 1}, "options": {"unique": true}} + {"collections": ["user"], "fields": {"id": 1}, "options": {"unique": true}} -{"collections": ["user"], "fields": {"projects.uid": 1, "id": 1}, "options": {"unique": true}} -{"collections": ["user"], "fields": {"projects.uuid": 1, "id": 1}, "options": {"unique": true}} -{"collections": ["user"], "fields": {"projects.fqn": 1, "id": 1}, "options": {"unique": true}} -{"collections": ["user"], "fields": {"projects.id": 1, "id": 1}, "options": {"unique": true}} {"collections": ["study"], "fields": {"uid": 1}, "options": {"unique": true}} {"collections": ["study"], "fields": {"uuid": 1}, "options": {"unique": true}} @@ -195,35 +207,36 @@ {"collections": ["panel", "panel_archive"], "fields": {"studyUid": 1, "release": 1, "_acl": 1}, "options": {}} {"collections": ["panel", "panel_archive"], "fields": {"internal.status.id": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"id": 1, "studyUid": 1}, "options": {"unique": true}} -{"collections": ["clinical"], "fields": {"uuid": 1}, "options": {"unique": true}} -{"collections": ["clinical"], "fields": {"uid": 1}, "options": {"unique": true}} -{"collections": ["clinical"], "fields": {"type": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"files.uid": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"proband.uid": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"proband.samples.uid": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"family.uid": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"family.members.uid": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"family.members.samples.uid": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"panels.uid": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"panelLock": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"locked": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"_dueDate": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"disorder.id": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"disorder.name": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"analyst.id": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"priority.id": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"flags.id": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"qualityControl.summary": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"_creationDate": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"_modificationDate": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"studyUid": 1, "_acl": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"status.id": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"internal.status.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"id": 1, "studyUid": 1}, "options": {"unique": true}} +{"collections": ["clinical", "clinical_archive"], "fields": {"uuid": 1}, "options": {"unique": true}} +{"collections": ["clinical", "clinical_archive"], "fields": {"uid": 1}, "options": {"unique": true}} +{"collections": ["clinical", "clinical_archive"], "fields": {"type": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"files.uid": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"proband.uid": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"proband.samples.uid": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"family.uid": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"family.members.uid": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"family.members.samples.uid": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"panels.uid": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"panelLocked": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"locked": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"_dueDate": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"disorder.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"disorder.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"analyst.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"priority.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"flags.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"qualityControl.summary": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"_creationDate": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"_modificationDate": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"studyUid": 1, "_acl": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"status.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"internal.status.id": 1, "studyUid": 1}, "options": {}} {"collections": ["interpretation", "interpretation_archive"], "fields": {"uuid": 1, "version": 1}, "options": {"unique": true}} {"collections": ["interpretation", "interpretation_archive"], "fields": {"uid": 1, "version": 1}, "options": {"unique": true}} {"collections": ["interpretation", "interpretation_archive"], "fields": {"id": 1, "version": 1, "studyUid": 1}, "options": {"unique": true}} +{"collections": ["interpretation", "interpretation_archive"], "fields": {"name": 1, "studyUid": 1}, "options": {}} {"collections": ["interpretation", "interpretation_archive"], "fields": {"clinicalAnalysisId": 1, "studyUid": 1}, "options": {}} {"collections": ["interpretation", "interpretation_archive"], "fields": {"analyst.id": 1, "studyUid": 1}, "options": {}} {"collections": ["interpretation", "interpretation_archive"], "fields": {"method.name": 1, "studyUid": 1}, "options": {}} diff --git a/opencga-catalog/src/main/resources/solr/cohort-managed-schema b/opencga-catalog/src/main/resources/solr/cohort-managed-schema deleted file mode 100644 index 6e60fa99126..00000000000 --- a/opencga-catalog/src/main/resources/solr/cohort-managed-schema +++ /dev/null @@ -1,593 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - id - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/opencga-catalog/src/main/resources/solr/family-managed-schema b/opencga-catalog/src/main/resources/solr/family-managed-schema deleted file mode 100644 index 88f0a348c6c..00000000000 --- a/opencga-catalog/src/main/resources/solr/family-managed-schema +++ /dev/null @@ -1,596 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - id - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/opencga-catalog/src/main/resources/solr/file-managed-schema b/opencga-catalog/src/main/resources/solr/file-managed-schema deleted file mode 100644 index 5f2044b984a..00000000000 --- a/opencga-catalog/src/main/resources/solr/file-managed-schema +++ /dev/null @@ -1,611 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - id - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/opencga-catalog/src/main/resources/solr/individual-managed-schema b/opencga-catalog/src/main/resources/solr/individual-managed-schema deleted file mode 100644 index 2ad93296e4c..00000000000 --- a/opencga-catalog/src/main/resources/solr/individual-managed-schema +++ /dev/null @@ -1,609 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - id - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/opencga-catalog/src/main/resources/solr/job-managed-schema b/opencga-catalog/src/main/resources/solr/job-managed-schema deleted file mode 100644 index 34e9e099a25..00000000000 --- a/opencga-catalog/src/main/resources/solr/job-managed-schema +++ /dev/null @@ -1,591 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - id - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/opencga-catalog/src/main/resources/solr/sample-managed-schema b/opencga-catalog/src/main/resources/solr/sample-managed-schema deleted file mode 100644 index 64956c45785..00000000000 --- a/opencga-catalog/src/main/resources/solr/sample-managed-schema +++ /dev/null @@ -1,603 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - id - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/auth/authentication/JwtSessionManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/auth/authentication/JwtSessionManagerTest.java index e2fef4763ad..c9d2ca8aec4 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/auth/authentication/JwtSessionManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/auth/authentication/JwtSessionManagerTest.java @@ -42,6 +42,7 @@ public class JwtSessionManagerTest extends GenericTest { private JwtManager jwtSessionManager; private String jwtToken; + private String organizationId = "test_org"; @Before public void setUp() throws Exception { @@ -52,7 +53,7 @@ public void setUp() throws Exception { @Test public void testCreateJWTToken() throws Exception { - jwtToken = jwtSessionManager.createJWTToken("testUser", Collections.emptyMap(), 60L); + jwtToken = jwtSessionManager.createJWTToken(organizationId, null, "testUser", Collections.emptyMap(), 60L); } @Test @@ -80,7 +81,7 @@ public void testInvalidSecretKey() throws CatalogAuthenticationException { @Test public void testNonExpiringToken() throws CatalogException { - String nonExpiringToken = jwtSessionManager.createJWTToken("System", null, -1L); + String nonExpiringToken = jwtSessionManager.createJWTToken(organizationId, null, "System", null, -1L); assertEquals(jwtSessionManager.getUser(nonExpiringToken), "System"); assertNull(jwtSessionManager.getExpiration(nonExpiringToken)); } diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/auth/authorization/CatalogAuthorizationManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/auth/authorization/CatalogAuthorizationManagerTest.java index 01121cbe2d3..b7c521076c8 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/auth/authorization/CatalogAuthorizationManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/auth/authorization/CatalogAuthorizationManagerTest.java @@ -16,41 +16,32 @@ package org.opencb.opencga.catalog.auth.authorization; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; -import org.junit.rules.ExpectedException; import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.commons.test.GenericTest; import org.opencb.opencga.TestParamConstants; import org.opencb.opencga.catalog.db.api.SampleDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.managers.*; +import org.opencb.opencga.catalog.managers.AbstractManagerTest; +import org.opencb.opencga.catalog.managers.SampleManager; +import org.opencb.opencga.catalog.managers.StudyManager; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.api.ParamConstants; -import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.models.AclEntry; import org.opencb.opencga.core.models.AclEntryList; -import org.opencb.opencga.core.models.cohort.Cohort; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.file.File; import org.opencb.opencga.core.models.file.FileAclParams; -import org.opencb.opencga.core.models.file.FileCreateParams; -import org.opencb.opencga.core.models.file.FilePermissions; -import org.opencb.opencga.core.models.individual.Individual; import org.opencb.opencga.core.models.job.Job; import org.opencb.opencga.core.models.project.Project; import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.sample.SampleAclParams; import org.opencb.opencga.core.models.sample.SamplePermissions; import org.opencb.opencga.core.models.study.*; -import org.opencb.opencga.core.models.user.Account; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.MediumTests; @@ -60,7 +51,6 @@ import java.util.*; import java.util.stream.Collectors; -import static org.apache.commons.lang3.StringUtils.join; import static org.junit.Assert.*; import static org.opencb.opencga.catalog.utils.ParamUtils.AclAction.*; @@ -70,159 +60,7 @@ * @author Jacobo Coll <jacobo167@gmail.com> */ @Category(MediumTests.class) -public class CatalogAuthorizationManagerTest extends GenericTest { - - private final String ownerUser = "owner"; - private final String studyAdminUser1 = "studyAdmin1"; - private final String studyAdminUser2 = "studyAdmin2"; - private final String memberUser = "member_member"; - private final String externalUser = "external"; - - private final String groupAdmin = "@admins"; - private final String groupMember = "@analyst"; - - private final String ALL_FILE_PERMISSIONS = join( - EnumSet.allOf(FilePermissions.class) - .stream() - .map(FilePermissions::name) - .collect(Collectors.toList()), - ","); - private final String DENY_FILE_PERMISSIONS = ""; - - private final String ALL_SAMPLE_PERMISSIONS = join( - EnumSet.allOf(SamplePermissions.class) - .stream() - .map(SamplePermissions::name) - .collect(Collectors.toList()), - ","); - private final String DENY_SAMPLE_PERMISSIONS = ""; - - private final SampleAclParams allSamplePermissions = new SampleAclParams(null, null, null, null, ALL_SAMPLE_PERMISSIONS); - private final SampleAclParams noSamplePermissions = new SampleAclParams(null, null, null, null, DENY_SAMPLE_PERMISSIONS); - - private static final QueryOptions INCLUDE_RESULT = new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true); - - @Rule - public ExpectedException thrown = ExpectedException.none(); - private CatalogManager catalogManager; - private FileManager fileManager; - private String opencgaToken; - private String ownerSessionId; - private String studyAdmin1SessionId; - private String studyAdmin2SessionId; - private String memberSessionId; - private String externalSessionId; - private String p1; - private String studyFqn; - private long studyUid; - private String data = "data/"; // - private String data_d1 = "data/d1/"; // Shared with member, Forbidden for @admins, Shared with studyAdmin1 - private String data_d1_d2 = "data/d1/d2/"; // Forbidden for @admins - private String data_d1_d2_d3 = "data/d1/d2/d3/"; // Forbidden for member - private String data_d1_d2_d3_d4 = "data/d1/d2/d3/d4/"; // Shared for @admins - private String data_d1_d2_d3_d4_txt = "data/d1/d2/d3/d4/my.txt"; // Shared for member - private Sample smp1; // Shared with member - private Sample smp2; // Shared with studyAdmin1 - private Sample smp3; // Shared with member - private Sample smp4; // Shared with @members - private Sample smp6; // Shared with * - private Sample smp5; // Shared with @members, forbidden for memberUse - private String ind1 = "ind1"; - private String ind2 = "ind2"; - - @Before - public void before() throws Exception { - Configuration configuration = Configuration.load(getClass().getResource("/configuration-test.yml").openStream()); - CatalogManagerExternalResource.clearCatalog(configuration); - - catalogManager = new CatalogManager(configuration); - catalogManager.installCatalogDB(configuration.getAdmin().getSecretKey(), TestParamConstants.ADMIN_PASSWORD, "opencga@admin.com", "", true, true); - opencgaToken = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); - - fileManager = catalogManager.getFileManager(); - - catalogManager.getUserManager().create(ownerUser, ownerUser, "email@ccc.ccc", TestParamConstants.PASSWORD, "ASDF", null, Account.AccountType.FULL, opencgaToken); - catalogManager.getUserManager().create(studyAdminUser1, studyAdminUser1, "email@ccc.ccc", TestParamConstants.PASSWORD, "ASDF", null, Account.AccountType.FULL, opencgaToken); - catalogManager.getUserManager().create(studyAdminUser2, studyAdminUser2, "email@ccc.ccc", TestParamConstants.PASSWORD, "ASDF", null, Account.AccountType.FULL, opencgaToken); - catalogManager.getUserManager().create(memberUser, memberUser, "email@ccc.ccc", TestParamConstants.PASSWORD, "ASDF", null, Account.AccountType.FULL, opencgaToken); - catalogManager.getUserManager().create(externalUser, externalUser, "email@ccc.ccc", TestParamConstants.PASSWORD, "ASDF", null, Account.AccountType.FULL, opencgaToken); - - ownerSessionId = catalogManager.getUserManager().login(ownerUser, TestParamConstants.PASSWORD).getToken(); - studyAdmin1SessionId = catalogManager.getUserManager().login(studyAdminUser1, TestParamConstants.PASSWORD).getToken(); - studyAdmin2SessionId = catalogManager.getUserManager().login(studyAdminUser2, TestParamConstants.PASSWORD).getToken(); - memberSessionId = catalogManager.getUserManager().login(memberUser, TestParamConstants.PASSWORD).getToken(); - externalSessionId = catalogManager.getUserManager().login(externalUser, TestParamConstants.PASSWORD).getToken(); - - p1 = catalogManager.getProjectManager().create("p1", "p1", null, "Homo sapiens", - null, "GRCh38", INCLUDE_RESULT, ownerSessionId).first().getId(); - Study study = catalogManager.getStudyManager().create(p1, "studyFqn", "studyFqn", "studyFqn", null, null, null, - null, null, INCLUDE_RESULT, ownerSessionId).first(); - studyFqn = study.getFqn(); - studyUid = study.getUid(); - data_d1 = catalogManager.getFileManager().createFolder(studyFqn, Paths.get("data/d1/").toString(), true, null, - INCLUDE_RESULT, ownerSessionId).first().getPath(); - data_d1_d2 = catalogManager.getFileManager().createFolder(studyFqn, Paths.get("data/d1/d2/").toString(), false, - null, INCLUDE_RESULT, ownerSessionId).first().getPath(); - data_d1_d2_d3 = catalogManager.getFileManager().createFolder(studyFqn, Paths.get("data/d1/d2/d3/").toString(), - false, null, INCLUDE_RESULT, ownerSessionId).first().getPath(); - data_d1_d2_d3_d4 = catalogManager.getFileManager().createFolder(studyFqn, Paths.get("data/d1/d2/d3/d4/").toString(), - false, null, INCLUDE_RESULT, ownerSessionId).first().getPath(); - catalogManager.getFileManager().create(studyFqn, - new FileCreateParams() - .setContent("file content") - .setPath("data/d1/d2/d3/d4/my.txt") - .setType(File.Type.FILE), - false, ownerSessionId); - - // Add studyAdminUser1 and studyAdminUser2 to admin group and admin role. - catalogManager.getStudyManager().updateGroup(studyFqn, groupAdmin, ParamUtils.BasicUpdateAction.SET, - new GroupUpdateParams(Arrays.asList(studyAdminUser1, studyAdminUser2)), ownerSessionId); - - StudyAclParams aclParams1 = new StudyAclParams("", AuthorizationManager.ROLE_ANALYST); - catalogManager.getStudyManager().updateAcl(Arrays.asList(studyFqn), memberUser, aclParams1, ADD, - studyAdmin1SessionId); - StudyAclParams aclParams = new StudyAclParams("", AuthorizationManager.ROLE_LOCKED); - catalogManager.getStudyManager().updateAcl(Arrays.asList(studyFqn), externalUser, aclParams, ADD, - studyAdmin1SessionId); - - fileManager.updateAcl(studyFqn, Arrays.asList(data_d1), externalUser, new FileAclParams(null, ALL_FILE_PERMISSIONS), - SET, ownerSessionId); - fileManager.updateAcl(studyFqn, Arrays.asList(data_d1_d2_d3), externalUser, - new FileAclParams(null, DENY_FILE_PERMISSIONS), SET, ownerSessionId); - fileManager.updateAcl(studyFqn, Arrays.asList(data_d1_d2_d3_d4_txt), externalUser, - new FileAclParams(null, ALL_FILE_PERMISSIONS), SET, ownerSessionId); - - smp1 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("smp1"), INCLUDE_RESULT, ownerSessionId).first(); - smp2 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("smp2"), INCLUDE_RESULT, ownerSessionId).first(); - smp3 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("smp3"), INCLUDE_RESULT, ownerSessionId).first(); - smp4 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("smp4"), INCLUDE_RESULT, ownerSessionId).first(); - smp5 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("smp5"), INCLUDE_RESULT, ownerSessionId).first(); - smp6 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("smp6"), INCLUDE_RESULT, ownerSessionId).first(); - - catalogManager.getCohortManager().create(studyFqn, new Cohort().setId("all").setSamples(Arrays.asList(smp1, smp2, smp3)), - QueryOptions.empty(), ownerSessionId); - - catalogManager.getIndividualManager().create(studyFqn, new Individual().setId(ind1), Collections.singletonList(smp1.getId()), - QueryOptions.empty(), ownerSessionId); - catalogManager.getIndividualManager().create(studyFqn, new Individual().setId(ind2), Collections.singletonList(smp2.getId()), - QueryOptions.empty(), ownerSessionId); - - catalogManager.getSampleManager().updateAcl(studyFqn, Collections.singletonList(smp1.getId()), externalUser, allSamplePermissions, - SET, ownerSessionId); - catalogManager.getSampleManager().updateAcl(studyFqn, Collections.singletonList(smp3.getId()), externalUser, noSamplePermissions, - SET, ownerSessionId); - catalogManager.getSampleManager().updateAcl(studyFqn, Collections.singletonList(smp2.getId()), "*", noSamplePermissions, - SET, ownerSessionId); - catalogManager.getSampleManager().updateAcl(studyFqn, Collections.singletonList(smp5.getId()), externalUser, noSamplePermissions, - SET, ownerSessionId); - catalogManager.getSampleManager().updateAcl(studyFqn, Collections.singletonList(smp6.getId()), "@members", allSamplePermissions, - SET, ownerSessionId); - } - - @After - public void after() throws Exception { - catalogManager.close(); - } +public class CatalogAuthorizationManagerTest extends AbstractManagerTest { /*--------------------------*/ // Add group members @@ -250,63 +88,64 @@ private DataResult updateGroup(String studyStr, String groupId, @Nullable @Test public void addMemberToGroup() throws CatalogException { - updateGroup(studyFqn, groupAdmin, externalUser, null, null, ownerSessionId); + updateGroup(studyFqn, ParamConstants.ADMINS_GROUP, normalUserId3, null, null, ownerToken); Map groups = getGroupMap(); - assertTrue(groups.get(groupAdmin).getUserIds().contains(externalUser)); + assertTrue(groups.get(ParamConstants.ADMINS_GROUP).getUserIds().contains(normalUserId3)); } @Test public void addMemberToGroupExistingNoPermission() throws CatalogException { thrown.expect(CatalogAuthorizationException.class); - updateGroup(studyFqn, groupAdmin, externalUser, null, null, externalSessionId); + updateGroup(studyFqn, ParamConstants.ADMINS_GROUP, normalUserId3, null, null, normalToken3); } @Test public void changeGroupMembership() throws CatalogException { - updateGroup(studyFqn, groupAdmin, externalUser, null, null, ownerSessionId); -// catalogManager.addUsersToGroup(studyFqn, groupAdmin, externalUser, ownerSessionId); + updateGroup(studyFqn, ParamConstants.ADMINS_GROUP, normalUserId3, null, null, ownerToken); +// catalogManager.addUsersToGroup(studyFqn, ParamConstants.ADMINS_GROUP, normalUserId3, token); Map groups = getGroupMap(); - assertTrue(groups.get(groupAdmin).getUserIds().contains(externalUser)); + assertTrue(groups.get(ParamConstants.ADMINS_GROUP).getUserIds().contains(normalUserId3)); // thrown.expect(CatalogException.class); -// catalogManager.addUsersToGroup(s1, groupMember, externalUser, ownerSessionId); - updateGroup(studyFqn, groupAdmin, null, externalUser, null, ownerSessionId); - catalogManager.getStudyManager().createGroup(studyFqn, new Group(groupMember, Collections.singletonList(externalUser)), - ownerSessionId); - // catalogManager.updateGroup(Long.toString(s1), groupMember, externalUser, null, null, ownerSessionId); +// catalogManager.addUsersToGroup(s1, restrictedGroup, normalUserId3, token); + updateGroup(studyFqn, ParamConstants.ADMINS_GROUP, null, normalUserId3, null, ownerToken); + updateGroup(studyFqn, restrictedGroup, normalUserId1, null, null, ownerToken); +// catalogManager.getStudyManager().createGroup(studyFqn, new Group(restrictedGroup, Collections.singletonList(normalUserId3)), +// ownerToken); + // catalogManager.updateGroup(Long.toString(s1), restrictedGroup, normalUserId3, null, null, token); groups = getGroupMap(); - assertTrue(groups.get(groupMember).getUserIds().contains(externalUser)); - assertTrue(!groups.get(groupAdmin).getUserIds().contains(externalUser)); + assertTrue(groups.get(restrictedGroup).getUserIds().contains(normalUserId1)); + assertTrue(!groups.get(ParamConstants.ADMINS_GROUP).getUserIds().contains(normalUserId1)); } @Test public void adminUserRemovesFromAdminsGroup() throws CatalogException { thrown.expectMessage("Only the owner"); thrown.expect(CatalogAuthorizationException.class); - catalogManager.getStudyManager().updateGroup(studyFqn, groupAdmin, ParamUtils.BasicUpdateAction.REMOVE, - new GroupUpdateParams(Collections.singletonList(studyAdminUser2)), studyAdmin1SessionId); + catalogManager.getStudyManager().updateGroup(studyFqn, ParamConstants.ADMINS_GROUP, ParamUtils.BasicUpdateAction.REMOVE, + new GroupUpdateParams(Collections.singletonList(orgAdminUserId2)), studyAdminToken1); } @Test public void adminUserAddsFromAdminsGroup() throws CatalogException { thrown.expectMessage("Only the owner"); thrown.expect(CatalogAuthorizationException.class); - catalogManager.getStudyManager().updateGroup(studyFqn, groupAdmin, ParamUtils.BasicUpdateAction.ADD, - new GroupUpdateParams(Collections.singletonList(externalUser)), studyAdmin1SessionId); + catalogManager.getStudyManager().updateGroup(studyFqn, ParamConstants.ADMINS_GROUP, ParamUtils.BasicUpdateAction.ADD, + new GroupUpdateParams(Collections.singletonList(normalUserId3)), studyAdminToken1); } @Test public void adminUserSetsFromAdminsGroup() throws CatalogException { thrown.expectMessage("Only the owner"); thrown.expect(CatalogAuthorizationException.class); - catalogManager.getStudyManager().updateGroup(studyFqn, groupAdmin, ParamUtils.BasicUpdateAction.SET, - new GroupUpdateParams(Collections.singletonList(studyAdminUser1)), studyAdmin1SessionId); + catalogManager.getStudyManager().updateGroup(studyFqn, ParamConstants.ADMINS_GROUP, ParamUtils.BasicUpdateAction.SET, + new GroupUpdateParams(Collections.singletonList(orgAdminUserId1)), studyAdminToken1); } // @Test // public void addMemberToTheBelongingGroup() throws CatalogException { -// catalogManager.addUsersToGroup(studyFqn, groupAdmin, externalUser, ownerSessionId); +// catalogManager.addUsersToGroup(studyFqn, ParamConstants.ADMINS_GROUP, normalUserId3, token); // thrown.expect(CatalogException.class); -// catalogManager.addUsersToGroup(studyFqn, groupAdmin, externalUser, ownerSessionId); +// catalogManager.addUsersToGroup(studyFqn, ParamConstants.ADMINS_GROUP, normalUserId3, token); // } /*--------------------------*/ @@ -315,45 +154,34 @@ public void adminUserSetsFromAdminsGroup() throws CatalogException { @Test public void removeMemberFromGroup() throws CatalogException { - // Create new group - catalogManager.getStudyManager().createGroup(studyFqn, - new Group(groupMember, Arrays.asList(studyAdminUser1, studyAdminUser2)), ownerSessionId); - // Remove one of the users - updateGroup(studyFqn, groupMember, null, studyAdminUser1, null, ownerSessionId); - assertFalse(getGroupMap().get(groupMember).getUserIds().contains(studyAdminUser1)); + assertTrue(getGroupMap().get(restrictedGroup).getUserIds().contains(normalUserId3)); + updateGroup(studyFqn, restrictedGroup, null, normalUserId3, null, ownerToken); + assertFalse(getGroupMap().get(restrictedGroup).getUserIds().contains(normalUserId3)); // Remove the last user in the admin group - updateGroup(studyFqn, groupMember, null, studyAdminUser2, null, ownerSessionId); - assertFalse(getGroupMap().get(groupMember).getUserIds().contains(studyAdminUser2)); + updateGroup(studyFqn, restrictedGroup, null, normalUserId2, null, ownerToken); + assertFalse(getGroupMap().get(restrictedGroup).getUserIds().contains(normalUserId2)); // // Cannot remove group with defined ACLs // thrown.expect(CatalogDBException.class); // thrown.expectMessage("ACL defined"); - catalogManager.getStudyManager().deleteGroup(studyFqn, groupMember, ownerSessionId); - assertNull(getGroupMap().get(groupMember)); - + catalogManager.getStudyManager().deleteGroup(studyFqn, restrictedGroup, ownerToken); + assertNull(getGroupMap().get(restrictedGroup)); } @Test public void removeMemberFromGroupNoPermission() throws CatalogException { thrown.expect(CatalogAuthorizationException.class); - updateGroup(studyFqn, groupAdmin, null, ownerUser, null, externalSessionId); -// catalogManager.removeUsersFromGroup(studyFqn, groupAdmin, ownerUser, externalSessionId); + updateGroup(studyFqn, ParamConstants.ADMINS_GROUP, null, orgOwnerUserId, null, normalToken3); +// catalogManager.removeUsersFromGroup(studyFqn, ParamConstants.ADMINS_GROUP, ownerUserId, normalToken3); } @Test public void removeMemberFromNonExistingGroup() throws CatalogException { thrown.expect(CatalogException.class); -// catalogManager.removeUsersFromGroup(studyFqn, "NO_GROUP", ownerUser, ownerSessionId); - updateGroup(studyFqn, "NO_GROUP", null, ownerUser, null, ownerSessionId); - } - - @Test - public void removeMemberFromNonBelongingGroup() throws CatalogException { - thrown.expect(CatalogException.class); -// catalogManager.removeUsersFromGroup(studyFqn, groupMember, ownerUser, ownerSessionId); - updateGroup(studyFqn, groupMember, null, ownerUser, null, ownerSessionId); +// catalogManager.removeUsersFromGroup(studyFqn, "NO_GROUP", ownerUserId, token); + updateGroup(studyFqn, "NO_GROUP", null, orgOwnerUserId, null, ownerToken); } /*--------------------------*/ @@ -364,33 +192,33 @@ public void removeMemberFromNonBelongingGroup() throws CatalogException { @Test public void addExistingUserToRole() throws CatalogException { String newUser = "newUser"; - catalogManager.getUserManager().create(newUser, newUser, "email@ccc.ccc", TestParamConstants.PASSWORD, "ASDF", null, Account.AccountType.FULL, opencgaToken); + catalogManager.getUserManager().create(newUser, newUser, "email@ccc.ccc", TestParamConstants.PASSWORD, organizationId, null, ownerToken); StudyAclParams aclParams = new StudyAclParams("", AuthorizationManager.ROLE_ANALYST); - catalogManager.getStudyManager().updateAcl(Arrays.asList(studyFqn), newUser, aclParams, ParamUtils.AclAction.ADD, ownerSessionId); + catalogManager.getStudyManager().updateAcl(studyFqn, newUser, aclParams, ParamUtils.AclAction.ADD, ownerToken); } // A user with no permissions tries to add an existing user to a role @Test public void addExistingUserToRole2() throws CatalogException { String newUser = "newUser"; - catalogManager.getUserManager().create(newUser, newUser, "email@ccc.ccc", TestParamConstants.PASSWORD, "ASDF", null, Account.AccountType.FULL, opencgaToken); + catalogManager.getUserManager().create(newUser, newUser, "email@ccc.ccc", TestParamConstants.PASSWORD, organizationId, null, ownerToken); thrown.expect(CatalogAuthorizationException.class); - thrown.expectMessage("Only owners or administrative users"); + thrown.expectMessage("administrators"); StudyAclParams aclParams = new StudyAclParams("", AuthorizationManager.ROLE_ANALYST); - catalogManager.getStudyManager().updateAcl(Arrays.asList(studyFqn), newUser, aclParams, ParamUtils.AclAction.ADD, memberSessionId); + catalogManager.getStudyManager().updateAcl(studyFqn, newUser, aclParams, ParamUtils.AclAction.ADD, normalToken3); } @Test public void addExistingGroupToRole() throws CatalogException { String newUser = "newUser"; - catalogManager.getUserManager().create(newUser, newUser, "email@ccc.ccc", TestParamConstants.PASSWORD, "ASDF", null, Account.AccountType.FULL, opencgaToken); + catalogManager.getUserManager().create(newUser, newUser, "email@ccc.ccc", TestParamConstants.PASSWORD, organizationId, null, ownerToken); String group = "@newGroup"; -// catalogManager.addUsersToGroup(studyFqn, group, newUser, studyAdmin1SessionId); - catalogManager.getStudyManager().createGroup(studyFqn, group, Collections.singletonList(newUser), studyAdmin1SessionId); +// catalogManager.addUsersToGroup(studyFqn, group, newUser, adminToken1); + catalogManager.getStudyManager().createGroup(studyFqn, group, Collections.singletonList(newUser), orgAdminToken1); StudyAclParams aclParams = new StudyAclParams("", AuthorizationManager.ROLE_ANALYST); - catalogManager.getStudyManager().updateAcl(Arrays.asList(studyFqn), group, aclParams, ParamUtils.AclAction.ADD, studyAdmin1SessionId); + catalogManager.getStudyManager().updateAcl(studyFqn, group, aclParams, ParamUtils.AclAction.ADD, orgAdminToken1); OpenCGAResult> studyAcls = catalogManager.getAuthorizationManager() - .getStudyAcl(studyAdminUser1, studyUid, group); + .getStudyAcl(organizationId, studyUid, group, orgAdminUserId1); assertEquals(1, studyAcls.getNumResults()); assertEquals(group, studyAcls.first().getAcl().get(0).getMember()); @@ -406,8 +234,8 @@ public void addNonExistingUserToRole() throws CatalogException { thrown.expect(CatalogDBException.class); thrown.expectMessage("does not exist"); StudyAclParams aclParams = new StudyAclParams("", AuthorizationManager.ROLE_ANALYST); - catalogManager.getStudyManager().updateAcl(Arrays.asList(studyFqn), userNotRegistered, aclParams, ParamUtils.AclAction.ADD, - studyAdmin1SessionId); + catalogManager.getStudyManager().updateAcl(studyFqn, userNotRegistered, aclParams, ParamUtils.AclAction.ADD, + orgAdminToken1); } @Test @@ -415,33 +243,33 @@ public void addNonExistingGroupToRole() throws CatalogException { String groupNotRegistered = "@groupNotRegistered"; thrown.expect(CatalogDBException.class); thrown.expectMessage("not found"); - catalogManager.getStudyManager().updateAcl(Arrays.asList(studyFqn), groupNotRegistered, new StudyAclParams("", - AuthorizationManager.ROLE_ANALYST), ParamUtils.AclAction.SET, studyAdmin1SessionId); + catalogManager.getStudyManager().updateAcl(studyFqn, groupNotRegistered, new StudyAclParams("", + AuthorizationManager.ROLE_ANALYST), ParamUtils.AclAction.SET, orgAdminToken1); } @Test public void changeUserRole() throws CatalogException { OpenCGAResult> studyAcls = catalogManager.getStudyManager() - .getAcls(Collections.singletonList(studyFqn), externalUser, false, studyAdmin1SessionId); + .getAcls(Collections.singletonList(studyFqn), normalUserId3, false, orgAdminToken1); assertEquals(1, studyAcls.getNumResults()); assertEquals(1, studyAcls.first().getAcl().size()); - assertEquals(externalUser, studyAcls.first().getAcl().get(0).getMember()); + assertEquals(normalUserId3, studyAcls.first().getAcl().get(0).getMember()); // Change role StudyAclParams aclParams1 = new StudyAclParams(null, null); - catalogManager.getStudyManager().updateAcl(Arrays.asList(studyFqn), externalUser, aclParams1, RESET, - studyAdmin1SessionId); + catalogManager.getStudyManager().updateAcl(studyFqn, normalUserId3, aclParams1, RESET, + orgAdminToken1); StudyAclParams aclParams = new StudyAclParams("", AuthorizationManager.ROLE_ANALYST); - catalogManager.getStudyManager().updateAcl(Arrays.asList(studyFqn), externalUser, aclParams, ParamUtils.AclAction.ADD, - studyAdmin1SessionId); + catalogManager.getStudyManager().updateAcl(studyFqn, normalUserId3, aclParams, ParamUtils.AclAction.ADD, + orgAdminToken1); - studyAcls = catalogManager.getStudyManager().getAcls(Collections.singletonList(studyFqn), externalUser, false, - studyAdmin1SessionId); + studyAcls = catalogManager.getStudyManager().getAcls(Collections.singletonList(studyFqn), normalUserId3, false, + orgAdminToken1); assertEquals(1, studyAcls.getNumResults()); assertEquals(1, studyAcls.first().getAcl().size()); - assertEquals(externalUser, studyAcls.first().getAcl().get(0).getMember()); + assertEquals(normalUserId3, studyAcls.first().getAcl().get(0).getMember()); assertEquals(AuthorizationManager.getAnalystAcls().size(), studyAcls.first().getAcl().get(0).getPermissions().size()); for (StudyPermissions.Permissions analystAcl : AuthorizationManager.getAnalystAcls()) { assertTrue(studyAcls.first().getAcl().get(0).getPermissions().contains(analystAcl)); @@ -456,13 +284,13 @@ public void changeUserRole() throws CatalogException { @Test public void removeUserFromRole() throws CatalogException { StudyAclParams aclParams = new StudyAclParams(null, null); - catalogManager.getStudyManager().updateAcl(Collections.singletonList(studyFqn), externalUser, aclParams, RESET, studyAdmin1SessionId); + catalogManager.getStudyManager().updateAcl(studyFqn, normalUserId3, aclParams, RESET, orgAdminToken1); OpenCGAResult> studyAcls = catalogManager.getStudyManager().getAcls( - Collections.singletonList(studyFqn), externalUser, false, studyAdmin1SessionId); + Collections.singletonList(studyFqn), normalUserId3, false, orgAdminToken1); assertEquals(1, studyAcls.getNumResults()); assertEquals(1, studyAcls.first().getAcl().size()); - assertEquals(externalUser, studyAcls.first().getAcl().get(0).getMember()); + assertEquals(normalUserId3, studyAcls.first().getAcl().get(0).getMember()); assertNull(studyAcls.first().getAcl().get(0).getPermissions()); } @@ -470,13 +298,13 @@ public void removeUserFromRole() throws CatalogException { @Test public void denyAllPermissions() throws CatalogException { StudyAclParams aclParams = new StudyAclParams("", null); - catalogManager.getStudyManager().updateAcl(Collections.singletonList(studyFqn), externalUser, aclParams, SET, studyAdmin1SessionId); + catalogManager.getStudyManager().updateAcl(studyFqn, normalUserId3, aclParams, SET, orgAdminToken1); OpenCGAResult> studyAcls = catalogManager.getStudyManager().getAcls( - Collections.singletonList(studyFqn), externalUser, false, studyAdmin1SessionId); + Collections.singletonList(studyFqn), normalUserId3, false, orgAdminToken1); assertEquals(1, studyAcls.getNumResults()); assertEquals(1, studyAcls.first().getAcl().size()); - assertEquals(externalUser, studyAcls.first().getAcl().get(0).getMember()); + assertEquals(normalUserId3, studyAcls.first().getAcl().get(0).getMember()); assertEquals(1, studyAcls.first().getAcl().get(0).getPermissions().size()); assertTrue(studyAcls.first().getAcl().get(0).getPermissions().contains(StudyPermissions.Permissions.NONE)); } @@ -486,19 +314,19 @@ public void denyAllPermissions() throws CatalogException { public void removeUserFromRole2() throws CatalogException { thrown.expect(CatalogAuthorizationException.class); StudyAclParams aclParams = new StudyAclParams(null, null); - catalogManager.getStudyManager().updateAcl(Arrays.asList(studyFqn), externalUser, aclParams, RESET, memberSessionId); + catalogManager.getStudyManager().updateAcl(studyFqn, normalUserId3, aclParams, RESET, normalToken3); } @Test public void removeGroupFromRole() throws CatalogException { String group = "@newGroup"; - catalogManager.getStudyManager().createGroup(studyFqn, new Group(group, Arrays.asList(studyAdminUser1, studyAdminUser2)), - studyAdmin1SessionId); - catalogManager.getStudyManager().updateAcl(Arrays.asList(studyFqn), group, new StudyAclParams("", "admin"), SET, ownerSessionId); + catalogManager.getStudyManager().createGroup(studyFqn, new Group(group, Arrays.asList(orgAdminUserId1, orgAdminUserId2)), + orgAdminToken1); + catalogManager.getStudyManager().updateAcl(studyFqn, group, new StudyAclParams("", "admin"), SET, ownerToken); - Study study = catalogManager.getStudyManager().get(studyFqn, QueryOptions.empty(), studyAdmin1SessionId).first(); - OpenCGAResult> studyAcls = catalogManager.getAuthorizationManager().getStudyAcl(studyAdminUser1, - study.getUid(), group); + Study study = catalogManager.getStudyManager().get(studyFqn, QueryOptions.empty(), orgAdminToken1).first(); + OpenCGAResult> studyAcls = catalogManager.getAuthorizationManager().getStudyAcl(organizationId, study.getUid(), group, orgAdminUserId1 + ); assertEquals(1, studyAcls.getNumResults()); assertEquals(group, studyAcls.first().getAcl().get(0).getMember()); @@ -508,9 +336,8 @@ public void removeGroupFromRole() throws CatalogException { } StudyAclParams aclParams = new StudyAclParams(null, null); - catalogManager.getStudyManager().updateAcl(Arrays.asList(studyFqn), group, aclParams, RESET, ownerSessionId); - String userId = catalogManager.getUserManager().getUserId(ownerSessionId); - studyAcls = catalogManager.getAuthorizationManager().getStudyAcl(userId, study.getUid(), group); + catalogManager.getStudyManager().updateAcl(studyFqn, group, aclParams, RESET, ownerToken); + studyAcls = catalogManager.getAuthorizationManager().getStudyAcl(organizationId, study.getUid(), group, orgOwnerUserId); assertEquals(1, studyAcls.getNumResults()); assertEquals(1, studyAcls.first().getAcl().size()); assertEquals(group, studyAcls.first().getAcl().get(0).getMember()); @@ -522,9 +349,9 @@ public void removeNonExistingUserFromRole() throws CatalogException { String userNotRegistered = "userNotRegistered"; thrown.expect(CatalogException.class); thrown.expectMessage("does not exist"); -// catalogManager.unshareStudy(studyFqn, userNotRegistered, ownerSessionId); +// catalogManager.unshareStudy(studyFqn, userNotRegistered, token); StudyAclParams aclParams = new StudyAclParams(null, null); - catalogManager.getStudyManager().updateAcl(Arrays.asList(studyFqn), userNotRegistered, aclParams, RESET, ownerSessionId); + catalogManager.getStudyManager().updateAcl(studyFqn, userNotRegistered, aclParams, RESET, ownerToken); } @Test @@ -533,7 +360,7 @@ public void removeNonExistingGroupFromRole() throws CatalogException { thrown.expect(CatalogException.class); thrown.expectMessage("not found"); StudyAclParams aclParams = new StudyAclParams(null, null); - catalogManager.getStudyManager().updateAcl(Arrays.asList(studyFqn), groupNotRegistered, aclParams, RESET, ownerSessionId); + catalogManager.getStudyManager().updateAcl(studyFqn, groupNotRegistered, aclParams, RESET, ownerToken); } /*--------------------------*/ @@ -542,18 +369,22 @@ public void removeNonExistingGroupFromRole() throws CatalogException { @Test public void readProject() throws CatalogException { - DataResult project = catalogManager.getProjectManager().get(p1, null, ownerSessionId); + DataResult project = catalogManager.getProjectManager().get(project1, null, ownerToken); assertEquals(1, project.getNumResults()); - project = catalogManager.getProjectManager().get(p1, null, memberSessionId); + project = catalogManager.getProjectManager().get(project1, null, normalToken3); assertEquals(1, project.getNumResults()); + + thrown.expect(CatalogAuthorizationException.class); + thrown.expectMessage("denied"); + catalogManager.getProjectManager().get(project3, null, normalToken3); } @Test public void readProjectDeny() throws CatalogException { catalogManager.getStudyManager().updateGroup(studyFqn, "@members", ParamUtils.BasicUpdateAction.REMOVE, - new GroupUpdateParams(Arrays.asList(externalUser, ParamConstants.ANONYMOUS_USER_ID)), ownerSessionId); + new GroupUpdateParams(Arrays.asList(normalUserId3, ParamConstants.ANONYMOUS_USER_ID)), ownerToken); thrown.expect(CatalogAuthorizationException.class); - catalogManager.getProjectManager().get(p1, null, externalSessionId); + catalogManager.getProjectManager().get(project3, null, normalToken3); } /*--------------------------*/ @@ -562,18 +393,18 @@ public void readProjectDeny() throws CatalogException { @Test public void readStudy() throws CatalogException { - DataResult study = catalogManager.getStudyManager().get(studyFqn, null, ownerSessionId); + DataResult study = catalogManager.getStudyManager().get(studyFqn, null, ownerToken); assertEquals(1, study.getNumResults()); - study = catalogManager.getStudyManager().get(studyFqn, null, memberSessionId); + study = catalogManager.getStudyManager().get(studyFqn, null, normalToken3); assertEquals(1, study.getNumResults()); } @Test public void readStudyDeny() throws CatalogException { catalogManager.getStudyManager().updateGroup(String.valueOf(studyFqn), "@members", ParamUtils.BasicUpdateAction.REMOVE, - new GroupUpdateParams(Arrays.asList(externalUser, ParamConstants.ANONYMOUS_USER_ID)), ownerSessionId); + new GroupUpdateParams(Arrays.asList(normalUserId3, ParamConstants.ANONYMOUS_USER_ID)), ownerToken); thrown.expect(CatalogAuthorizationException.class); - catalogManager.getStudyManager().get(studyFqn, null, externalSessionId); + catalogManager.getStudyManager().get(studyFqn, null, normalToken3); } /*--------------------------*/ @@ -582,60 +413,60 @@ public void readStudyDeny() throws CatalogException { @Test public void readFileByOwner() throws CatalogException { - DataResult file = catalogManager.getFileManager().get(studyFqn, data_d1, null, ownerSessionId); + DataResult file = catalogManager.getFileManager().get(studyFqn, data_d1, null, ownerToken); assertEquals(1, file.getNumResults()); } @Test public void readExplicitlySharedFolder() throws CatalogException { - catalogManager.getFileManager().get(studyFqn, data_d1, null, externalSessionId); + catalogManager.getFileManager().get(studyFqn, data_d1, null, normalToken3); } @Test public void readExplicitlyUnsharedFile() throws CatalogException { - DataResult file = catalogManager.getFileManager().get(studyFqn, data_d1, null, memberSessionId); + DataResult file = catalogManager.getFileManager().get(studyFqn, data_d1, null, normalToken3); assertEquals(1, file.getNumResults()); // Set an ACL with no permissions - fileManager.updateAcl(studyFqn, Arrays.asList(data_d1), memberUser, new FileAclParams(null, null), SET, ownerSessionId); + catalogManager.getFileManager().updateAcl(studyFqn, Arrays.asList(data_d1), normalUserId3, new FileAclParams(null, null), SET, ownerToken); thrown.expect(CatalogAuthorizationException.class); - catalogManager.getFileManager().get(studyFqn, data_d1, null, memberSessionId); + catalogManager.getFileManager().get(studyFqn, data_d1, null, normalToken3); } @Test public void readInheritedSharedFile() throws CatalogException { - DataResult file = catalogManager.getFileManager().get(studyFqn, data_d1_d2, null, externalSessionId); + DataResult file = catalogManager.getFileManager().get(studyFqn, data_d1_d2, null, normalToken3); assertEquals(1, file.getNumResults()); } @Test public void readExplicitlyForbiddenFile() throws CatalogException { thrown.expect(CatalogAuthorizationException.class); - catalogManager.getFileManager().get(studyFqn, data_d1_d2_d3, null, externalSessionId); + catalogManager.getFileManager().get(studyFqn, data_d1_d2_d3, null, normalToken3); } @Test public void readInheritedForbiddenFile() throws CatalogException { thrown.expect(CatalogAuthorizationException.class); - catalogManager.getFileManager().get(studyFqn, data_d1_d2_d3_d4, null, externalSessionId); + catalogManager.getFileManager().get(studyFqn, data_d1_d2_d3_d4, null, normalToken3); } @Test public void readExplicitlySharedFile() throws CatalogException { - DataResult file = catalogManager.getFileManager().get(studyFqn, data_d1_d2_d3_d4_txt, null, externalSessionId); + DataResult file = catalogManager.getFileManager().get(studyFqn, data_d1_d2_d3_d4_txt, null, normalToken3); assertEquals(1, file.getNumResults()); } @Test public void readNonSharedFile() throws CatalogException { thrown.expect(CatalogAuthorizationException.class); - catalogManager.getFileManager().get(studyFqn, data, null, externalSessionId); + catalogManager.getFileManager().get(studyFqn, data, null, normalToken3); } @Test public void readFileNoStudyMember() throws CatalogException, IOException { String newUser = "newUser"; - catalogManager.getUserManager().create(newUser, newUser, "asda@mail.com", TestParamConstants.PASSWORD, "org", 1000L, Account.AccountType.FULL, opencgaToken); - String sessionId = catalogManager.getUserManager().login(newUser, TestParamConstants.PASSWORD).getToken(); + catalogManager.getUserManager().create(newUser, newUser, "asda@mail.com", TestParamConstants.PASSWORD, organizationId, 1000L, ownerToken); + String sessionId = catalogManager.getUserManager().login(organizationId, newUser, TestParamConstants.PASSWORD).getToken(); thrown.expect(CatalogAuthorizationException.class); catalogManager.getFileManager().get(studyFqn, data, null, sessionId); } @@ -644,45 +475,43 @@ public void readFileNoStudyMember() throws CatalogException, IOException { public void readFileSharedForGroup() throws CatalogException, IOException { // Add a new user to a new group String newUser = "newUser"; - catalogManager.getUserManager().create(newUser, newUser, "asda@mail.com", TestParamConstants.PASSWORD, "org", 1000L, Account.AccountType.FULL, opencgaToken); - String sessionId = catalogManager.getUserManager().login(newUser, TestParamConstants.PASSWORD).getToken(); + catalogManager.getUserManager().create(newUser, newUser, "asda@mail.com", TestParamConstants.PASSWORD, organizationId, 1000L, ownerToken); + String sessionId = catalogManager.getUserManager().login(organizationId, newUser, TestParamConstants.PASSWORD).getToken(); String newGroup = "@external"; -// catalogManager.addUsersToGroup(studyFqn, "@external", newUser, ownerSessionId); - catalogManager.getStudyManager().createGroup(studyFqn, newGroup, Collections.singletonList(newUser), ownerSessionId); +// catalogManager.addUsersToGroup(studyFqn, "@external", newUser, token); + catalogManager.getStudyManager().createGroup(studyFqn, newGroup, Collections.singletonList(newUser), ownerToken); // Add the group to the locked role, so no permissions will be given StudyAclParams aclParams = new StudyAclParams("", AuthorizationManager.ROLE_LOCKED); - catalogManager.getStudyManager().updateAcl(Arrays.asList(studyFqn), newGroup, aclParams, ADD, ownerSessionId); + catalogManager.getStudyManager().updateAcl(studyFqn, newGroup, aclParams, ADD, ownerToken); // Specify all file permissions for that concrete file - fileManager.updateAcl(studyFqn, Arrays.asList(data_d1_d2_d3_d4), newGroup, new FileAclParams(null, ALL_FILE_PERMISSIONS), SET, - ownerSessionId); + catalogManager.getFileManager().updateAcl(studyFqn, Arrays.asList(data_d1_d2_d3_d4), newGroup, new FileAclParams(null, ALL_FILE_PERMISSIONS), SET, + ownerToken); catalogManager.getFileManager().get(studyFqn, data_d1_d2_d3_d4, null, sessionId); } @Test public void readFileForbiddenForUser() throws CatalogException { // Remove all permissions to the admin group in that folder - catalogManager.getStudyManager().createGroup(studyFqn, groupMember, Collections.singletonList(externalUser), ownerSessionId); - catalogManager.getStudyManager().updateAcl(Arrays.asList(studyFqn), groupMember, new StudyAclParams("", "admin"), SET, - ownerSessionId); - fileManager.updateAcl(studyFqn, Arrays.asList(data_d1_d2), externalUser, new FileAclParams(null, DENY_FILE_PERMISSIONS), SET, - ownerSessionId); + catalogManager.getStudyManager().updateAcl(studyFqn, restrictedGroup, new StudyAclParams("", AuthorizationManager.ROLE_ADMIN), SET, ownerToken); + catalogManager.getFileManager().updateAcl(studyFqn, Arrays.asList(data_d1_d2), normalUserId3, new FileAclParams(null, DENY_FILE_PERMISSIONS), SET, + ownerToken); thrown.expect(CatalogAuthorizationException.class); - catalogManager.getFileManager().get(studyFqn, data_d1_d2, null, externalSessionId); + catalogManager.getFileManager().get(studyFqn, data_d1_d2, null, normalToken3); } @Test public void readFileForbiddenForGroupSharedWithUser() throws CatalogException, IOException { // Add a new user to a new group String newUser = "newUser"; - catalogManager.getUserManager().create(newUser, newUser, "asda@mail.com", TestParamConstants.PASSWORD, "org", 1000L, Account.AccountType.FULL, opencgaToken); - String sessionId = catalogManager.getUserManager().login(ownerUser, TestParamConstants.PASSWORD).getToken(); + catalogManager.getUserManager().create(newUser, newUser, "asda@mail.com", TestParamConstants.PASSWORD, organizationId, 1000L, ownerToken); + String sessionId = catalogManager.getUserManager().login(organizationId, orgOwnerUserId, TestParamConstants.PASSWORD).getToken(); String newGroup = "@external"; -// catalogManager.addUsersToGroup(studyFqn, "@external", newUser, ownerSessionId); - catalogManager.getStudyManager().createGroup(studyFqn, newGroup, Collections.singletonList(newUser), ownerSessionId); +// catalogManager.addUsersToGroup(studyFqn, "@external", newUser, token); + catalogManager.getStudyManager().createGroup(studyFqn, newGroup, Collections.singletonList(newUser), ownerToken); // Add the group to the locked role, so no permissions will be given StudyAclParams aclParams = new StudyAclParams("", AuthorizationManager.ROLE_LOCKED); - catalogManager.getStudyManager().updateAcl(Arrays.asList(studyFqn), newGroup, aclParams, ADD, ownerSessionId); - fileManager.updateAcl(studyFqn, Arrays.asList(data_d1_d2), newGroup, new FileAclParams(null, ALL_FILE_PERMISSIONS), SET, ownerSessionId); + catalogManager.getStudyManager().updateAcl(studyFqn, newGroup, aclParams, ADD, ownerToken); + catalogManager.getFileManager().updateAcl(studyFqn, Arrays.asList(data_d1_d2), newGroup, new FileAclParams(null, ALL_FILE_PERMISSIONS), SET, ownerToken); DataResult file = catalogManager.getFileManager().get(studyFqn, data_d1_d2, null, sessionId); assertEquals(1, file.getNumResults()); } @@ -694,48 +523,48 @@ public void readFileForbiddenForGroupSharedWithUser() throws CatalogException, I @Test public void createFileByOwnerNoGroups() throws CatalogException { DataResult folder = catalogManager.getFileManager().createFolder(studyFqn, Paths.get("data/newFolder").toString(), - true, null, QueryOptions.empty(), ownerSessionId); + true, null, QueryOptions.empty(), ownerToken); assertEquals(1, folder.getNumResults()); } @Test public void createExplicitlySharedFile() throws CatalogException { catalogManager.getFileManager().createFolder(studyFqn, Paths.get("data/d1/folder/").toString(), false, null, - QueryOptions.empty(), externalSessionId); + QueryOptions.empty(), normalToken3); } @Test public void createInheritedSharedFile() throws CatalogException { catalogManager.getFileManager().createFolder(studyFqn, Paths.get("data/d1/d2/folder/").toString(), false, - null, QueryOptions.empty(), externalSessionId); + null, QueryOptions.empty(), normalToken3); } @Test public void createExplicitlyForbiddenFile() throws CatalogException { thrown.expect(CatalogAuthorizationException.class); catalogManager.getFileManager().createFolder(studyFqn, Paths.get("data/d1/d2/d3/folder/").toString(), false, - null, QueryOptions.empty(), externalSessionId); + null, QueryOptions.empty(), normalToken3); } @Test public void createInheritedForbiddenFile() throws CatalogException { thrown.expect(CatalogAuthorizationException.class); catalogManager.getFileManager().createFolder(studyFqn, Paths.get("data/d1/d2/d3/d4/folder/").toString(), false, - null, QueryOptions.empty(), externalSessionId); + null, QueryOptions.empty(), normalToken3); } @Test public void createNonSharedFile() throws CatalogException { thrown.expect(CatalogAuthorizationException.class); catalogManager.getFileManager().createFolder(studyFqn, Paths.get("folder/").toString(), false, null, - QueryOptions.empty(), externalSessionId); + QueryOptions.empty(), normalToken3); } @Test public void createFileNoStudyMember() throws CatalogException, IOException { String newUser = "newUser"; - catalogManager.getUserManager().create(newUser, newUser, "asda@mail.com", TestParamConstants.PASSWORD, "org", 1000L, Account.AccountType.FULL, opencgaToken); - String sessionId = catalogManager.getUserManager().login(newUser, TestParamConstants.PASSWORD).getToken(); + catalogManager.getUserManager().create(newUser, newUser, "asda@mail.com", TestParamConstants.PASSWORD, organizationId, 1000L, ownerToken); + String sessionId = catalogManager.getUserManager().login(organizationId, newUser, TestParamConstants.PASSWORD).getToken(); thrown.expect(CatalogAuthorizationException.class); catalogManager.getFileManager().createFolder(studyFqn, Paths.get("data/my_folder/").toString(), false, null, QueryOptions.empty(), sessionId); @@ -747,100 +576,101 @@ public void createFileNoStudyMember() throws CatalogException, IOException { @Test public void readSampleOwnerUser() throws CatalogException { - DataResult sample = catalogManager.getSampleManager().get(studyFqn, smp1.getId(), null, ownerSessionId); + DataResult sample = catalogManager.getSampleManager().get(studyFqn, s_1Id, null, ownerToken); assertEquals(1, sample.getNumResults()); - sample = catalogManager.getSampleManager().get(studyFqn, smp2.getId(), null, ownerSessionId); + sample = catalogManager.getSampleManager().get(studyFqn, s_2Id, null, ownerToken); assertEquals(1, sample.getNumResults()); - sample = catalogManager.getSampleManager().get(studyFqn, smp3.getId(), null, ownerSessionId); + sample = catalogManager.getSampleManager().get(studyFqn, s_3Id, null, ownerToken); assertEquals(1, sample.getNumResults()); // Owner always have access even if he has been removed all the permissions StudyAclParams aclParams = new StudyAclParams("", null); - catalogManager.getStudyManager().updateAcl(Collections.singletonList(studyFqn), ownerUser, aclParams, ADD, ownerSessionId); - catalogManager.getSampleManager().updateAcl(studyFqn, Collections.singletonList(smp1.getId()), ownerUser, noSamplePermissions, SET, - ownerSessionId); + catalogManager.getStudyManager().updateAcl(studyFqn, orgOwnerUserId, aclParams, ADD, ownerToken); + catalogManager.getSampleManager().updateAcl(studyFqn, Collections.singletonList(s_1Id), orgOwnerUserId, noSamplePermissions, SET, + ownerToken); - sample = catalogManager.getSampleManager().get(studyFqn, smp1.getId(), null, ownerSessionId); + sample = catalogManager.getSampleManager().get(studyFqn, s_1Id, null, ownerToken); assertEquals(1, sample.getNumResults()); } @Test public void readSampleExplicitShared() throws CatalogException { - DataResult sample = catalogManager.getSampleManager().get(studyFqn, smp1.getId(), null, externalSessionId); + DataResult sample = catalogManager.getSampleManager().get(studyFqn, s_1Id, null, normalToken3); assertEquals(1, sample.getNumResults()); } @Test public void readSampleExplicitUnshared() throws CatalogException { - DataResult sample = catalogManager.getSampleManager().get(studyFqn, smp1.getId(), null, externalSessionId); + DataResult sample = catalogManager.getSampleManager().get(studyFqn, s_1Id, null, normalToken3); assertEquals(1, sample.getNumResults()); - catalogManager.getAuthorizationManager().removeAcls(Collections.singletonList(externalUser), - new AuthorizationManager.CatalogAclParams(Collections.singletonList(smp1.getUid()), null, Enums.Resource.SAMPLE)); + catalogManager.getAuthorizationManager().removeAcls(organizationId, Collections.singletonList(restrictedGroup), + new AuthorizationManager.CatalogAclParams(Collections.singletonList(s_1Uid), null, Enums.Resource.SAMPLE)); thrown.expect(CatalogAuthorizationException.class); - catalogManager.getSampleManager().get(studyFqn, smp1.getId(), null, externalSessionId); + catalogManager.getSampleManager().get(studyFqn, s_1Id, null, normalToken3); } @Test public void readSampleNoShared() throws CatalogException { thrown.expect(CatalogAuthorizationException.class); - catalogManager.getSampleManager().get(studyFqn, smp2.getId(), null, externalSessionId); + catalogManager.getSampleManager().get(studyFqn, s_2Id, null, normalToken3); } @Test public void readSampleExplicitForbidden() throws CatalogException { thrown.expect(CatalogAuthorizationException.class); - catalogManager.getSampleManager().get(studyFqn, smp3.getId(), null, externalSessionId); + catalogManager.getSampleManager().get(studyFqn, s_3Id, null, normalToken3); } @Test - public void readSampleExternalUser() throws CatalogException, IOException { + public void readSampleExternalUser() throws CatalogException { String newUser = "newUser"; - catalogManager.getUserManager().create(newUser, newUser, "asda@mail.com", TestParamConstants.PASSWORD, "org", 1000L, Account.AccountType.FULL, opencgaToken); - String sessionId = catalogManager.getUserManager().login(newUser, TestParamConstants.PASSWORD).getToken(); + catalogManager.getUserManager().create(newUser, newUser, "asda@mail.com", TestParamConstants.PASSWORD, organizationId, 1000L, + ownerToken); + String sessionId = catalogManager.getUserManager().login(organizationId, newUser, TestParamConstants.PASSWORD).getToken(); thrown.expect(CatalogAuthorizationException.class); - catalogManager.getSampleManager().get(studyFqn, smp2.getId(), null, sessionId); + catalogManager.getSampleManager().get(studyFqn, s_2Id, null, sessionId); } @Test public void readSampleAdminUser() throws CatalogException { - DataResult sample = catalogManager.getSampleManager().get(studyFqn, smp1.getId(), null, studyAdmin1SessionId); + DataResult sample = catalogManager.getSampleManager().get(studyFqn, s_1Id, null, orgAdminToken1); assertEquals(1, sample.getNumResults()); - sample = catalogManager.getSampleManager().get(studyFqn, smp3.getId(), null, studyAdmin1SessionId); + sample = catalogManager.getSampleManager().get(studyFqn, s_3Id, null, orgAdminToken1); assertEquals(1, sample.getNumResults()); } @Test public void readSampleForbiddenForExternalUser() throws CatalogException { - catalogManager.getSampleManager().updateAcl(String.valueOf(studyFqn), Arrays.asList(smp2.getId()), externalUser, - new SampleAclParams(null, null, null, null, ""), SET, ownerSessionId); + catalogManager.getSampleManager().updateAcl(String.valueOf(studyFqn), Arrays.asList(s_2Id), normalUserId3, + new SampleAclParams(null, null, null, null, ""), SET, ownerToken); thrown.expect(CatalogAuthorizationException.class); - catalogManager.getSampleManager().get(studyFqn, smp2.getId(), null, externalSessionId); + catalogManager.getSampleManager().get(studyFqn, s_2Id, null, normalToken3); } @Test public void readSampleSharedForGroup() throws CatalogException, IOException { // Add a new user to a new group String newUser = "newUser"; - catalogManager.getUserManager().create(newUser, newUser, "asda@mail.com", TestParamConstants.PASSWORD, "org", 1000L, Account.AccountType.FULL, opencgaToken); - String sessionId = catalogManager.getUserManager().login(ownerUser, TestParamConstants.PASSWORD).getToken(); + catalogManager.getUserManager().create(newUser, newUser, "asda@mail.com", TestParamConstants.PASSWORD, organizationId, 1000L, ownerToken); + String sessionId = catalogManager.getUserManager().login(organizationId, orgOwnerUserId, TestParamConstants.PASSWORD).getToken(); String newGroup = "@external"; -// catalogManager.addUsersToGroup(studyFqn, "@external", newUser, ownerSessionId); - catalogManager.getStudyManager().createGroup(studyFqn, newGroup, Collections.singletonList(newUser), ownerSessionId); +// catalogManager.addUsersToGroup(studyFqn, "@external", newUser, token); + catalogManager.getStudyManager().createGroup(studyFqn, newGroup, Collections.singletonList(newUser), ownerToken); // Add the group to the locked role, so no permissions will be given StudyAclParams aclParams = new StudyAclParams("", AuthorizationManager.ROLE_LOCKED); - catalogManager.getStudyManager().updateAcl(Collections.singletonList(studyFqn), newGroup, aclParams, ADD, ownerSessionId); + catalogManager.getStudyManager().updateAcl(studyFqn, newGroup, aclParams, ADD, ownerToken); // Share the sample with the group - catalogManager.getSampleManager().updateAcl(studyFqn, Collections.singletonList(smp4.getId()), newGroup, allSamplePermissions, SET, - ownerSessionId); + catalogManager.getSampleManager().updateAcl(studyFqn, Collections.singletonList(s_4Id), newGroup, allSamplePermissions, SET, + ownerToken); - DataResult sample = catalogManager.getSampleManager().get(studyFqn, smp4.getId(), null, sessionId); + DataResult sample = catalogManager.getSampleManager().get(studyFqn, s_4Id, null, sessionId); assertEquals(1, sample.getNumResults()); } @Test public void readSampleSharedForOthers() throws CatalogException { - DataResult sample = catalogManager.getSampleManager().get(studyFqn, smp6.getId(), null, externalSessionId); + DataResult sample = catalogManager.getSampleManager().get(studyFqn, s_6Id, null, normalToken3); assertEquals(1, sample.getNumResults()); } @@ -849,130 +679,133 @@ public void readSampleSharedForOthers() throws CatalogException { public void readSampleSharedForOthersNotWithStudyPermissions() throws CatalogException, IOException { // Add a new user to a new group String newUser = "newUser"; - catalogManager.getUserManager().create(newUser, newUser, "asda@mail.com", TestParamConstants.PASSWORD, "org", 1000L, Account.AccountType.FULL, opencgaToken); - String sessionId = catalogManager.getUserManager().login(ownerUser, TestParamConstants.PASSWORD).getToken(); + catalogManager.getUserManager().create(newUser, newUser, "asda@mail.com", TestParamConstants.PASSWORD, organizationId, 1000L, + ownerToken); + String sessionId = catalogManager.getUserManager().login(organizationId, orgOwnerUserId, TestParamConstants.PASSWORD).getToken(); catalogManager.getStudyManager().updateGroup(studyFqn, "@members", ParamUtils.BasicUpdateAction.ADD, - new GroupUpdateParams(Collections.singletonList(newUser)), ownerSessionId); + new GroupUpdateParams(Collections.singletonList(newUser)), ownerToken); - DataResult sample = catalogManager.getSampleManager().get(studyFqn, smp6.getId(), null, sessionId); + DataResult sample = catalogManager.getSampleManager().get(studyFqn, s_6Id, null, sessionId); assertEquals(1, sample.getNumResults()); } @Test public void adminShareSampleWithOtherUser() throws CatalogException { - catalogManager.getSampleManager().updateAcl(studyFqn, Collections.singletonList(smp4.getId()), externalUser, allSamplePermissions, - SET, studyAdmin1SessionId); - DataResult sample = catalogManager.getSampleManager().get(studyFqn, smp4.getId(), null, externalSessionId); + catalogManager.getSampleManager().updateAcl(studyFqn, Collections.singletonList(s_4Id), normalUserId3, allSamplePermissions, + SET, orgAdminToken1); + DataResult sample = catalogManager.getSampleManager().get(studyFqn, s_4Id, null, normalToken3); assertEquals(1, sample.getNumResults()); } @Test public void readAllSamplesOwner() throws CatalogException { - Map sampleMap = catalogManager.getSampleManager().search(studyFqn, new Query(), new QueryOptions(), ownerSessionId) + Map sampleMap = catalogManager.getSampleManager().search(studyFqn, new Query(), new QueryOptions(), ownerToken) .getResults().stream().collect(Collectors.toMap(Sample::getUid, f -> f)); - assertTrue(sampleMap.containsKey(smp1.getUid())); - assertTrue(sampleMap.containsKey(smp2.getUid())); - assertTrue(sampleMap.containsKey(smp3.getUid())); + assertTrue(sampleMap.containsKey(s_1Uid)); + assertTrue(sampleMap.containsKey(s_2Uid)); + assertTrue(sampleMap.containsKey(s_3Uid)); } // @Test // public void readAllSamplesAdmin() throws CatalogException { -// Map sampleMap = catalogManager.getSampleManager().get(studyFqn, new Query(), new QueryOptions(), studyAdmin1SessionId) +// Map sampleMap = catalogManager.getSampleManager().get(studyFqn, new Query(), new QueryOptions(), adminToken1) // .getResults().stream().collect(Collectors.toMap(Sample::getId, f -> f)); // -// assertTrue(sampleMap.containsKey(smp1.getId())); -// assertFalse(sampleMap.containsKey(smp2.getId())); -// assertTrue(sampleMap.containsKey(smp3.getId())); +// assertTrue(sampleMap.containsKey(s_1.getId())); +// assertFalse(sampleMap.containsKey(s_2.getId())); +// assertTrue(sampleMap.containsKey(s_3.getId())); // } @Test public void readAllSamplesMember() throws CatalogException { - Map sampleMap = catalogManager.getSampleManager().search(studyFqn, new Query(), new QueryOptions(), externalSessionId) + Map sampleMap = catalogManager.getSampleManager().search(studyFqn, new Query(), new QueryOptions(), normalToken3) .getResults().stream().collect(Collectors.toMap(Sample::getUid, f -> f)); - assertTrue(sampleMap.containsKey(smp1.getUid())); - assertFalse(sampleMap.containsKey(smp2.getUid())); - assertFalse(sampleMap.containsKey(smp3.getUid())); + assertTrue(sampleMap.containsKey(s_1Uid)); + assertFalse(sampleMap.containsKey(s_2Uid)); + assertFalse(sampleMap.containsKey(s_3Uid)); } @Test public void aclQuery() throws CatalogException { + catalogManager.getStudyManager().updateAcl(studyFqn, restrictedGroup, new StudyAclParams(null, AuthorizationManager.ROLE_ANALYST), + ADD, ownerToken); SampleManager sampleManager = catalogManager.getSampleManager(); - sampleManager.create(studyFqn, new Sample().setId("s1"), QueryOptions.empty(), ownerSessionId); - sampleManager.create(studyFqn, new Sample().setId("s2"), QueryOptions.empty(), ownerSessionId); - sampleManager.create(studyFqn, new Sample().setId("s3"), QueryOptions.empty(), ownerSessionId); - sampleManager.create(studyFqn, new Sample().setId("s4"), QueryOptions.empty(), ownerSessionId); - sampleManager.create(studyFqn, new Sample().setId("s5"), QueryOptions.empty(), ownerSessionId); + sampleManager.create(studyFqn, new Sample().setId("s1"), QueryOptions.empty(), ownerToken); + sampleManager.create(studyFqn, new Sample().setId("s2"), QueryOptions.empty(), ownerToken); + sampleManager.create(studyFqn, new Sample().setId("s3"), QueryOptions.empty(), ownerToken); + sampleManager.create(studyFqn, new Sample().setId("s4"), QueryOptions.empty(), ownerToken); + sampleManager.create(studyFqn, new Sample().setId("s5"), QueryOptions.empty(), ownerToken); - sampleManager.updateAcl(studyFqn, Collections.singletonList("s1"), memberUser, - new SampleAclParams(null, null, null, null, SamplePermissions.DELETE.name()), SET, ownerSessionId); - sampleManager.updateAcl(studyFqn, Collections.singletonList("s2"), memberUser, + sampleManager.updateAcl(studyFqn, Collections.singletonList("s1"), normalUserId3, + new SampleAclParams(null, null, null, null, SamplePermissions.DELETE.name()), SET, ownerToken); + sampleManager.updateAcl(studyFqn, Collections.singletonList("s2"), normalUserId3, new SampleAclParams(null, null, null, null, SamplePermissions.VIEW_ANNOTATIONS.name()), SET, - ownerSessionId); - sampleManager.updateAcl(studyFqn, Collections.singletonList("s3"), memberUser, new SampleAclParams( - null, null, null, null, SamplePermissions.DELETE.name()), SET, ownerSessionId); - sampleManager.updateAcl(studyFqn, Collections.singletonList("s4"), memberUser, new SampleAclParams( - null, null, null, null, SamplePermissions.VIEW.name()), SET, ownerSessionId); + ownerToken); + sampleManager.updateAcl(studyFqn, Collections.singletonList("s3"), normalUserId3, new SampleAclParams( + null, null, null, null, SamplePermissions.DELETE.name()), SET, ownerToken); + sampleManager.updateAcl(studyFqn, Collections.singletonList("s4"), normalUserId3, new SampleAclParams( + null, null, null, null, SamplePermissions.VIEW.name()), SET, ownerToken); QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.ID.key()); // member user wants to know the samples for which he has DELETE permissions - Query query = new Query(ParamConstants.ACL_PARAM, memberUser + ":" + SamplePermissions.DELETE.name()); - List results = sampleManager.search(studyFqn, query, options, memberSessionId).getResults(); - assertEquals(3, results.stream().map(Sample::getId).collect(Collectors.toSet()).size()); - assertTrue(Arrays.asList("s1", "s3", "smp6").containsAll(results.stream().map(Sample::getId).collect(Collectors.toSet()))); + Query query = new Query(ParamConstants.ACL_PARAM, normalUserId3 + ":" + SamplePermissions.DELETE.name()); + List results = sampleManager.search(studyFqn, query, options, normalToken3).getResults(); + assertEquals(4, results.stream().map(Sample::getId).collect(Collectors.toSet()).size()); + assertTrue(Arrays.asList("s1", "s3", s_1Id, s_6Id).containsAll(results.stream().map(Sample::getId).collect(Collectors.toSet()))); // member user wants to know the samples for which he has UPDATE permissions // He should have UPDATE permissions in the same samples where he has DELETE permissions - query = new Query(ParamConstants.ACL_PARAM, memberUser + ":" + SamplePermissions.WRITE.name()); - results = sampleManager.search(studyFqn, query, options, memberSessionId).getResults(); + query = new Query(ParamConstants.ACL_PARAM, normalUserId3 + ":" + SamplePermissions.WRITE.name()); + results = sampleManager.search(studyFqn, query, options, normalToken3).getResults(); System.out.println(results.stream().map(Sample::getId).collect(Collectors.toSet())); - assertEquals(8, results.stream().map(Sample::getId).collect(Collectors.toSet()).size()); - assertTrue(Arrays.asList("s1", "s3", "s5", "smp1", "smp3", "smp4", "smp5", "smp6") + assertEquals(9, results.stream().map(Sample::getId).collect(Collectors.toSet()).size()); + assertTrue(Arrays.asList("s1", "s3", "s5", s_1Id, s_4Id, s_6Id, s_7Id, s_8Id, s_9Id) .containsAll(results.stream().map(Sample::getId).collect(Collectors.toSet()))); // Owner user wants to know the samples for which member user has DELETE permissions - query = new Query(ParamConstants.ACL_PARAM, memberUser + ":" + SamplePermissions.DELETE.name()); - results = sampleManager.search(studyFqn, query, options, ownerSessionId).getResults(); - assertEquals(3, results.stream().map(Sample::getId).collect(Collectors.toSet()).size()); - assertTrue(Arrays.asList("s1", "s3", "smp6").containsAll(results.stream().map(Sample::getId).collect(Collectors.toSet()))); + query = new Query(ParamConstants.ACL_PARAM, normalUserId3 + ":" + SamplePermissions.DELETE.name()); + results = sampleManager.search(studyFqn, query, options, ownerToken).getResults(); + assertEquals(4, results.stream().map(Sample::getId).collect(Collectors.toSet()).size()); + assertTrue(Arrays.asList("s1", "s3", s_1Id, s_6Id).containsAll(results.stream().map(Sample::getId).collect(Collectors.toSet()))); // member user wants to know the samples for which other user has DELETE permissions - query = new Query(ParamConstants.ACL_PARAM, ownerUser + ":" + SamplePermissions.DELETE.name()); + query = new Query(ParamConstants.ACL_PARAM, orgOwnerUserId + ":" + SamplePermissions.DELETE.name()); thrown.expect(CatalogAuthorizationException.class); thrown.expectMessage("Only study owners or admins"); - sampleManager.search(studyFqn, query, options, memberSessionId); + sampleManager.search(studyFqn, query, options, normalToken3); } @Test public void getAclsTest() throws CatalogException { StudyManager studyManager = catalogManager.getStudyManager(); - studyManager.createGroup(studyFqn, "group1", Collections.singletonList(externalUser), ownerSessionId); - studyManager.createGroup(studyFqn, "group2", Collections.singletonList(externalUser), ownerSessionId); + studyManager.createGroup(studyFqn, "group1", Collections.singletonList(normalUserId3), ownerToken); + studyManager.createGroup(studyFqn, "group2", Collections.singletonList(normalUserId3), ownerToken); - studyManager.updateAcl(Collections.singletonList(studyFqn), "@group1", - new StudyAclParams(StudyPermissions.Permissions.VIEW_COHORTS.name(), ""), ADD, ownerSessionId); - studyManager.updateAcl(Collections.singletonList(studyFqn), externalUser, - new StudyAclParams(StudyPermissions.Permissions.VIEW_FILES.name(), ""), ADD, ownerSessionId); + studyManager.updateAcl(studyFqn, "@group1", + new StudyAclParams(StudyPermissions.Permissions.VIEW_COHORTS.name(), ""), ADD, ownerToken); + studyManager.updateAcl(studyFqn, normalUserId3, + new StudyAclParams(StudyPermissions.Permissions.VIEW_FILES.name(), ""), ADD, ownerToken); - OpenCGAResult> acls = studyManager.getAcls(Collections.singletonList(studyFqn), externalUser, false, - externalSessionId); + OpenCGAResult> acls = studyManager.getAcls(Collections.singletonList(studyFqn), normalUserId3, false, + normalToken3); assertEquals(1, acls.getNumResults()); assertEquals(1, acls.first().getAcl().size()); - assertEquals(externalUser, acls.first().getAcl().get(0).getMember()); + assertEquals(normalUserId3, acls.first().getAcl().get(0).getMember()); assertEquals(1, acls.first().getAcl().get(0).getGroups().stream().filter(g -> "@group1".equals(g.getId())).count()); -// assertTrue(acls.first().keySet().containsAll(Arrays.asList(externalUser, "@group1"))); +// assertTrue(acls.first().keySet().containsAll(Arrays.asList(normalUserId3, "@group1"))); - studyManager.updateAcl(Collections.singletonList(studyFqn), "@group2", - new StudyAclParams(StudyPermissions.Permissions.VIEW_SAMPLES.name(), ""), ADD, ownerSessionId); - acls = studyManager.getAcls(Collections.singletonList(studyFqn), externalUser, false, - externalSessionId); + studyManager.updateAcl(studyFqn, "@group2", + new StudyAclParams(StudyPermissions.Permissions.VIEW_SAMPLES.name(), ""), ADD, ownerToken); + acls = studyManager.getAcls(Collections.singletonList(studyFqn), normalUserId3, false, + normalToken3); assertEquals(1, acls.getNumResults()); assertEquals(1, acls.first().getAcl().size()); - assertEquals(externalUser, acls.first().getAcl().get(0).getMember()); + assertEquals(normalUserId3, acls.first().getAcl().get(0).getMember()); assertEquals(2, acls.first().getAcl().get(0).getGroups().stream() .filter(g -> "@group1".equals(g.getId()) || "@group2".equals(g.getId())) .count()); @@ -994,8 +827,8 @@ public void getAclsTest() throws CatalogException { @Test public void readCohort() throws CatalogException { - assertEquals(1, catalogManager.getCohortManager().search(studyFqn, new Query(), null, ownerSessionId).getNumResults()); - assertEquals(0, catalogManager.getCohortManager().search(studyFqn, new Query(), null, externalSessionId).getNumResults()); + assertEquals(1, catalogManager.getCohortManager().search(studyFqn, new Query(), null, ownerToken).getNumResults()); + assertEquals(0, catalogManager.getCohortManager().search(studyFqn, new Query(), null, normalToken3).getNumResults()); } /*--------------------------*/ @@ -1004,25 +837,25 @@ public void readCohort() throws CatalogException { @Test public void readIndividualByReadingSomeSample() throws CatalogException { - catalogManager.getIndividualManager().get(null, ind1, null, memberSessionId); + catalogManager.getIndividualManager().get(null, ind1, null, normalToken3); } @Test public void readIndividualForbidden() throws CatalogException { thrown.expect(CatalogAuthorizationException.class); - catalogManager.getIndividualManager().get(null, ind2, null, externalSessionId); + catalogManager.getIndividualManager().get(null, ind2, null, normalToken3); } @Test public void readIndividualStudyManager() throws CatalogException { - catalogManager.getIndividualManager().get(null, ind2, null, studyAdmin1SessionId); + catalogManager.getIndividualManager().get(studyFqn, ind2, null, orgAdminToken1); } @Test public void readAllIndividuals() throws CatalogException { - assertEquals(2, catalogManager.getIndividualManager().search(studyFqn, new Query(), null, ownerSessionId).getNumResults()); - assertEquals(2, catalogManager.getIndividualManager().search(studyFqn, new Query(), null, studyAdmin1SessionId).getNumResults()); - assertEquals(0, catalogManager.getIndividualManager().search(studyFqn, new Query(), null, externalSessionId).getNumResults()); + assertEquals(2, catalogManager.getIndividualManager().search(studyFqn, new Query(), null, ownerToken).getNumResults()); + assertEquals(2, catalogManager.getIndividualManager().search(studyFqn, new Query(), null, orgAdminToken1).getNumResults()); + assertEquals(1, catalogManager.getIndividualManager().search(studyFqn, new Query(), null, normalToken3).getNumResults()); } /*--------------------------*/ @@ -1034,20 +867,20 @@ public void getAllJobs() throws CatalogException { Job job = new Job() .setId("job1") .setOutDir(new File().setPath(data_d1_d2)); - long job1 = catalogManager.getJobManager().create(studyFqn, job, INCLUDE_RESULT, ownerSessionId).first().getUid(); + long job1 = catalogManager.getJobManager().create(studyFqn, job, INCLUDE_RESULT, ownerToken).first().getUid(); job.setId("job2"); - long job2 = catalogManager.getJobManager().create(studyFqn, job, INCLUDE_RESULT, ownerSessionId).first().getUid(); + long job2 = catalogManager.getJobManager().create(studyFqn, job, INCLUDE_RESULT, ownerToken).first().getUid(); job.setId("job3"); - long job3 = catalogManager.getJobManager().create(studyFqn, job, INCLUDE_RESULT, ownerSessionId).first().getUid(); + long job3 = catalogManager.getJobManager().create(studyFqn, job, INCLUDE_RESULT, ownerToken).first().getUid(); job.setId("job4"); - long job4 = catalogManager.getJobManager().create(studyFqn, job, INCLUDE_RESULT, ownerSessionId).first().getUid(); + long job4 = catalogManager.getJobManager().create(studyFqn, job, INCLUDE_RESULT, ownerToken).first().getUid(); - checkGetAllJobs(Arrays.asList(job1, job2, job3, job4), ownerSessionId); //Owner can see everything - checkGetAllJobs(Collections.emptyList(), externalSessionId); //Can't see inside data_d1_d2_d3 + checkGetAllJobs(Arrays.asList(job1, job2, job3, job4), ownerToken); //Owner can see everything + checkGetAllJobs(Collections.emptyList(), normalToken3); //Can't see inside data_d1_d2_d3 } private void checkGetAllJobs(Collection expectedJobs, String sessionId) throws CatalogException { @@ -1060,7 +893,7 @@ private void checkGetAllJobs(Collection expectedJobs, String sessionId) th /////////// Aux methods private Map getGroupMap() throws CatalogException { - return catalogManager.getStudyManager().get(studyFqn, null, ownerSessionId).first().getGroups().stream() + return catalogManager.getStudyManager().get(studyFqn, null, ownerToken).first().getGroups().stream() .collect(Collectors.toMap(Group::getId, f -> f)); } diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/AbstractMongoDBAdaptorTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/AbstractMongoDBAdaptorTest.java new file mode 100644 index 00000000000..632b13694cc --- /dev/null +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/AbstractMongoDBAdaptorTest.java @@ -0,0 +1,90 @@ +package org.opencb.opencga.catalog.db.mongodb; + +import org.junit.After; +import org.junit.Before; +import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.catalog.db.api.*; +import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; +import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogParameterException; +import org.opencb.opencga.catalog.managers.AbstractManagerTest; +import org.opencb.opencga.core.models.individual.Individual; +import org.opencb.opencga.core.models.job.Job; +import org.opencb.opencga.core.models.project.Project; +import org.opencb.opencga.core.models.sample.Sample; +import org.opencb.opencga.core.models.study.Study; + +public class AbstractMongoDBAdaptorTest extends AbstractManagerTest { + + protected MongoDBAdaptorFactory dbAdaptorFactory; + + protected UserMongoDBAdaptor catalogUserDBAdaptor; + protected ProjectMongoDBAdaptor catalogProjectDBAdaptor; + protected FileMongoDBAdaptor catalogFileDBAdaptor; + protected SampleMongoDBAdaptor catalogSampleDBAdaptor; + protected JobMongoDBAdaptor catalogJobDBAdaptor; + protected StudyMongoDBAdaptor catalogStudyDBAdaptor; + protected IndividualMongoDBAdaptor catalogIndividualDBAdaptor; + protected PanelMongoDBAdaptor catalogPanelDBAdaptor; + + @Before + public void setUp() throws Exception { + super.setUp(); + dbAdaptorFactory = new MongoDBAdaptorFactory(catalogManager.getConfiguration(), catalogManager.getIoManagerFactory()); + catalogUserDBAdaptor = (UserMongoDBAdaptor) dbAdaptorFactory.getCatalogUserDBAdaptor(organizationId); + catalogStudyDBAdaptor = (StudyMongoDBAdaptor) dbAdaptorFactory.getCatalogStudyDBAdaptor(organizationId); + catalogProjectDBAdaptor = (ProjectMongoDBAdaptor) dbAdaptorFactory.getCatalogProjectDbAdaptor(organizationId); + catalogFileDBAdaptor = (FileMongoDBAdaptor) dbAdaptorFactory.getCatalogFileDBAdaptor(organizationId); + catalogSampleDBAdaptor = (SampleMongoDBAdaptor) dbAdaptorFactory.getCatalogSampleDBAdaptor(organizationId); + catalogJobDBAdaptor = (JobMongoDBAdaptor) dbAdaptorFactory.getCatalogJobDBAdaptor(organizationId); + catalogIndividualDBAdaptor = (IndividualMongoDBAdaptor) dbAdaptorFactory.getCatalogIndividualDBAdaptor(organizationId); + catalogPanelDBAdaptor = (PanelMongoDBAdaptor) dbAdaptorFactory.getCatalogPanelDBAdaptor(organizationId); + } + + @After + public void after() { + dbAdaptorFactory.close(); + } + + protected Sample getSample(long studyUid, String sampleId) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + return getSample(studyUid, sampleId, QueryOptions.empty()); + } + + protected Sample getSample(long studyUid, String sampleId, QueryOptions options) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + Query query = new Query() + .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) + .append(SampleDBAdaptor.QueryParams.ID.key(), sampleId); + return catalogSampleDBAdaptor.get(query, options).first(); + } + + protected Individual getIndividual(long studyUid, String individualId) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + Query query = new Query() + .append(IndividualDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) + .append(IndividualDBAdaptor.QueryParams.ID.key(), individualId); + return catalogIndividualDBAdaptor.get(query, QueryOptions.empty()).first(); + } + + protected Job getJob(long studyUid, String jobId) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + Query query = new Query() + .append(JobDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) + .append(JobDBAdaptor.QueryParams.ID.key(), jobId); + return catalogJobDBAdaptor.get(query, QueryOptions.empty()).first(); + } + + Project getProject(String projectId) throws CatalogDBException { + Query query = new Query() + .append(ProjectDBAdaptor.QueryParams.ID.key(), projectId); + return catalogProjectDBAdaptor.get(query, QueryOptions.empty()).first(); + } + + Study getStudy(long projectUid, String studyId) throws CatalogDBException { + Query query = new Query() + .append(StudyDBAdaptor.QueryParams.PROJECT_UID.key(), projectUid) + .append(StudyDBAdaptor.QueryParams.ID.key(), studyId); + return catalogStudyDBAdaptor.get(query, QueryOptions.empty()).first(); + } +} diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/AuditMongoDBAdaptorTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/AuditMongoDBAdaptorTest.java index 3a2cf146184..228ca61f98a 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/AuditMongoDBAdaptorTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/AuditMongoDBAdaptorTest.java @@ -16,21 +16,12 @@ package org.opencb.opencga.catalog.db.mongodb; -import org.apache.commons.lang3.StringUtils; -import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; -import org.opencb.commons.datastore.core.DataStoreServerAddress; import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.commons.datastore.mongodb.MongoDBConfiguration; -import org.opencb.commons.datastore.mongodb.MongoDataStore; -import org.opencb.commons.datastore.mongodb.MongoDataStoreManager; -import org.opencb.opencga.catalog.io.IOManagerFactory; -import org.opencb.opencga.core.models.audit.AuditRecord; -import org.opencb.opencga.catalog.db.api.AuditDBAdaptor; import org.opencb.opencga.catalog.utils.UuidUtils; import org.opencb.opencga.core.common.TimeUtils; -import org.opencb.opencga.core.config.Configuration; +import org.opencb.opencga.core.models.audit.AuditRecord; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.testclassification.duration.MediumTests; @@ -40,61 +31,25 @@ * @author Jacobo Coll <jacobo167@gmail.com> */ @Category(MediumTests.class) -public class AuditMongoDBAdaptorTest { - - private AuditDBAdaptor auditDbAdaptor; - - @Before - public void beforeClass() throws Exception { - Configuration configuration = Configuration.load(getClass().getResource("/configuration-test.yml") - .openStream()); - - DataStoreServerAddress dataStoreServerAddress = new DataStoreServerAddress( - configuration.getCatalog().getDatabase().getHosts().get(0).split(":")[0], 27017); - - MongoDBConfiguration mongoDBConfiguration = MongoDBConfiguration.builder() - .add("username", configuration.getCatalog().getDatabase().getUser()) - .add("password", configuration.getCatalog().getDatabase().getPassword()) - .add("authenticationDatabase", configuration.getCatalog().getDatabase().getOptions().get("authenticationDatabase")) - .build(); - -// String database = catalogConfiguration.getDatabase().getDatabase(); - String database; - if(StringUtils.isNotEmpty(configuration.getDatabasePrefix())) { - if (!configuration.getDatabasePrefix().endsWith("_")) { - database = configuration.getDatabasePrefix() + "_catalog"; - } else { - database = configuration.getDatabasePrefix() + "catalog"; - } - } else { - database = "opencga_test_catalog"; - } - - /** - * Database is cleared before each execution - */ -// clearDB(dataStoreServerAddress, mongoCredentials); - MongoDataStoreManager mongoManager = new MongoDataStoreManager(dataStoreServerAddress.getHost(), dataStoreServerAddress.getPort()); - MongoDataStore db = mongoManager.get(database); - db.getDb().drop(); - - auditDbAdaptor = new MongoDBAdaptorFactory(configuration, new IOManagerFactory()).getCatalogAuditDbAdaptor(); - } +public class AuditMongoDBAdaptorTest extends AbstractMongoDBAdaptorTest { @Test public void testInsertAuditRecord() throws Exception { - auditDbAdaptor.insertAuditRecord(new AuditRecord(UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT), - UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT), "user", "api", Enums.Action.CREATE, - Enums.Resource.SAMPLE, "sampleId", "sampleUuid", "studyId", "studyUuid", new ObjectMap(), - new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS), TimeUtils.getDate(), new ObjectMap())); - auditDbAdaptor.insertAuditRecord(new AuditRecord(UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT), - UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT), "user", "api", Enums.Action.CREATE, - Enums.Resource.SAMPLE, "sampleId2", "sampleUuid2", "studyId", "studyUuid", new ObjectMap(), - new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS), TimeUtils.getDate(), new ObjectMap())); - auditDbAdaptor.insertAuditRecord(new AuditRecord(UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT), - UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT), "user", "api", Enums.Action.CREATE, - Enums.Resource.SAMPLE, "sampleId3", "sampleUuid3", "studyId", "studyUuid", new ObjectMap(), - new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS), TimeUtils.getDate(), new ObjectMap())); + dbAdaptorFactory.getCatalogAuditDbAdaptor(organizationId) + .insertAuditRecord(new AuditRecord(UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT), + UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT), "user", "api", Enums.Action.CREATE, + Enums.Resource.SAMPLE, "sampleId", "sampleUuid", "studyId", "studyUuid", new ObjectMap(), + new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS), TimeUtils.getDate(), new ObjectMap())); + dbAdaptorFactory.getCatalogAuditDbAdaptor(organizationId) + .insertAuditRecord(new AuditRecord(UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT), + UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT), "user", "api", Enums.Action.CREATE, + Enums.Resource.SAMPLE, "sampleId2", "sampleUuid2", "studyId", "studyUuid", new ObjectMap(), + new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS), TimeUtils.getDate(), new ObjectMap())); + dbAdaptorFactory.getCatalogAuditDbAdaptor(organizationId) + .insertAuditRecord(new AuditRecord(UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT), + UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT), "user", "api", Enums.Action.CREATE, + Enums.Resource.SAMPLE, "sampleId3", "sampleUuid3", "studyId", "studyUuid", new ObjectMap(), + new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS), TimeUtils.getDate(), new ObjectMap())); } // @Test diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBAdaptorTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBAdaptorTest.java index bee94ddaefa..82aeed9ef57 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBAdaptorTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBAdaptorTest.java @@ -18,21 +18,14 @@ import org.junit.After; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; -import org.junit.rules.ExpectedException; import org.opencb.biodata.models.common.Status; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.catalog.auth.authorization.AuthorizationDBAdaptor; import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager; -import org.opencb.opencga.catalog.db.DBAdaptorFactory; -import org.opencb.opencga.catalog.db.api.SampleDBAdaptor; -import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; -import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.models.AclEntry; import org.opencb.opencga.core.models.AclEntryList; @@ -41,11 +34,9 @@ import org.opencb.opencga.core.models.sample.SampleInternal; import org.opencb.opencga.core.models.sample.SamplePermissions; import org.opencb.opencga.core.models.study.PermissionRule; -import org.opencb.opencga.core.models.user.User; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.MediumTests; -import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -58,16 +49,9 @@ * Created by pfurio on 21/04/17. */ @Category(MediumTests.class) -public class AuthorizationMongoDBAdaptorTest { +public class AuthorizationMongoDBAdaptorTest extends AbstractMongoDBAdaptorTest { - @Rule - public ExpectedException thrown = ExpectedException.none(); private AuthorizationDBAdaptor aclDBAdaptor; - private DBAdaptorFactory dbAdaptorFactory; - private User user1; - private User user2; - private User user3; - private long studyId; private Sample s1; AclEntryList acls; @@ -77,47 +61,35 @@ public void after() { } @Before - public void before() throws IOException, CatalogException { - MongoDBAdaptorTest dbAdaptorTest = new MongoDBAdaptorTest(); - dbAdaptorTest.before(); - - user1 = MongoDBAdaptorTest.user1; - user2 = MongoDBAdaptorTest.user2; - user3 = MongoDBAdaptorTest.user3; - dbAdaptorFactory = dbAdaptorTest.catalogDBAdaptor; - aclDBAdaptor = new AuthorizationMongoDBAdaptor(dbAdaptorFactory, dbAdaptorTest.getConfiguration()); - - studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); - dbAdaptorFactory.getCatalogSampleDBAdaptor().insert(studyId, new Sample("s1", TimeUtils.getTime(), TimeUtils.getTime(), null, null, + public void setUp() throws Exception { + super.setUp(); + + OrganizationMongoDBAdaptorFactory orgFactory = dbAdaptorFactory.getOrganizationMongoDBAdaptorFactory(organizationId); + aclDBAdaptor = new AuthorizationMongoDBAdaptor(orgFactory, catalogManager.getConfiguration()); + + dbAdaptorFactory.getCatalogSampleDBAdaptor(organizationId).insert(studyUid, new Sample("s1", TimeUtils.getTime(), TimeUtils.getTime(), null, null, null, null, 1, 1, "", false, Collections.emptyList(), new ArrayList<>(), new Status(), SampleInternal.init(), Collections.emptyMap()), Collections.emptyList(), QueryOptions.empty()); - s1 = getSample(studyId, "s1"); + s1 = getSample(studyUid, "s1"); acls = new AclEntryList<>(); - acls.getAcl().add(new AclEntry<>(user1.getId(), EnumSet.noneOf(SamplePermissions.class))); - acls.getAcl().add(new AclEntry<>(user2.getId(), EnumSet.of(SamplePermissions.VIEW, SamplePermissions.VIEW_ANNOTATIONS, SamplePermissions.WRITE))); + acls.getAcl().add(new AclEntry<>(normalUserId1, EnumSet.noneOf(SamplePermissions.class))); + acls.getAcl().add(new AclEntry<>(normalUserId2, EnumSet.of(SamplePermissions.VIEW, SamplePermissions.VIEW_ANNOTATIONS, SamplePermissions.WRITE))); aclDBAdaptor.setAcls(Collections.singletonList(s1.getUid()), acls, Enums.Resource.SAMPLE); } - Sample getSample(long studyUid, String sampleId) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { - Query query = new Query() - .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) - .append(SampleDBAdaptor.QueryParams.ID.key(), sampleId); - return dbAdaptorFactory.getCatalogSampleDBAdaptor().get(query, QueryOptions.empty()).first(); - } - @Test public void addSetGetAndRemoveAcls() throws Exception { - aclDBAdaptor.resetMembersFromAllEntries(studyId, Arrays.asList(user1.getId(), user2.getId())); + aclDBAdaptor.resetMembersFromAllEntries(studyUid, Arrays.asList(normalUserId1, normalUserId2)); - aclDBAdaptor.addToMembers(studyId, Arrays.asList("user1", "user2", "user3"), Collections.singletonList( + aclDBAdaptor.addToMembers(studyUid, Arrays.asList("user1", "user2", "user3"), Collections.singletonList( new AuthorizationManager.CatalogAclParams(Arrays.asList(s1.getUid()), Arrays.asList(SamplePermissions.VIEW.name(), SamplePermissions.WRITE.name()), Enums.Resource.SAMPLE))); - aclDBAdaptor.addToMembers(studyId, Arrays.asList("user4"), Collections.singletonList( + aclDBAdaptor.addToMembers(studyUid, Arrays.asList("user4"), Collections.singletonList( new AuthorizationManager.CatalogAclParams(Arrays.asList(s1.getUid()), Collections.emptyList(), Enums.Resource.SAMPLE))); // We attempt to store the same permissions - aclDBAdaptor.addToMembers(studyId, Arrays.asList("user1", "user2", "user3"), Collections.singletonList( + aclDBAdaptor.addToMembers(studyUid, Arrays.asList("user1", "user2", "user3"), Collections.singletonList( new AuthorizationManager.CatalogAclParams(Arrays.asList(s1.getUid()), Arrays.asList(SamplePermissions.VIEW.name(), SamplePermissions.WRITE.name()), Enums.Resource.SAMPLE))); @@ -139,7 +111,7 @@ public void addSetGetAndRemoveAcls() throws Exception { assertTrue(sampleAcl.first().getAcl().get(1).getPermissions() .containsAll(Arrays.asList(SamplePermissions.VIEW, SamplePermissions.WRITE))); - aclDBAdaptor.setToMembers(studyId, Collections.singletonList("user1"), Collections.singletonList( + aclDBAdaptor.setToMembers(studyUid, Collections.singletonList("user1"), Collections.singletonList( new AuthorizationManager.CatalogAclParams(Collections.singletonList(s1.getUid()), Collections.singletonList("DELETE"), Enums.Resource.SAMPLE))); sampleAcl = aclDBAdaptor.get(s1.getUid(), Arrays.asList("user1", "user2"), null, Enums.Resource.SAMPLE, SamplePermissions.class); assertEquals(1, sampleAcl.getNumResults()); @@ -169,7 +141,7 @@ public void addSetGetAndRemoveAcls() throws Exception { assertNull(sampleAcl.first().getAcl().get(0).getPermissions()); // Remove from all samples (there is only one) in study - aclDBAdaptor.removeFromStudy(studyId, "user3", Enums.Resource.SAMPLE); + aclDBAdaptor.removeFromStudy(studyUid, "user3", Enums.Resource.SAMPLE); sampleAcl = aclDBAdaptor.get(s1.getUid(), Arrays.asList("user3"), null, Enums.Resource.SAMPLE, SamplePermissions.class); assertEquals(1, sampleAcl.getNumResults()); assertEquals(1, sampleAcl.first().getAcl().size()); @@ -212,15 +184,15 @@ public void addSetGetAndRemoveAcls() throws Exception { @Test public void getSampleAcl() throws Exception { - OpenCGAResult> sampleAcl = aclDBAdaptor.get(s1.getUid(), Arrays.asList(user1.getId(), user2.getId()), + OpenCGAResult> sampleAcl = aclDBAdaptor.get(s1.getUid(), Arrays.asList(normalUserId1, normalUserId2), null, Enums.Resource.SAMPLE, SamplePermissions.class); assertEquals(1, sampleAcl.getNumResults()); assertEquals(2, sampleAcl.first().getAcl().size()); - assertEquals(user1.getId(), sampleAcl.first().getAcl().get(0).getMember()); + assertEquals(normalUserId1, sampleAcl.first().getAcl().get(0).getMember()); assertEquals(1, sampleAcl.first().getAcl().get(0).getPermissions().size()); assertTrue(sampleAcl.first().getAcl().get(0).getPermissions().contains(SamplePermissions.NONE)); - assertEquals(user1.getId(), sampleAcl.first().getAcl().get(0).getMember()); + assertEquals(normalUserId1, sampleAcl.first().getAcl().get(0).getMember()); assertEquals(1, sampleAcl.first().getAcl().get(0).getPermissions().size()); assertTrue(acls.getAcl().get(1).getPermissions().containsAll(sampleAcl.first().getAcl().get(1).getPermissions())); } @@ -237,11 +209,11 @@ public void getSampleAclWrongUser() throws Exception { @Test public void getSampleAclFromUserWithoutAcl() throws Exception { - OpenCGAResult> sampleAcl = aclDBAdaptor.get(s1.getUid(), Collections.singletonList(user3.getId()), + OpenCGAResult> sampleAcl = aclDBAdaptor.get(s1.getUid(), Collections.singletonList(normalUserId3), null, Enums.Resource.SAMPLE, SamplePermissions.class); assertEquals(1, sampleAcl.getNumResults()); assertEquals(1, sampleAcl.first().getAcl().size()); - assertEquals(user3.getId(), sampleAcl.first().getAcl().get(0).getMember()); + assertEquals(normalUserId3, sampleAcl.first().getAcl().get(0).getMember()); assertNull(sampleAcl.first().getAcl().get(0).getPermissions()); assertTrue(sampleAcl.first().getAcl().get(0).getGroups().isEmpty()); } @@ -251,16 +223,16 @@ public void getSampleAclFromUserWithoutAcl() throws Exception { public void unsetSampleAcl2() throws Exception { // Unset permissions OpenCGAResult> sampleAcl = aclDBAdaptor.get(s1.getUid(), - Collections.singletonList(user2.getId()), null, Enums.Resource.SAMPLE, SamplePermissions.class); + Collections.singletonList(normalUserId2), null, Enums.Resource.SAMPLE, SamplePermissions.class); assertEquals(1, sampleAcl.getNumResults()); assertEquals(1, sampleAcl.first().getAcl().size()); assertEquals(3, sampleAcl.first().getAcl().get(0).getPermissions().size()); - aclDBAdaptor.removeFromMembers(Collections.singletonList(user2.getId()), Collections.singletonList( + aclDBAdaptor.removeFromMembers(Collections.singletonList(normalUserId2), Collections.singletonList( new AuthorizationManager.CatalogAclParams(Collections.singletonList(s1.getUid()), Arrays.asList("VIEW_ANNOTATIONS", "DELETE", "VIEW"), Enums.Resource.SAMPLE))); -// sampleDBAdaptor.unsetSampleAcl(s1.getId(), Arrays.asList(user2.getId()), +// sampleDBAdaptor.unsetSampleAcl(s1.getId(), Arrays.asList(normalUserId2), // Arrays.asList("VIEW_ANNOTATIONS", "DELETE", "VIEW")); - sampleAcl = aclDBAdaptor.get(s1.getUid(), Collections.singletonList(user2.getId()), null, Enums.Resource.SAMPLE, + sampleAcl = aclDBAdaptor.get(s1.getUid(), Collections.singletonList(normalUserId2), null, Enums.Resource.SAMPLE, SamplePermissions.class); assertEquals(1, sampleAcl.getNumResults()); assertEquals(1, sampleAcl.first().getAcl().get(0).getPermissions().size()); @@ -271,42 +243,42 @@ public void unsetSampleAcl2() throws Exception { public void setSampleAclOverride() throws Exception { // user2 permissions check assertEquals(acls.getAcl().get(1).getPermissions(), - aclDBAdaptor.get(s1.getUid(), Collections.singletonList(user2.getId()), null, Enums.Resource.SAMPLE, SamplePermissions.class).first().getAcl().get(0).getPermissions()); + aclDBAdaptor.get(s1.getUid(), Collections.singletonList(normalUserId2), null, Enums.Resource.SAMPLE, SamplePermissions.class).first().getAcl().get(0).getPermissions()); - aclDBAdaptor.setToMembers(studyId, Collections.singletonList(user2.getId()), Collections.singletonList( + aclDBAdaptor.setToMembers(studyUid, Collections.singletonList(normalUserId2), Collections.singletonList( new AuthorizationManager.CatalogAclParams(Arrays.asList(s1.getUid()), Arrays.asList(SamplePermissions.DELETE.name()), Enums.Resource.SAMPLE))); assertEquals(EnumSet.of(SamplePermissions.DELETE), - aclDBAdaptor.get(s1.getUid(), Collections.singletonList(user2.getId()), null, Enums.Resource.SAMPLE, SamplePermissions.class).first().getAcl().get(0).getPermissions()); + aclDBAdaptor.get(s1.getUid(), Collections.singletonList(normalUserId2), null, Enums.Resource.SAMPLE, SamplePermissions.class).first().getAcl().get(0).getPermissions()); } @Test public void testPermissionRulesPlusManualPermissions() throws CatalogException { // We create a new sample s2 - dbAdaptorFactory.getCatalogSampleDBAdaptor().insert(studyId, new Sample("s2", TimeUtils.getTime(), TimeUtils.getTime(), null, null, + dbAdaptorFactory.getCatalogSampleDBAdaptor(organizationId).insert(studyUid, new Sample("s2", TimeUtils.getTime(), TimeUtils.getTime(), null, null, null, null, 1, 1, "", false, Collections.emptyList(), new ArrayList<>(), new Status(), SampleInternal.init(), Collections.emptyMap()), Collections.emptyList(), QueryOptions.empty()); - Sample s2 = getSample(studyId, "s2"); + Sample s2 = getSample(studyUid, "s2"); // We create a new permission rule - PermissionRule pr = new PermissionRule("myPermissionRule", new Query(), Arrays.asList(user3.getId()), + PermissionRule pr = new PermissionRule("myPermissionRule", new Query(), Arrays.asList(normalUserId3), Arrays.asList(SamplePermissions.VIEW.name())); - dbAdaptorFactory.getCatalogStudyDBAdaptor().createPermissionRule(studyId, Enums.Entity.SAMPLES, pr); + dbAdaptorFactory.getCatalogStudyDBAdaptor(organizationId).createPermissionRule(studyUid, Enums.Entity.SAMPLES, pr); // Apply the permission rule - aclDBAdaptor.applyPermissionRules(studyId, pr, Enums.Entity.SAMPLES); + aclDBAdaptor.applyPermissionRules(studyUid, pr, Enums.Entity.SAMPLES); // All the samples should have view permissions for user user2 OpenCGAResult> dataResult = aclDBAdaptor.get(Arrays.asList(s1.getUid(), s2.getUid()), - Arrays.asList(user3.getId()), null, Enums.Resource.SAMPLE, SamplePermissions.class); + Arrays.asList(normalUserId3), null, Enums.Resource.SAMPLE, SamplePermissions.class); assertEquals(2, dataResult.getNumResults()); for (AclEntryList result : dataResult.getResults()) { assertTrue(result.getAcl().get(0).getPermissions().contains(SamplePermissions.VIEW)); } // Assign a manual permission to s2 - aclDBAdaptor.addToMembers(studyId, Arrays.asList(user3.getId()), Collections.singletonList( + aclDBAdaptor.addToMembers(studyUid, Arrays.asList(normalUserId3), Collections.singletonList( new AuthorizationManager.CatalogAclParams(Arrays.asList(s2.getUid()), Arrays.asList(SamplePermissions.DELETE.name()), Enums.Resource.SAMPLE))); } diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptorTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptorTest.java index 0c2dda0a32e..6a4509671f6 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptorTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptorTest.java @@ -16,408 +16,400 @@ package org.opencb.opencga.catalog.db.mongodb; -import org.apache.commons.lang3.RandomStringUtils; -import org.bson.Document; -import org.junit.Test; import org.junit.experimental.categories.Category; -import org.opencb.commons.datastore.core.DataResult; -import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.commons.datastore.core.Query; -import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.catalog.db.api.FileDBAdaptor; -import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; -import org.opencb.opencga.catalog.exceptions.CatalogDBException; -import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.exceptions.CatalogParameterException; -import org.opencb.opencga.catalog.utils.Constants; -import org.opencb.opencga.catalog.utils.ParamUtils; -import org.opencb.opencga.core.models.AclEntry; -import org.opencb.opencga.core.models.common.InternalStatus; -import org.opencb.opencga.core.models.file.File; -import org.opencb.opencga.core.models.file.FileInternal; -import org.opencb.opencga.core.models.file.FilePermissions; -import org.opencb.opencga.core.models.file.FileStatus; -import org.opencb.opencga.core.models.sample.Sample; -import org.opencb.opencga.core.models.sample.SampleInternal; import org.opencb.opencga.core.testclassification.duration.MediumTests; -import java.io.IOException; -import java.util.*; -import java.util.stream.Collectors; - -import static org.junit.Assert.*; - /** * Created by pfurio on 3/2/16. */ @Category(MediumTests.class) -public class FileMongoDBAdaptorTest extends MongoDBAdaptorTest { - - @Test - public void createFileToStudyTest() throws CatalogException { - long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); - assertTrue(studyId >= 0); - File file; - file = new File("jobs/", File.Type.DIRECTORY, File.Format.PLAIN, File.Bioformat.NONE, "jobs/", null, "", - FileInternal.init(), 1000, 1); - LinkedList> acl = new LinkedList<>(); - acl.push(new AclEntry<>("jcoll", EnumSet.of(FilePermissions.VIEW, FilePermissions.VIEW_CONTENT, FilePermissions.VIEW_HEADER, - FilePermissions.DELETE))); - acl.push(new AclEntry<>("jmmut", EnumSet.noneOf(FilePermissions.class))); - System.out.println(catalogFileDBAdaptor.insert(studyId, file, null, null, null, null)); - file = new File("file.sam", File.Type.FILE, File.Format.PLAIN, File.Bioformat.ALIGNMENT, "data/file.sam", null, "", - FileInternal.init(), 1000, 1); - System.out.println(catalogFileDBAdaptor.insert(studyId, file, null, null, null, null)); - file = new File("file.bam", File.Type.FILE, File.Format.BINARY, File.Bioformat.ALIGNMENT, "data/file.bam", null, "", - FileInternal.init(), 1000, 1); - System.out.println(catalogFileDBAdaptor.insert(studyId, file, null, null, null, null)); - file = new File("file.vcf", File.Type.FILE, File.Format.PLAIN, File.Bioformat.VARIANT, "data/file2.vcf", null, "", - FileInternal.init(), 1000, 1); - - try { - System.out.println(catalogFileDBAdaptor.insert(-20, file, null, null, null, null)); - fail("Expected \"StudyId not found\" exception"); - } catch (CatalogDBException e) { - System.out.println(e); - } - - System.out.println(catalogFileDBAdaptor.insert(studyId, file, null, null, null, null)); - - try { - System.out.println(catalogFileDBAdaptor.insert(studyId, file, null, null, null, null)); - fail("Expected \"File already exist\" exception"); - } catch (CatalogDBException e) { - System.out.println(e); - } - } - - @Test - public void getFileTest() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { - File file = catalogFileDBAdaptor.get(new Query() - .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), user3.getProjects().get(0).getStudies().get(0).getUid()) - .append(FileDBAdaptor.QueryParams.NAME.key(), "file.vcf"), QueryOptions.empty()).first(); - DataResult fileDataResult = catalogFileDBAdaptor.get(file.getUid(), null); - System.out.println(fileDataResult); - try { - System.out.println(catalogFileDBAdaptor.get(-1, null)); - fail("Expected \"FileId not found\" exception"); - } catch (CatalogDBException e) { - System.out.println(e); - } - } - - @Test - public void getAllFilesStudyNotValidTest() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { - thrown.expect(CatalogDBException.class); - thrown.expectMessage("not valid"); - catalogFileDBAdaptor.getAllInStudy(-1, null); - } +public class FileMongoDBAdaptorTest extends AbstractMongoDBAdaptorTest { - @Test - public void getAllFilesStudyNotExistsTest() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { - thrown.expect(CatalogDBException.class); - thrown.expectMessage("not exist"); - catalogFileDBAdaptor.getAllInStudy(216544, null); - } - - @Test - public void getAllFilesTest() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { - List files = catalogFileDBAdaptor.get( - new Query(FileDBAdaptor.QueryParams.STUDY_UID.key(), user3.getProjects().get(0).getStudies().get(0).getUid()), - QueryOptions.empty()).getResults(); - assertEquals(2, files.size()); - } - -// // Test if the lookup operation works fine // @Test -// public void getFileWithJob() throws CatalogDBException { -// long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); -// QueryOptions queryOptions = new QueryOptions(); -// -// // We create a job -// String jobName = "jobName"; -// String jobDescription = "This is the description of the job"; -// Job myJob = new Job().setName(jobName).setDescription(jobDescription); -// DataResult jobInsert = catalogJobDBAdaptor.insert(myJob, studyId, queryOptions); -// -// // We create a new file giving that job -// File file = new File().setName("Filename").setPath("data/Filename").setJob(jobInsert.first()); -// DataResult fileInsert = catalogFileDBAdaptor.insert(file, studyId, queryOptions); -// -// // Get the file -// DataResult noJobInfoDataResult = catalogFileDBAdaptor.get(fileInsert.first().getUid(), queryOptions); -// assertNull(noJobInfoDataResult.first().getJob().getName()); -// assertNull(noJobInfoDataResult.first().getJob().getDescription()); -// -// queryOptions.put("lazy", false); -// DataResult jobInfoDataResult = catalogFileDBAdaptor.get(fileInsert.first().getUid(), queryOptions); -// assertEquals(jobName, jobInfoDataResult.first().getJob().getName()); -// assertEquals(jobDescription, jobInfoDataResult.first().getJob().getDescription()); +// public void createFileToStudyTest() throws CatalogException { +// File file = new File("jobs/", File.Type.DIRECTORY, File.Format.PLAIN, File.Bioformat.NONE, "jobs/", null, "", FileInternal.init(), 1000, 1); +// LinkedList> acl = new LinkedList<>(); +// acl.push(new AclEntry<>("jcoll", EnumSet.of(FilePermissions.VIEW, FilePermissions.VIEW_CONTENT, FilePermissions.VIEW_HEADER, +// FilePermissions.DELETE))); +// acl.push(new AclEntry<>("jmmut", EnumSet.noneOf(FilePermissions.class))); +// System.out.println(catalogFileDBAdaptor.insert(studyUid, file, null, null, null, null)); +// file = new File("file.sam", File.Type.FILE, File.Format.PLAIN, File.Bioformat.ALIGNMENT, "data/file.sam", null, "", +// FileInternal.init(), 1000, 1); +// System.out.println(catalogFileDBAdaptor.insert(studyUid, file, null, null, null, null)); +// file = new File("file.bam", File.Type.FILE, File.Format.BINARY, File.Bioformat.ALIGNMENT, "data/file.bam", null, "", +// FileInternal.init(), 1000, 1); +// System.out.println(catalogFileDBAdaptor.insert(studyUid, file, null, null, null, null)); +// file = new File("file.vcf", File.Type.FILE, File.Format.PLAIN, File.Bioformat.VARIANT, "data/file2.vcf", null, "", +// FileInternal.init(), 1000, 1); +// +// try { +// System.out.println(catalogFileDBAdaptor.insert(-20, file, null, null, null, null)); +// fail("Expected \"StudyId not found\" exception"); +// } catch (CatalogDBException e) { +// System.out.println(e); +// } +// +// System.out.println(catalogFileDBAdaptor.insert(studyUid, file, null, null, null, null)); +// +// try { +// System.out.println(catalogFileDBAdaptor.insert(studyUid, file, null, null, null, null)); +// fail("Expected \"File already exist\" exception"); +// } catch (CatalogDBException e) { +// System.out.println(e); +// } +// } +// +// @Test +// public void getFileTest() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { +// File file = catalogFileDBAdaptor.get(new Query() +// .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) +// .append(FileDBAdaptor.QueryParams.NAME.key(), "file.vcf"), QueryOptions.empty()).first(); +// DataResult fileDataResult = catalogFileDBAdaptor.get(file.getUid(), null); +// System.out.println(fileDataResult); +// try { +// System.out.println(catalogFileDBAdaptor.get(-1, null)); +// fail("Expected \"FileId not found\" exception"); +// } catch (CatalogDBException e) { +// System.out.println(e); +// } +// } +// +// @Test +// public void getAllFilesStudyNotValidTest() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { +// thrown.expect(CatalogDBException.class); +// thrown.expectMessage("not valid"); +// catalogFileDBAdaptor.getAllInStudy(-1, null); +// } +// +// @Test +// public void getAllFilesStudyNotExistsTest() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { +// thrown.expect(CatalogDBException.class); +// thrown.expectMessage("not exist"); +// catalogFileDBAdaptor.getAllInStudy(216544, null); +// } +// +// @Test +// public void getAllFilesTest() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { +// List files = catalogFileDBAdaptor.get( +// new Query(FileDBAdaptor.QueryParams.STUDY_UID.key(), user3.getProjects().get(0).getStudies().get(0).getUid()), +// QueryOptions.empty()).getResults(); +// assertEquals(2, files.size()); +// } +// +//// // Test if the lookup operation works fine +//// @Test +//// public void getFileWithJob() throws CatalogDBException { +//// long studyUid = user3.getProjects().get(0).getStudies().get(0).getUid(); +//// QueryOptions queryOptions = new QueryOptions(); +//// +//// // We create a job +//// String jobName = "jobName"; +//// String jobDescription = "This is the description of the job"; +//// Job myJob = new Job().setName(jobName).setDescription(jobDescription); +//// DataResult jobInsert = catalogJobDBAdaptor.insert(myJob, studyUid, queryOptions); +//// +//// // We create a new file giving that job +//// File file = new File().setName("Filename").setPath("data/Filename").setJob(jobInsert.first()); +//// DataResult fileInsert = catalogFileDBAdaptor.insert(file, studyUid, queryOptions); +//// +//// // Get the file +//// DataResult noJobInfoDataResult = catalogFileDBAdaptor.get(fileInsert.first().getUid(), queryOptions); +//// assertNull(noJobInfoDataResult.first().getJob().getName()); +//// assertNull(noJobInfoDataResult.first().getJob().getDescription()); +//// +//// queryOptions.put("lazy", false); +//// DataResult jobInfoDataResult = catalogFileDBAdaptor.get(fileInsert.first().getUid(), queryOptions); +//// assertEquals(jobName, jobInfoDataResult.first().getJob().getName()); +//// assertEquals(jobDescription, jobInfoDataResult.first().getJob().getDescription()); +//// } +// +// @Test +// public void modifyFileTest() throws CatalogDBException, IOException, CatalogParameterException, CatalogAuthorizationException { +// File file = catalogFileDBAdaptor.get(new Query() +// .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), user3.getProjects().get(0).getStudies().get(0).getUid()) +// .append(FileDBAdaptor.QueryParams.NAME.key(), "file.vcf"), QueryOptions.empty()).first(); +// long fileId = file.getUid(); +// +// Document stats = new Document("stat1", 1).append("stat2", true).append("stat3", "ok" + RandomStringUtils.randomAlphanumeric(20)); +// +// ObjectMap parameters = new ObjectMap(); +// parameters.put("status.name", FileStatus.READY); +// parameters.put("stats", stats); +// System.out.println(catalogFileDBAdaptor.update(fileId, parameters, QueryOptions.empty())); +// +// file = catalogFileDBAdaptor.get(fileId, null).first(); +// assertEquals(file.getInternal().getStatus().getId(), FileStatus.READY); +// assertEquals(file.getStats(), stats); +// +// parameters = new ObjectMap(); +// parameters.put("stats", "{}"); +// System.out.println(catalogFileDBAdaptor.update(fileId, parameters, QueryOptions.empty())); +// +// file = catalogFileDBAdaptor.get(fileId, null).first(); +// assertEquals(file.getStats(), new LinkedHashMap()); +// } +// +// @Test +// public void renameFileTest() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { +// String newName = "newFile.bam"; +// String parentPath = "data/"; +// long fileId = catalogFileDBAdaptor.getId(user3.getProjects().get(0).getStudies().get(0).getUid(), "data/file.vcf"); +// System.out.println(catalogFileDBAdaptor.rename(fileId, parentPath + newName, "", null)); +// +// File file = catalogFileDBAdaptor.get(fileId, null).first(); +// assertEquals(file.getName(), newName); +// assertEquals(file.getPath(), parentPath + newName); +// +// try { +// catalogFileDBAdaptor.rename(-1, "noFile", "", null); +// fail("error: expected \"file not found\"exception"); +// } catch (CatalogDBException e) { +// System.out.println("correct exception: " + e); +// } +// +// long folderId = catalogFileDBAdaptor.getId(user3.getProjects().get(0).getStudies().get(0).getUid(), "data/"); +// String folderName = "folderName"; +// catalogFileDBAdaptor.rename(folderId, folderName, "", null); +// assertTrue(catalogFileDBAdaptor.get(fileId, null).first().getPath().equals(folderName + "/" + newName)); +// } +// +// @Test +// public void includeFields() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { +// DataResult fileDataResult = catalogFileDBAdaptor.get(7, +// new QueryOptions(QueryOptions.INCLUDE, FileDBAdaptor.QueryParams.PATH.key())); +// List files = fileDataResult.getResults(); +// assertEquals("Include path does not work.", "data/file.vcf", files.get(0).getPath()); +// assertEquals("Include not working.", null, files.get(0).getName()); +// } +// +// @Test +// public void testDistinct() throws Exception { +// long studyUid = user3.getProjects().get(0).getStudies().get(0).getUid(); +// +// List distinctTypes = catalogFileDBAdaptor.distinct(studyUid, FileDBAdaptor.QueryParams.TYPE.key(), null, user3.getId() +// ).getResults(); +// assertEquals(Arrays.asList("DIRECTORY", "FILE"), distinctTypes); +// +// List distinctFormats = catalogFileDBAdaptor.distinct(studyUid, FileDBAdaptor.QueryParams.FORMAT.key(), null, user3.getId() +// ).getResults(); +// assertTrue(Arrays.asList("PLAIN").containsAll(distinctFormats)); +// +// studyUid = user4.getProjects().get(0).getStudies().get(0).getUid(); +// distinctFormats = catalogFileDBAdaptor.distinct(studyUid, FileDBAdaptor.QueryParams.FORMAT.key(), null, user4.getId() +// ).getResults(); +// assertTrue(Arrays.asList("BAM", "COMMA_SEPARATED_VALUES", "UNKNOWN").containsAll(distinctFormats)); +// } +// +// @Test +// public void testRank() throws Exception { +// List pfurioStudies = Arrays.asList(9L, 14L); +// List rankedFilesPerDiskUsage = catalogFileDBAdaptor.rank( +// new Query(FileDBAdaptor.QueryParams.STUDY_UID.key(), pfurioStudies), +// FileDBAdaptor.QueryParams.SIZE.key(), 100, false).getResults(); +// +// assertEquals(3, rankedFilesPerDiskUsage.size()); +// assertTrue(Arrays.asList(10, 100, 5000) +// .containsAll(rankedFilesPerDiskUsage.stream().map(d -> d.get("_id")).collect(Collectors.toSet()))); +// +// for (Document document : rankedFilesPerDiskUsage) { +// switch (document.getInteger("_id")) { +// case 10: +// case 5000: +// assertEquals(2, document.get("count")); +// break; +// case 100: +// assertEquals(3, document.get("count")); +// break; +// } +// } +// } +// +// @Test +// public void testGroupBy() throws Exception { +// List pfurioStudies = Arrays.asList(9L, 14L); +// +// List groupByBioformat = catalogFileDBAdaptor.groupBy(new Query(FileDBAdaptor.QueryParams.STUDY_UID.key(), pfurioStudies), +// FileDBAdaptor.QueryParams.BIOFORMAT.key(), new QueryOptions()).getResults(); +// +// assertTrue(Arrays.asList("ALIGNMENT", "NONE") +// .containsAll(groupByBioformat.stream().map(d -> d.get("_id")) +// .map(d -> ((Document) d).get(FileDBAdaptor.QueryParams.BIOFORMAT.key())).collect(Collectors.toSet()))); +// for (Document document : groupByBioformat) { +// switch (((Document) document.get("_id")).getString(FileDBAdaptor.QueryParams.BIOFORMAT.key())) { +// case "ALIGNMENT": +// assertTrue(Arrays.asList("m_alignment.bam", "alignment.bam").containsAll(document.getList("items", String.class))); +// break; +// case "NONE": +// assertTrue(Arrays.asList("m_file1.txt", "file2.txt", "file1.txt", "data/") +// .containsAll(document.getList("items", String.class))); +// break; +// } +// } +// +// groupByBioformat = catalogFileDBAdaptor.groupBy(new Query(FileDBAdaptor.QueryParams.STUDY_UID.key(), 14), // MINECO study +// FileDBAdaptor.QueryParams.BIOFORMAT.key(), new QueryOptions()).getResults(); +// +// assertTrue(Arrays.asList("ALIGNMENT", "NONE") +// .containsAll(groupByBioformat.stream().map(d -> d.get("_id")) +// .map(d -> ((Document) d).get(FileDBAdaptor.QueryParams.BIOFORMAT.key())).collect(Collectors.toSet()))); +// for (Document document : groupByBioformat) { +// switch (((Document) document.get("_id")).getString(FileDBAdaptor.QueryParams.BIOFORMAT.key())) { +// case "ALIGNMENT": +// assertTrue(Arrays.asList("m_alignment.bam").containsAll(document.getList("items", String.class))); +// break; +// case "NONE": +// assertTrue(Arrays.asList("m_file1.txt", "data/") +// .containsAll(document.getList("items", String.class))); +// break; +// } +// } +// } +// +// @Test +// public void testGroupBy1() throws Exception { +// +// List pfurioStudies = Arrays.asList(9L, 14L); +// List groupByBioformat = catalogFileDBAdaptor.groupBy( +// new Query(FileDBAdaptor.QueryParams.STUDY_UID.key(), pfurioStudies), +// Arrays.asList(FileDBAdaptor.QueryParams.BIOFORMAT.key(), FileDBAdaptor.QueryParams.TYPE.key()), +// new QueryOptions()).getResults(); +// +// assertEquals(3, groupByBioformat.size()); +// for (Document document : groupByBioformat) { +// Document d = (Document) document.get("_id"); +// +// switch (d.getString(FileDBAdaptor.QueryParams.BIOFORMAT.key()) + "_" + d.getString(FileDBAdaptor.QueryParams.TYPE.key())) { +// case "ALIGNMENT_FILE": +// assertTrue(Arrays.asList("m_alignment.bam", "alignment.bam").containsAll(document.getList("items", String.class))); +// break; +// case "NONE_FILE": +// assertTrue(Arrays.asList("m_file1.txt", "file2.txt", "file1.txt").containsAll(document.getList("items", String.class))); +// break; +// case "NONE_FOLDER": +// assertTrue(Arrays.asList("data/").containsAll(document.getList("items", String.class))); +// break; +// } +// } +// } +// +// @Test +// public void testAddSamples() throws Exception { +// long studyUid = user3.getProjects().get(0).getStudies().get(0).getUid(); +// new InternalStatus(); +// catalogSampleDBAdaptor.insert(studyUid, new Sample().setId("sample1").setInternal(SampleInternal.init()), +// Collections.emptyList(), QueryOptions.empty()); +// Sample sample1 = getSample(studyUid, "sample1"); +// new InternalStatus(); +// catalogSampleDBAdaptor.insert(studyUid, new Sample().setId("sample2").setInternal(SampleInternal.init()), +// Collections.emptyList(), QueryOptions.empty()); +// Sample sample2 = getSample(studyUid, "sample2"); +// +// ObjectMap action = new ObjectMap(FileDBAdaptor.QueryParams.SAMPLE_IDS.key(), ParamUtils.BasicUpdateAction.ADD); +// QueryOptions options = new QueryOptions(Constants.ACTIONS, action); +// +// File file = catalogFileDBAdaptor.get(new Query() +// .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), user3.getProjects().get(0).getStudies().get(0).getUid()) +// .append(FileDBAdaptor.QueryParams.NAME.key(), "file.vcf"), QueryOptions.empty()).first(); +// catalogFileDBAdaptor.update(file.getUid(), new ObjectMap(FileDBAdaptor.QueryParams.SAMPLE_IDS.key(), +// Arrays.asList(sample1.getId(), sample2.getId())), options); +// +// DataResult fileDataResult = catalogFileDBAdaptor.get(file.getUid(), QueryOptions.empty()); +// assertEquals(2, fileDataResult.first().getSampleIds().size()); +// assertTrue(Arrays.asList(sample1.getId(), sample2.getId()).containsAll(fileDataResult.first().getSampleIds())); +// +// catalogSampleDBAdaptor.insert(studyUid, new Sample().setId("sample3").setInternal(SampleInternal.init()), +// Collections.emptyList(), QueryOptions.empty()); +// Sample sample3 = getSample(studyUid, "sample3"); +// // Test we avoid duplicities +// catalogFileDBAdaptor.update(file.getUid(), new ObjectMap(FileDBAdaptor.QueryParams.SAMPLE_IDS.key(), +// Arrays.asList(sample1.getId(), sample2.getId(), sample2.getId(), sample3.getId())), options); +// fileDataResult = catalogFileDBAdaptor.get(file.getUid(), QueryOptions.empty()); +// assertEquals(3, fileDataResult.first().getSampleIds().size()); +// assertTrue(Arrays.asList(sample1.getId(), sample2.getId(), sample3.getId()).containsAll(fileDataResult.first().getSampleIds())); +// } +// +// @Test +// public void testRemoveSamples() throws Exception { +// long studyUid = user3.getProjects().get(0).getStudies().get(0).getUid(); +// new InternalStatus(); +// catalogSampleDBAdaptor.insert(studyUid, new Sample().setId("sample1").setInternal(SampleInternal.init()), +// Collections.emptyList(), QueryOptions.empty()); +// Sample sample1 = getSample(studyUid, "sample1"); +// new InternalStatus(); +// catalogSampleDBAdaptor.insert(studyUid, new Sample().setId("sample2").setInternal(SampleInternal.init()), +// Collections.emptyList(), QueryOptions.empty()); +// Sample sample2 = getSample(studyUid, "sample2"); +// new InternalStatus(); +// catalogSampleDBAdaptor.insert(studyUid, new Sample().setId("sample3").setInternal(SampleInternal.init()), +// Collections.emptyList(), QueryOptions.empty()); +// Sample sample3 = getSample(studyUid, "sample3"); +// List files = catalogFileDBAdaptor.get(new Query() +// .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), user3.getProjects().get(0).getStudies().get(0).getUid()), +// QueryOptions.empty()).getResults(); +// File file = files.get(0); +// File file2 = files.get(1); +// ObjectMap action = new ObjectMap(FileDBAdaptor.QueryParams.SAMPLE_IDS.key(), ParamUtils.BasicUpdateAction.ADD); +// QueryOptions options = new QueryOptions(Constants.ACTIONS, action); +// +// catalogFileDBAdaptor.update(file.getUid(), new ObjectMap(FileDBAdaptor.QueryParams.SAMPLE_IDS.key(), +// Arrays.asList(sample1.getId(), sample2.getId(), sample3.getId())), options); +// catalogFileDBAdaptor.update(file2.getUid(), new ObjectMap(FileDBAdaptor.QueryParams.SAMPLE_IDS.key(), +// Arrays.asList(sample1.getId(), sample2.getId(), sample3.getId())), options); +// +// DataResult fileDataResult = catalogFileDBAdaptor.get(file.getUid(), QueryOptions.empty()); +// assertEquals(3, fileDataResult.first().getSampleIds().size()); +// assertTrue(Arrays.asList(sample1.getId(), sample2.getId(), sample3.getId()).containsAll(fileDataResult.first().getSampleIds())); +// +// fileDataResult = catalogFileDBAdaptor.get(file2.getUid(), QueryOptions.empty()); +// assertEquals(3, fileDataResult.first().getSampleIds().size()); +// assertTrue(Arrays.asList(sample1.getId(), sample2.getId(), sample3.getId()).containsAll(fileDataResult.first().getSampleIds())); +// +// catalogFileDBAdaptor.removeSampleReferences(null, studyUid, sample1); +// catalogFileDBAdaptor.removeSampleReferences(null, studyUid, sample3); +// fileDataResult = catalogFileDBAdaptor.get(file.getUid(), QueryOptions.empty()); +// assertEquals(1, fileDataResult.first().getSampleIds().size()); +// assertEquals(sample2.getId(), fileDataResult.first().getSampleIds().get(0)); +// +// fileDataResult = catalogFileDBAdaptor.get(file2.getUid(), QueryOptions.empty()); +// assertEquals(1, fileDataResult.first().getSampleIds().size()); +// assertEquals(sample2.getId(), fileDataResult.first().getSampleIds().get(0)); +// } +// +// @Test +// public void testGroupByDates() throws Exception { +// List pfurioStudies = Arrays.asList(9L, 14L); +// +// List groupByBioformat = catalogFileDBAdaptor.groupBy( +// new Query(FileDBAdaptor.QueryParams.STUDY_UID.key(), pfurioStudies), +// Arrays.asList(FileDBAdaptor.QueryParams.BIOFORMAT.key(), FileDBAdaptor.QueryParams.TYPE.key(), "day"), +// new QueryOptions()).getResults(); +// +// assertEquals(3, groupByBioformat.size()); +// +// for (int i = 0; i < groupByBioformat.size(); i++) { +// String bioformat = ((Document) groupByBioformat.get(i).get("_id")).getString("bioformat"); +// String type = ((Document) groupByBioformat.get(i).get("_id")).getString("type"); +// switch (bioformat) { +// case "NONE": +// switch (type) { +// case "FILE": +// assertEquals(5, ((Document) groupByBioformat.get(i).get("_id")).size()); // None - File +// assertTrue(Arrays.asList("m_file1.txt", "file2.txt", "file1.txt").containsAll(groupByBioformat.get(i).getList("items", String.class))); +// break; +// default: +// assertEquals(5, ((Document) groupByBioformat.get(i).get("_id")).size()); // None - Folder +// assertEquals(Arrays.asList("data/"), groupByBioformat.get(i).get("items")); +// break; +// } +// break; +// case "ALIGNMENT": +// assertEquals(5, ((Document) groupByBioformat.get(i).get("_id")).size()); +// assertTrue(Arrays.asList("m_alignment.bam", "alignment.bam").containsAll(groupByBioformat.get(i).getList("items", String.class))); +// break; +// default: +// fail("This case should not happen."); +// break; +// } +// } // } - - @Test - public void modifyFileTest() throws CatalogDBException, IOException, CatalogParameterException, CatalogAuthorizationException { - File file = catalogFileDBAdaptor.get(new Query() - .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), user3.getProjects().get(0).getStudies().get(0).getUid()) - .append(FileDBAdaptor.QueryParams.NAME.key(), "file.vcf"), QueryOptions.empty()).first(); - long fileId = file.getUid(); - - Document stats = new Document("stat1", 1).append("stat2", true).append("stat3", "ok" + RandomStringUtils.randomAlphanumeric(20)); - - ObjectMap parameters = new ObjectMap(); - parameters.put("status.name", FileStatus.READY); - parameters.put("stats", stats); - System.out.println(catalogFileDBAdaptor.update(fileId, parameters, QueryOptions.empty())); - - file = catalogFileDBAdaptor.get(fileId, null).first(); - assertEquals(file.getInternal().getStatus().getId(), FileStatus.READY); - assertEquals(file.getStats(), stats); - - parameters = new ObjectMap(); - parameters.put("stats", "{}"); - System.out.println(catalogFileDBAdaptor.update(fileId, parameters, QueryOptions.empty())); - - file = catalogFileDBAdaptor.get(fileId, null).first(); - assertEquals(file.getStats(), new LinkedHashMap()); - } - - @Test - public void includeFields() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { - DataResult fileDataResult = catalogFileDBAdaptor.get(7, - new QueryOptions(QueryOptions.INCLUDE, FileDBAdaptor.QueryParams.PATH.key())); - List files = fileDataResult.getResults(); - assertEquals("Include path does not work.", "data/file.vcf", files.get(0).getPath()); - assertEquals("Include not working.", null, files.get(0).getName()); - } - - @Test - public void testDistinct() throws Exception { - long studyUid = user3.getProjects().get(0).getStudies().get(0).getUid(); - - List distinctTypes = catalogFileDBAdaptor.distinct(studyUid, FileDBAdaptor.QueryParams.TYPE.key(), null, user3.getId() - ).getResults(); - assertEquals(Arrays.asList("DIRECTORY", "FILE"), distinctTypes); - - List distinctFormats = catalogFileDBAdaptor.distinct(studyUid, FileDBAdaptor.QueryParams.FORMAT.key(), null, user3.getId() - ).getResults(); - assertTrue(Arrays.asList("PLAIN").containsAll(distinctFormats)); - - studyUid = user4.getProjects().get(0).getStudies().get(0).getUid(); - distinctFormats = catalogFileDBAdaptor.distinct(studyUid, FileDBAdaptor.QueryParams.FORMAT.key(), null, user4.getId() - ).getResults(); - assertTrue(Arrays.asList("BAM", "COMMA_SEPARATED_VALUES", "UNKNOWN").containsAll(distinctFormats)); - } - - @Test - public void testRank() throws Exception { - List pfurioStudies = Arrays.asList(9L, 14L); - List rankedFilesPerDiskUsage = catalogFileDBAdaptor.rank( - new Query(FileDBAdaptor.QueryParams.STUDY_UID.key(), pfurioStudies), - FileDBAdaptor.QueryParams.SIZE.key(), 100, false).getResults(); - - assertEquals(3, rankedFilesPerDiskUsage.size()); - assertTrue(Arrays.asList(10, 100, 5000) - .containsAll(rankedFilesPerDiskUsage.stream().map(d -> d.get("_id")).collect(Collectors.toSet()))); - - for (Document document : rankedFilesPerDiskUsage) { - switch (document.getInteger("_id")) { - case 10: - case 5000: - assertEquals(2, document.get("count")); - break; - case 100: - assertEquals(3, document.get("count")); - break; - } - } - } - - @Test - public void testGroupBy() throws Exception { - List pfurioStudies = Arrays.asList(9L, 14L); - - List groupByBioformat = catalogFileDBAdaptor.groupBy(new Query(FileDBAdaptor.QueryParams.STUDY_UID.key(), pfurioStudies), - FileDBAdaptor.QueryParams.BIOFORMAT.key(), new QueryOptions()).getResults(); - - assertTrue(Arrays.asList("ALIGNMENT", "NONE") - .containsAll(groupByBioformat.stream().map(d -> d.get("_id")) - .map(d -> ((Document) d).get(FileDBAdaptor.QueryParams.BIOFORMAT.key())).collect(Collectors.toSet()))); - for (Document document : groupByBioformat) { - switch (((Document) document.get("_id")).getString(FileDBAdaptor.QueryParams.BIOFORMAT.key())) { - case "ALIGNMENT": - assertTrue(Arrays.asList("m_alignment.bam", "alignment.bam").containsAll(document.getList("items", String.class))); - break; - case "NONE": - assertTrue(Arrays.asList("m_file1.txt", "file2.txt", "file1.txt", "data/") - .containsAll(document.getList("items", String.class))); - break; - } - } - - groupByBioformat = catalogFileDBAdaptor.groupBy(new Query(FileDBAdaptor.QueryParams.STUDY_UID.key(), 14), // MINECO study - FileDBAdaptor.QueryParams.BIOFORMAT.key(), new QueryOptions()).getResults(); - - assertTrue(Arrays.asList("ALIGNMENT", "NONE") - .containsAll(groupByBioformat.stream().map(d -> d.get("_id")) - .map(d -> ((Document) d).get(FileDBAdaptor.QueryParams.BIOFORMAT.key())).collect(Collectors.toSet()))); - for (Document document : groupByBioformat) { - switch (((Document) document.get("_id")).getString(FileDBAdaptor.QueryParams.BIOFORMAT.key())) { - case "ALIGNMENT": - assertTrue(Arrays.asList("m_alignment.bam").containsAll(document.getList("items", String.class))); - break; - case "NONE": - assertTrue(Arrays.asList("m_file1.txt", "data/") - .containsAll(document.getList("items", String.class))); - break; - } - } - } - - @Test - public void testGroupBy1() throws Exception { - - List pfurioStudies = Arrays.asList(9L, 14L); - List groupByBioformat = catalogFileDBAdaptor.groupBy( - new Query(FileDBAdaptor.QueryParams.STUDY_UID.key(), pfurioStudies), - Arrays.asList(FileDBAdaptor.QueryParams.BIOFORMAT.key(), FileDBAdaptor.QueryParams.TYPE.key()), - new QueryOptions()).getResults(); - - assertEquals(3, groupByBioformat.size()); - for (Document document : groupByBioformat) { - Document d = (Document) document.get("_id"); - - switch (d.getString(FileDBAdaptor.QueryParams.BIOFORMAT.key()) + "_" + d.getString(FileDBAdaptor.QueryParams.TYPE.key())) { - case "ALIGNMENT_FILE": - assertTrue(Arrays.asList("m_alignment.bam", "alignment.bam").containsAll(document.getList("items", String.class))); - break; - case "NONE_FILE": - assertTrue(Arrays.asList("m_file1.txt", "file2.txt", "file1.txt").containsAll(document.getList("items", String.class))); - break; - case "NONE_FOLDER": - assertTrue(Arrays.asList("data/").containsAll(document.getList("items", String.class))); - break; - } - } - } - - @Test - public void testAddSamples() throws Exception { - long studyUid = user3.getProjects().get(0).getStudies().get(0).getUid(); - new InternalStatus(); - catalogDBAdaptor.getCatalogSampleDBAdaptor().insert(studyUid, new Sample().setId("sample1").setInternal(SampleInternal.init()), - Collections.emptyList(), QueryOptions.empty()); - Sample sample1 = getSample(studyUid, "sample1"); - new InternalStatus(); - catalogDBAdaptor.getCatalogSampleDBAdaptor().insert(studyUid, new Sample().setId("sample2").setInternal(SampleInternal.init()), - Collections.emptyList(), QueryOptions.empty()); - Sample sample2 = getSample(studyUid, "sample2"); - - ObjectMap action = new ObjectMap(FileDBAdaptor.QueryParams.SAMPLE_IDS.key(), ParamUtils.BasicUpdateAction.ADD); - QueryOptions options = new QueryOptions(Constants.ACTIONS, action); - - File file = catalogFileDBAdaptor.get(new Query() - .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), user3.getProjects().get(0).getStudies().get(0).getUid()) - .append(FileDBAdaptor.QueryParams.NAME.key(), "file.vcf"), QueryOptions.empty()).first(); - catalogFileDBAdaptor.update(file.getUid(), new ObjectMap(FileDBAdaptor.QueryParams.SAMPLE_IDS.key(), - Arrays.asList(sample1.getId(), sample2.getId())), options); - - DataResult fileDataResult = catalogFileDBAdaptor.get(file.getUid(), QueryOptions.empty()); - assertEquals(2, fileDataResult.first().getSampleIds().size()); - assertTrue(Arrays.asList(sample1.getId(), sample2.getId()).containsAll(fileDataResult.first().getSampleIds())); - - catalogDBAdaptor.getCatalogSampleDBAdaptor().insert(studyUid, new Sample().setId("sample3").setInternal(SampleInternal.init()), - Collections.emptyList(), QueryOptions.empty()); - Sample sample3 = getSample(studyUid, "sample3"); - // Test we avoid duplicities - catalogFileDBAdaptor.update(file.getUid(), new ObjectMap(FileDBAdaptor.QueryParams.SAMPLE_IDS.key(), - Arrays.asList(sample1.getId(), sample2.getId(), sample2.getId(), sample3.getId())), options); - fileDataResult = catalogFileDBAdaptor.get(file.getUid(), QueryOptions.empty()); - assertEquals(3, fileDataResult.first().getSampleIds().size()); - assertTrue(Arrays.asList(sample1.getId(), sample2.getId(), sample3.getId()).containsAll(fileDataResult.first().getSampleIds())); - } - - @Test - public void testRemoveSamples() throws Exception { - long studyUid = user3.getProjects().get(0).getStudies().get(0).getUid(); - new InternalStatus(); - catalogDBAdaptor.getCatalogSampleDBAdaptor().insert(studyUid, new Sample().setId("sample1").setInternal(SampleInternal.init()), - Collections.emptyList(), QueryOptions.empty()); - Sample sample1 = getSample(studyUid, "sample1"); - new InternalStatus(); - catalogDBAdaptor.getCatalogSampleDBAdaptor().insert(studyUid, new Sample().setId("sample2").setInternal(SampleInternal.init()), - Collections.emptyList(), QueryOptions.empty()); - Sample sample2 = getSample(studyUid, "sample2"); - new InternalStatus(); - catalogDBAdaptor.getCatalogSampleDBAdaptor().insert(studyUid, new Sample().setId("sample3").setInternal(SampleInternal.init()), - Collections.emptyList(), QueryOptions.empty()); - Sample sample3 = getSample(studyUid, "sample3"); - List files = catalogFileDBAdaptor.get(new Query() - .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), user3.getProjects().get(0).getStudies().get(0).getUid()), - QueryOptions.empty()).getResults(); - File file = files.get(0); - File file2 = files.get(1); - ObjectMap action = new ObjectMap(FileDBAdaptor.QueryParams.SAMPLE_IDS.key(), ParamUtils.BasicUpdateAction.ADD); - QueryOptions options = new QueryOptions(Constants.ACTIONS, action); - - catalogFileDBAdaptor.update(file.getUid(), new ObjectMap(FileDBAdaptor.QueryParams.SAMPLE_IDS.key(), - Arrays.asList(sample1.getId(), sample2.getId(), sample3.getId())), options); - catalogFileDBAdaptor.update(file2.getUid(), new ObjectMap(FileDBAdaptor.QueryParams.SAMPLE_IDS.key(), - Arrays.asList(sample1.getId(), sample2.getId(), sample3.getId())), options); - - DataResult fileDataResult = catalogFileDBAdaptor.get(file.getUid(), QueryOptions.empty()); - assertEquals(3, fileDataResult.first().getSampleIds().size()); - assertTrue(Arrays.asList(sample1.getId(), sample2.getId(), sample3.getId()).containsAll(fileDataResult.first().getSampleIds())); - - fileDataResult = catalogFileDBAdaptor.get(file2.getUid(), QueryOptions.empty()); - assertEquals(3, fileDataResult.first().getSampleIds().size()); - assertTrue(Arrays.asList(sample1.getId(), sample2.getId(), sample3.getId()).containsAll(fileDataResult.first().getSampleIds())); - - catalogFileDBAdaptor.removeSampleReferences(null, studyUid, sample1); - catalogFileDBAdaptor.removeSampleReferences(null, studyUid, sample3); - fileDataResult = catalogFileDBAdaptor.get(file.getUid(), QueryOptions.empty()); - assertEquals(1, fileDataResult.first().getSampleIds().size()); - assertEquals(sample2.getId(), fileDataResult.first().getSampleIds().get(0)); - - fileDataResult = catalogFileDBAdaptor.get(file2.getUid(), QueryOptions.empty()); - assertEquals(1, fileDataResult.first().getSampleIds().size()); - assertEquals(sample2.getId(), fileDataResult.first().getSampleIds().get(0)); - } - - @Test - public void testGroupByDates() throws Exception { - List pfurioStudies = Arrays.asList(9L, 14L); - - List groupByBioformat = catalogFileDBAdaptor.groupBy( - new Query(FileDBAdaptor.QueryParams.STUDY_UID.key(), pfurioStudies), - Arrays.asList(FileDBAdaptor.QueryParams.BIOFORMAT.key(), FileDBAdaptor.QueryParams.TYPE.key(), "day"), - new QueryOptions()).getResults(); - - assertEquals(3, groupByBioformat.size()); - - for (int i = 0; i < groupByBioformat.size(); i++) { - String bioformat = ((Document) groupByBioformat.get(i).get("_id")).getString("bioformat"); - String type = ((Document) groupByBioformat.get(i).get("_id")).getString("type"); - switch (bioformat) { - case "NONE": - switch (type) { - case "FILE": - assertEquals(5, ((Document) groupByBioformat.get(i).get("_id")).size()); // None - File - assertTrue(Arrays.asList("m_file1.txt", "file2.txt", "file1.txt").containsAll(groupByBioformat.get(i).getList("items", String.class))); - break; - default: - assertEquals(5, ((Document) groupByBioformat.get(i).get("_id")).size()); // None - Folder - assertEquals(Arrays.asList("data/"), groupByBioformat.get(i).get("items")); - break; - } - break; - case "ALIGNMENT": - assertEquals(5, ((Document) groupByBioformat.get(i).get("_id")).size()); - assertTrue(Arrays.asList("m_alignment.bam", "alignment.bam").containsAll(groupByBioformat.get(i).getList("items", String.class))); - break; - default: - fail("This case should not happen."); - break; - } - } - } } diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptorTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptorTest.java index 4d359aec97b..8882260fabc 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptorTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptorTest.java @@ -17,7 +17,6 @@ package org.opencb.opencga.catalog.db.mongodb; import org.bson.Document; -import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; import org.opencb.biodata.models.core.SexOntologyTermAnnotation; @@ -27,16 +26,15 @@ import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.catalog.db.api.IndividualDBAdaptor; -import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; -import org.opencb.opencga.catalog.exceptions.CatalogParameterException; -import org.opencb.opencga.core.models.common.InternalStatus; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.core.models.individual.Individual; import org.opencb.opencga.core.models.individual.IndividualInternal; import org.opencb.opencga.core.models.individual.IndividualPopulation; import org.opencb.opencga.core.models.individual.Location; import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.sample.SampleInternal; +import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.MediumTests; import java.util.Arrays; @@ -44,112 +42,116 @@ import java.util.List; import java.util.stream.Collectors; +import static com.mongodb.internal.connection.tlschannel.util.Util.assertTrue; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; /** * Created by hpccoll1 on 19/06/15. */ @Category(MediumTests.class) -public class IndividualMongoDBAdaptorTest extends MongoDBAdaptorTest { +public class IndividualMongoDBAdaptorTest extends AbstractMongoDBAdaptorTest { @Test public void testCreateIndividual() throws Exception { - long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); - catalogIndividualDBAdaptor.insert(studyId, new Individual().setId("individual"), null, null); + OpenCGAResult result = catalogIndividualDBAdaptor.get(studyUid, new Query(IndividualDBAdaptor.QueryParams.ID.key(), "individual"), QueryOptions.empty(), orgOwnerUserId); + assertEquals(0, result.getNumResults()); + + OpenCGAResult individual = catalogIndividualDBAdaptor.insert(studyUid, new Individual().setId("individual"), null, null); + assertEquals(1, individual.getNumInserted()); + + result = catalogIndividualDBAdaptor.get(studyUid, new Query(IndividualDBAdaptor.QueryParams.ID.key(), "individual"), QueryOptions.empty(), orgOwnerUserId); + assertEquals(1, result.getNumResults()); } @Test public void testCreateIndividualStudyNotFound() throws Exception { thrown.expect(CatalogDBException.class); + thrown.expectMessage("not valid"); catalogIndividualDBAdaptor.insert(-10, new Individual(), null, null); } @Test public void testCreateIndividualFatherNotFound() throws Exception { - long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); thrown.expect(CatalogDBException.class); - catalogIndividualDBAdaptor.insert(studyId, new Individual("in1", "in1", new Individual().setId("father").setUid(10), null, + thrown.expectMessage("not found"); + catalogIndividualDBAdaptor.insert(studyUid, new Individual("in1", "in1", new Individual().setId("father").setUid(10), null, null, null, null, null, null, null, "", Collections.emptyList(), false, 1, Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), IndividualInternal.init(), null), null, null); } @Test public void testCreateIndividualAlreadyExists() throws Exception { - long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); - catalogIndividualDBAdaptor.insert(studyId, new Individual("in1", "in1", new Individual(), new Individual(), new Location(), null, null, null, null, null, "", + catalogIndividualDBAdaptor.insert(studyUid, new Individual("in1", "in1", new Individual(), new Individual(), new Location(), null, null, null, null, null, "", Collections.emptyList(), false, 1, Collections .emptyList(), Collections.emptyList(), Collections.emptyList(), IndividualInternal.init(), null), null, null); thrown.expect(CatalogDBException.class); //Name already exists - catalogIndividualDBAdaptor.insert(studyId, new Individual("in1", "in1", new Individual(), new Individual(), new Location(), null, null, null, null, null, "", + thrown.expectMessage("already exists"); + catalogIndividualDBAdaptor.insert(studyUid, new Individual("in1", "in1", new Individual(), new Individual(), new Location(), null, null, null, null, null, "", Collections.emptyList(), false, 1, Collections .emptyList(), Collections.emptyList(), Collections.emptyList(), IndividualInternal.init(), null), null, null); } @Test public void testGetIndividual() throws Exception { - long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); Individual individual = new Individual("an_individual", "An Individual", new Individual(), new Individual(), new Location(), SexOntologyTermAnnotation.initMale(), null, null, new IndividualPopulation(), null, "", Collections.emptyList(), false, 1, Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), IndividualInternal.init(), null); - catalogIndividualDBAdaptor.insert(studyId, individual, null, null); + catalogIndividualDBAdaptor.insert(studyUid, individual, null, null); Individual individual2 = catalogIndividualDBAdaptor.get(individual.getUid(), null).first(); assertEquals(individual.getId(), individual2.getId()); } @Test public void testGetIndividualNoExists() throws Exception { - long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); Individual individual = new Individual("in1", "An Individual", new Individual(), new Individual(), new Location(), SexOntologyTermAnnotation.initMale(), null, null, new IndividualPopulation(), null, "", Collections.emptyList(), false, 1, Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), IndividualInternal.init(), null); - catalogIndividualDBAdaptor.insert(studyId, individual, null, null); + catalogIndividualDBAdaptor.insert(studyUid, individual, null, null); catalogIndividualDBAdaptor.get(individual.getUid(), null).first(); - thrown.expect(CatalogDBException.class); //Id not found + thrown.expect(CatalogDBException.class); + thrown.expectMessage("not exist"); catalogIndividualDBAdaptor.get(9999, null); } @Test public void testGetAllIndividuals() throws Exception { - long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); - catalogIndividualDBAdaptor.insert(studyId, new Individual("ind_1", "ind_1", new Individual(), new Individual(), new Location(), SexOntologyTermAnnotation.initMale(), null, null, new IndividualPopulation(), null, "", + catalogIndividualDBAdaptor.insert(studyUid, new Individual("ind_1", "ind_1", new Individual(), new Individual(), new Location(), SexOntologyTermAnnotation.initMale(), null, null, new IndividualPopulation(), null, "", Collections.emptyList(), false, 1, Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), IndividualInternal.init(), null), null, null); - catalogIndividualDBAdaptor.insert(studyId, new Individual("ind_2", "ind_2", new Individual(), new Individual(), new Location(), SexOntologyTermAnnotation.initFemale(), null, null, new + catalogIndividualDBAdaptor.insert(studyUid, new Individual("ind_2", "ind_2", new Individual(), new Individual(), new Location(), SexOntologyTermAnnotation.initFemale(), null, null, new IndividualPopulation(), null, "", Collections.emptyList(), false, 1, Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), IndividualInternal.init(), null), null, null); - catalogIndividualDBAdaptor.insert(studyId, new Individual("ind_3", "ind_3", new Individual(), new Individual(), new Location(), SexOntologyTermAnnotation.initMale(), null, null, new IndividualPopulation(), null, "", + catalogIndividualDBAdaptor.insert(studyUid, new Individual("ind_3", "ind_3", new Individual(), new Individual(), new Location(), SexOntologyTermAnnotation.initMale(), null, null, new IndividualPopulation(), null, "", Collections.emptyList(), false, 1, Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), IndividualInternal.init(), null), null, null); - Individual father = getIndividual(studyId, "ind_3"); - catalogIndividualDBAdaptor.insert(studyId, new Individual("ind_4", "ind_4", new Individual(), new Individual(), new Location(), SexOntologyTermAnnotation.initFemale(), null, null, new IndividualPopulation(), null, "", + Individual father = getIndividual(studyUid, "ind_3"); + catalogIndividualDBAdaptor.insert(studyUid, new Individual("ind_4", "ind_4", new Individual(), new Individual(), new Location(), SexOntologyTermAnnotation.initFemale(), null, null, new IndividualPopulation(), null, "", Collections.emptyList(), false, 1, Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), IndividualInternal.init(), null), null, null); - Individual mother = getIndividual(studyId, "ind_4"); - catalogIndividualDBAdaptor.insert(studyId, new Individual("ind_5", "ind_5", father, mother, null, SexOntologyTermAnnotation.initMale(), + Individual mother = getIndividual(studyUid, "ind_4"); + catalogIndividualDBAdaptor.insert(studyUid, new Individual("ind_5", "ind_5", father, mother, null, SexOntologyTermAnnotation.initMale(), IndividualProperty.KaryotypicSex.XY, null, new IndividualPopulation(), null, null, null, true, 1, Collections.emptyList(), null, null, IndividualInternal.init(), null), null, null); - catalogIndividualDBAdaptor.insert(studyId, new Individual("ind_6", "ind_6", new Individual(), new Individual(), new Location(), SexOntologyTermAnnotation.initFemale(), null, null, new IndividualPopulation(), null, "", + catalogIndividualDBAdaptor.insert(studyUid, new Individual("ind_6", "ind_6", new Individual(), new Individual(), new Location(), SexOntologyTermAnnotation.initFemale(), null, null, new IndividualPopulation(), null, "", Collections.emptyList(), false, 1, Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), IndividualInternal.init(), null), null, null); DataResult result; result = catalogIndividualDBAdaptor.get(new Query(IndividualDBAdaptor.QueryParams.ID.key(), - "~ind_[1-3]").append(IndividualDBAdaptor.QueryParams.STUDY_UID.key(), studyId), new QueryOptions()); + "~ind_[1-3]").append(IndividualDBAdaptor.QueryParams.STUDY_UID.key(), studyUid), new QueryOptions()); assertEquals(3, result.getNumResults()); result = catalogIndividualDBAdaptor.get(new Query(IndividualDBAdaptor.QueryParams.SEX_ID.key(), - IndividualProperty.Sex.FEMALE).append(IndividualDBAdaptor.QueryParams.STUDY_UID.key(), studyId), new QueryOptions()); + IndividualProperty.Sex.FEMALE).append(IndividualDBAdaptor.QueryParams.STUDY_UID.key(), studyUid), new QueryOptions()); assertEquals(3, result.getNumResults()); result = catalogIndividualDBAdaptor.get(new Query(IndividualDBAdaptor.QueryParams.FATHER_UID.key(), ">0") - .append(IndividualDBAdaptor.QueryParams.STUDY_UID.key(), studyId), new QueryOptions()); + .append(IndividualDBAdaptor.QueryParams.STUDY_UID.key(), studyUid), new QueryOptions()); assertEquals(1, result.getNumResults()); } @Test public void testModifyIndividual() throws Exception { - long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); - catalogIndividualDBAdaptor.insert(studyId, new Individual("in1", "in1", new Individual(), new Individual(), new Location(), SexOntologyTermAnnotation.initUnknown(), null, null, null, null, "", + catalogIndividualDBAdaptor.insert(studyUid, new Individual("in1", "in1", new Individual(), new Individual(), new Location(), SexOntologyTermAnnotation.initUnknown(), null, null, null, null, "", Collections.emptyList(), false, 1, Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), IndividualInternal.init(), null), Collections.emptyList(), null); - long individualUid = getIndividual(studyId, "in1").getUid(); + long individualUid = getIndividual(studyUid, "in1").getUid(); ObjectMap params = new ObjectMap("sex", SexOntologyTermAnnotation.initMale()); catalogIndividualDBAdaptor.update(individualUid, params, QueryOptions.empty()); @@ -159,10 +161,9 @@ public void testModifyIndividual() throws Exception { @Test public void testModifyIndividualBadFatherId() throws Exception { - long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); - catalogIndividualDBAdaptor.insert(studyId, new Individual("in1", "in1", new Individual(), new Individual(), new Location(), SexOntologyTermAnnotation.initUnknown(), null, null, null, null, "", + catalogIndividualDBAdaptor.insert(studyUid, new Individual("in1", "in1", new Individual(), new Individual(), new Location(), SexOntologyTermAnnotation.initUnknown(), null, null, null, null, "", Collections.emptyList(), false, 1, Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), IndividualInternal.init(), null), Collections.emptyList(), null); - long individualUid = getIndividual(studyId, "in1").getUid(); + long individualUid = getIndividual(studyUid, "in1").getUid(); thrown.expect(CatalogDBException.class); catalogIndividualDBAdaptor.update(individualUid, new ObjectMap(IndividualDBAdaptor.QueryParams.FATHER_UID.key(), 4000), @@ -171,10 +172,9 @@ public void testModifyIndividualBadFatherId() throws Exception { @Test public void testModifyIndividualNegativeFatherId() throws Exception { - long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); - catalogIndividualDBAdaptor.insert(studyId, new Individual("in1", "in1", new Individual(), new Individual(), new Location(), SexOntologyTermAnnotation.initUnknown(), null, null, null, null, "", + catalogIndividualDBAdaptor.insert(studyUid, new Individual("in1", "in1", new Individual(), new Individual(), new Location(), SexOntologyTermAnnotation.initUnknown(), null, null, null, null, "", Collections.emptyList(), false, 1, Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), IndividualInternal.init(), null), Collections.emptyList(), null); - long individualUid = getIndividual(studyId, "in1").getUid(); + long individualUid = getIndividual(studyUid, "in1").getUid(); DataResult result = catalogIndividualDBAdaptor.update(individualUid, new ObjectMap(IndividualDBAdaptor.QueryParams.FATHER_UID.key(), -1), QueryOptions.empty()); @@ -185,39 +185,21 @@ public void testModifyIndividualNegativeFatherId() throws Exception { assertEquals(-1, individual.getFather().getUid()); } - // FIXME: This should be tested in the managers - @Ignore @Test - public void testModifyIndividualExistingName() throws Exception { - long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); - catalogIndividualDBAdaptor.insert(studyId, new Individual("in1", "in1", new Individual(), new Individual(), new Location(), SexOntologyTermAnnotation.initUnknown(), null, null, null, null, "", - Collections.emptyList(), false, 1, Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), IndividualInternal.init(), null), Collections.emptyList(), null); - long individualUid = getIndividual(studyId, "in1").getUid(); - catalogIndividualDBAdaptor.insert(studyId, new Individual("in2", "in2", new Individual(), new Individual(), new Location(), SexOntologyTermAnnotation.initUnknown(), null, null, null, null, "", - Collections.emptyList(), false, 1, Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), IndividualInternal.init(), null), Collections.emptyList(), null); - - thrown.expect(CatalogDBException.class); - catalogIndividualDBAdaptor.update(individualUid, new ObjectMap("name", "in2"), QueryOptions.empty()); - } - - @Test - public void testAvoidDuplicatedSamples() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { - long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); - new InternalStatus(); - catalogDBAdaptor.getCatalogSampleDBAdaptor().insert(studyId, new Sample().setId("sample1").setInternal(SampleInternal.init()), + public void testAvoidDuplicatedSamples() throws CatalogException { + dbAdaptorFactory.getCatalogSampleDBAdaptor(organizationId).insert(studyUid, new Sample().setId("sample1").setInternal(SampleInternal.init()), Collections.emptyList(), QueryOptions.empty()); - Sample sample1 = getSample(studyId, "sample1"); - new InternalStatus(); - catalogDBAdaptor.getCatalogSampleDBAdaptor().insert(studyId, new Sample().setId("sample2").setInternal(SampleInternal.init()), + Sample sample1 = getSample(studyUid, "sample1"); + dbAdaptorFactory.getCatalogSampleDBAdaptor(organizationId).insert(studyUid, new Sample().setId("sample2").setInternal(SampleInternal.init()), Collections.emptyList(), QueryOptions.empty()); - Sample sample2 = getSample(studyId, "sample2"); + Sample sample2 = getSample(studyUid, "sample2"); Individual individual = new Individual() .setId("in2") .setInternal(IndividualInternal.init()) .setSamples(Arrays.asList(sample1, sample1, sample2)); - catalogIndividualDBAdaptor.insert(studyId, individual, Collections.emptyList(), null); - Individual individualStored = getIndividual(studyId, "in2"); + catalogIndividualDBAdaptor.insert(studyUid, individual, Collections.emptyList(), null); + Individual individualStored = getIndividual(studyUid, "in2"); assertEquals(2, individualStored.getSamples().size()); assertTrue(individualStored.getSamples().stream().map(Sample::getUid).collect(Collectors.toSet()).containsAll(Arrays.asList( sample1.getUid(), sample2.getUid()))); @@ -240,14 +222,14 @@ public void testAvoidDuplicatedSamples() throws CatalogDBException, CatalogParam } // @Test -// public void testSeveralAnnotationsInUniqueVariableSet() throws CatalogDBException { -// long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); -// long individualId = catalogIndividualDBAdaptor.insert(studyId, new Individual("in1", "in1", 0, 0, "", IndividualProperty.Sex -// .UNKNOWN, "", null, 1, Collections.emptyList(), null), null).first().getUid(); +// public void testSeveralAnnotationsInUniqueVariableSet() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { +// Individual individual = new Individual().setId("in1").setName("in1"); +// catalogIndividualDBAdaptor.insert(studyUid, individual, null, null); +// long individualUid = individual.getUid(); // // Set variableSet = new HashSet<>(); -// variableSet.add(new Variable().setId("key").setType(Variable.VariableType.TEXT)); -// variableSet.add(new Variable().setId("key2").setType(Variable.VariableType.TEXT)); +// variableSet.add(new Variable().setId("key").setType(Variable.VariableType.STRING)); +// variableSet.add(new Variable().setId("key2").setType(Variable.VariableType.STRING)); // variableSet.add(new Variable().setId("key3").setType(Variable.VariableType.INTEGER)); // variableSet.add(new Variable().setId("key4").setType(Variable.VariableType.BOOLEAN)); // VariableSet vs = new VariableSet().setUid(3L).setVariables(variableSet).setUnique(true).setId("vsId"); @@ -260,21 +242,21 @@ public void testAvoidDuplicatedSamples() throws CatalogDBException, CatalogParam // // AnnotationSet annot1 = new AnnotationSet("annot1", vs.getId(), annotationSet, "", 1, Collections.emptyMap()); // AnnotationSet annot2 = new AnnotationSet("annot2", vs.getId(), annotationSet, "", 1, Collections.emptyMap()); -// catalogIndividualDBAdaptor.createAnnotationSetForMigration(individualId, vs, annot1); +// catalogIndividualDBAdaptor.createAnnotationSetForMigration(individualUid, vs, annot1); // thrown.expect(CatalogDBException.class); // thrown.expectMessage("unique VariableSet"); -// catalogIndividualDBAdaptor.createAnnotationSetForMigration(individualId, vs, annot2); +// catalogIndividualDBAdaptor.createAnnotationSetForMigration(individualUid, vs, annot2); // } // // @Test // public void testAnnotateIndividual() throws Exception { -// long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); -// long individualId = catalogIndividualDBAdaptor.insert(studyId, new Individual("in1", "in1", 0, 0, "", IndividualProperty.Sex -// .UNKNOWN, "", null, 1, Collections.emptyList(), null), null).first().getUid(); +// Individual individual = new Individual().setId("in1").setName("in1"); +// catalogIndividualDBAdaptor.insert(studyUid, individual, null, null); +// long individualUid = individual.getUid(); // // Set variableSet = new HashSet<>(); -// variableSet.add(new Variable().setId("key").setType(Variable.VariableType.TEXT)); -// variableSet.add(new Variable().setId("key2").setType(Variable.VariableType.TEXT)); +// variableSet.add(new Variable().setId("key").setType(Variable.VariableType.STRING)); +// variableSet.add(new Variable().setId("key2").setType(Variable.VariableType.STRING)); // variableSet.add(new Variable().setId("key3").setType(Variable.VariableType.INTEGER)); // variableSet.add(new Variable().setId("key4").setType(Variable.VariableType.BOOLEAN)); // VariableSet vs = new VariableSet().setUid(3L).setVariables(variableSet).setId("vsId"); @@ -287,53 +269,51 @@ public void testAvoidDuplicatedSamples() throws CatalogDBException, CatalogParam // // AnnotationSet annot1 = new AnnotationSet("annot1", vs.getId(), annotationSet, "", 1, Collections.emptyMap()); // AnnotationSet annot2 = new AnnotationSet("annot2", vs.getId(), annotationSet, "", 1, Collections.emptyMap()); -// catalogIndividualDBAdaptor.createAnnotationSetForMigration(individualId, vs, annot1); -// catalogIndividualDBAdaptor.createAnnotationSetForMigration(individualId, vs, annot2); +// catalogIndividualDBAdaptor.createAnnotationSetForMigration(individualUid, vs, annot1); +// catalogIndividualDBAdaptor.createAnnotationSetForMigration(individualUid, vs, annot2); // -// Individual individual = catalogIndividualDBAdaptor.get(individualId, +// individual = catalogIndividualDBAdaptor.get(individualUid, // new QueryOptions(QueryOptions.INCLUDE, Constants.VARIABLE_SET + "." + 3)).first(); // Map annotationSets = individual.getAnnotationSets().stream().collect(Collectors.toMap // (AnnotationSet::getId, Function.identity())); // assertEquals(2, annotationSets.size()); // } - +// // @Test // public void testAnnotateIndividualExistingAnnotationId() throws Exception { -// long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); -// long individualId = catalogIndividualDBAdaptor.insert(studyId, new Individual("in1", "in1", 0, 0, "", IndividualProperty.Sex -// .UNKNOWN, "", null, 1, Collections.emptyList(), null), null).first().getUid(); +// Individual individual = new Individual().setId("in1").setName("in1"); +// catalogIndividualDBAdaptor.insert(studyUid, individual, null, null); +// long individualUid = individual.getUid(); // // Set variableSet = new HashSet<>(); -// variableSet.add(new Variable().setId("key").setType(Variable.VariableType.TEXT)); +// variableSet.add(new Variable().setId("key").setType(Variable.VariableType.STRING)); // VariableSet vs = new VariableSet().setUid(3L).setVariables(variableSet).setId("vsId"); // -// catalogIndividualDBAdaptor.createAnnotationSetForMigration(individualId, vs, +// catalogIndividualDBAdaptor.createAnnotationSetForMigration(individualUid, vs, // new AnnotationSet("annot1", vs.getId(), new ObjectMap("key", "hello"), "", 1, Collections.emptyMap())); // thrown.expect(CatalogDBException.class); // thrown.expectMessage("already exists"); -// catalogIndividualDBAdaptor.createAnnotationSetForMigration(individualId, vs, +// catalogIndividualDBAdaptor.createAnnotationSetForMigration(individualUid, vs, // new AnnotationSet("annot1", vs.getId(), new ObjectMap("key", "hello"), "", 1, Collections.emptyMap())); // } @Test public void testGetStudyIdByIndividualId() throws Exception { - long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); - catalogIndividualDBAdaptor.insert(studyId, new Individual().setId("individual").setInternal(IndividualInternal.init()), + catalogIndividualDBAdaptor.insert(studyUid, new Individual().setId("individual").setInternal(IndividualInternal.init()), Collections.emptyList(), null); - long individualUid = getIndividual(studyId, "individual").getUid(); + long individualUid = getIndividual(studyUid, "individual").getUid(); long studyIdByIndividualId = catalogIndividualDBAdaptor.getStudyId(individualUid); - assertEquals(studyId, studyIdByIndividualId); + assertEquals(studyUid, studyIdByIndividualId); } @Test public void testNativeGet() throws Exception { - long studyId = user4.getProjects().get(0).getStudies().get(0).getUid(); Individual individual = new Individual("in1", "An Individual", new Individual(), new Individual(), new Location(), SexOntologyTermAnnotation.initMale(), null, null, new IndividualPopulation(), null, "", Collections.emptyList(), false, 1, Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), IndividualInternal.init(), null); - catalogIndividualDBAdaptor.insert(studyId, individual, Collections.emptyList(), null); + catalogIndividualDBAdaptor.insert(studyUid, individual, Collections.emptyList(), null); Individual individual2 = new Individual("in2", "Another Individual", new Individual(), new Individual(), new Location(), SexOntologyTermAnnotation.initFemale(), null, null, new IndividualPopulation(), null, "", Collections.emptyList(), false, 1, Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), IndividualInternal.init(), null); - catalogIndividualDBAdaptor.insert(studyId, individual2, Collections.emptyList(), null); + catalogIndividualDBAdaptor.insert(studyUid, individual2, Collections.emptyList(), null); DataResult queryResult = catalogIndividualDBAdaptor.nativeGet(new Query(IndividualDBAdaptor.QueryParams.ID.key(), individual.getId()), new QueryOptions()); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptorTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptorTest.java index fb2f54527b4..a382282764d 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptorTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptorTest.java @@ -48,7 +48,7 @@ * Created by pfurio on 3/2/16. */ @Category(MediumTests.class) -public class JobMongoDBAdaptorTest extends MongoDBAdaptorTest { +public class JobMongoDBAdaptorTest extends AbstractMongoDBAdaptorTest { private Job getNewJob(String id) { // Constructor containining the minimal fields that are necessary and automatically populated by the manager @@ -60,13 +60,12 @@ private Job getNewJob(String id) { } @Test - public void createJobTest() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { - long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); + public void createJobTest() throws CatalogException { Job job = getNewJob("jobName1"); - System.out.println(catalogJobDBAdaptor.insert(studyId, job, null)); + System.out.println(catalogJobDBAdaptor.insert(studyUid, job, null)); job = getNewJob("jobName2"); - System.out.println(catalogJobDBAdaptor.insert(studyId, job, null)); + System.out.println(catalogJobDBAdaptor.insert(studyUid, job, null)); try { catalogJobDBAdaptor.insert(-1, job, null); fail("error: expected exception"); @@ -77,12 +76,10 @@ public void createJobTest() throws CatalogDBException, CatalogParameterException @Test public void deleteJobTest() throws CatalogException { - long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); - - catalogJobDBAdaptor.insert(studyId, getNewJob("name") - .setUserId(user3.getId()) + catalogJobDBAdaptor.insert(studyUid, getNewJob("name") + .setUserId(orgOwnerUserId) .setOutDir(new File().setUid(4)), null); - Job job = getJob(studyId, "name"); + Job job = getJob(studyUid, "name"); assertEquals(Enums.ExecutionStatus.PENDING, job.getInternal().getStatus().getId()); catalogJobDBAdaptor.delete(job); @@ -99,19 +96,16 @@ public void deleteJobTest() throws CatalogException { @Test public void getAllJobTest() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { - long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); - DataResult allJobs = catalogJobDBAdaptor.getAllInStudy(studyId, null); + DataResult allJobs = catalogJobDBAdaptor.getAllInStudy(studyUid, null); System.out.println(allJobs); } @Test public void getJobTest() throws CatalogException { - long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); - - catalogJobDBAdaptor.insert(studyId, getNewJob("name").setUserId(user3.getId()) + catalogJobDBAdaptor.insert(studyUid, getNewJob("name").setUserId(orgOwnerUserId) .setOutDir(new File().setUid(4)), null); - Job job = getJob(studyId, "name"); + Job job = getJob(studyUid, "name"); job = catalogJobDBAdaptor.get(job.getUid(), null).first(); System.out.println(job); @@ -126,9 +120,7 @@ public void getJobTest() throws CatalogException { } @Test - public void testSortResultsPriorityAndCreationDate() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { - long studyUid = user3.getProjects().get(0).getStudies().get(0).getUid(); - + public void testSortResultsPriorityAndCreationDate() throws CatalogException { Date startDate = TimeUtils.getDate(); // Create 100 jobs @@ -170,9 +162,9 @@ public void testSortResultsPriorityAndCreationDate() throws CatalogDBException, // @Test // public void SetVisitedJob() throws CatalogException { -// long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); -// catalogJobDBAdaptor.insert(studyId, new Job("name", user3.getId(), "", "", "", new File().setUid(4), Collections.emptyList(), 1), null); -// Job jobBefore = getJob(studyId, "name"); +// long studyUid = user3.getProjects().get(0).getStudies().get(0).getUid(); +// catalogJobDBAdaptor.insert(studyUid, new Job("name", user3.getId(), "", "", "", new File().setUid(4), Collections.emptyList(), 1), null); +// Job jobBefore = getJob(studyUid, "name"); // long jobId = jobBefore.getUid(); // assertTrue(!jobBefore.isVisited()); // @@ -184,9 +176,7 @@ public void testSortResultsPriorityAndCreationDate() throws CatalogDBException, // } @Test - public void getJobsOrderedByDate() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { - long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); - + public void getJobsOrderedByDate() throws CatalogException { // Job with current date Job job1 = getNewJob("job1"); @@ -200,8 +190,8 @@ public void getJobsOrderedByDate() throws CatalogDBException, CatalogParameterEx .setCreationDate(TimeUtils.getTime(oneHourBack)); // We create the jobs - catalogJobDBAdaptor.insert(studyId, job1, new QueryOptions()); - catalogJobDBAdaptor.insert(studyId, job2, new QueryOptions()); + catalogJobDBAdaptor.insert(studyUid, job1, new QueryOptions()); + catalogJobDBAdaptor.insert(studyUid, job2, new QueryOptions()); // Obtain the jobs in descending order QueryOptions queryOptions = new QueryOptions() @@ -229,9 +219,8 @@ public void getJobsOrderedByDate() throws CatalogDBException, CatalogParameterEx public void updateInputAndOutputFiles() throws Exception { Job job = getNewJob("jobName1") .setOutDir(new File().setUid(5)); - long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); - catalogJobDBAdaptor.insert(studyId, job, null); - job = getJob(studyId, "jobName1"); + catalogJobDBAdaptor.insert(studyUid, job, null); + job = getJob(studyUid, "jobName1"); List fileInput = Arrays.asList( new File().setUid(5L).setName("file1").setInternal(FileInternal.init()), @@ -261,8 +250,7 @@ public void updateInputAndOutputFiles() throws Exception { } @Test - public void groupByStatus() throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException { - long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); + public void groupByStatus() throws CatalogException { for (int i = 0; i < 10; i++) { Enums.ExecutionStatus status = new Enums.ExecutionStatus(); if (i < 5) { @@ -273,12 +261,12 @@ public void groupByStatus() throws CatalogDBException, CatalogAuthorizationExcep Job job = getNewJob("jobName" + i) .setOutDir(new File().setUid(5)); job.getInternal().setStatus(status); - catalogJobDBAdaptor.insert(studyId, job, null); + catalogJobDBAdaptor.insert(studyUid, job, null); } - Query query = new Query(JobDBAdaptor.QueryParams.STUDY_UID.key(), studyId); + Query query = new Query(JobDBAdaptor.QueryParams.STUDY_UID.key(), studyUid); OpenCGAResult openCGAResult = catalogJobDBAdaptor.groupBy(query, Collections.singletonList("internal.status.id"), - new QueryOptions(QueryOptions.COUNT, true), user3.getId()); + new QueryOptions(QueryOptions.COUNT, true), orgOwnerUserId); assertEquals(2, openCGAResult.getResults().size()); for (Object o : openCGAResult.getResults()) { diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MetaMongoDBAdaptorTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MetaMongoDBAdaptorTest.java deleted file mode 100644 index bbbd1a973a0..00000000000 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MetaMongoDBAdaptorTest.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2015-2020 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.catalog.db.mongodb; - -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.opencb.opencga.core.testclassification.duration.MediumTests; - -/** - * Created by imedina on 07/04/16. - */ -@Category(MediumTests.class) -public class MetaMongoDBAdaptorTest extends MongoDBAdaptorTest { - - @Test - public void createIndex() throws Exception { - catalogDBAdaptor.getCatalogMetaDBAdaptor().createIndexes(); - } - -} \ No newline at end of file diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MongoBackupUtils.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MongoBackupUtils.java new file mode 100644 index 00000000000..252085c353b --- /dev/null +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MongoBackupUtils.java @@ -0,0 +1,380 @@ +package org.opencb.opencga.catalog.db.mongodb; + +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoCursor; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.model.Projections; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.time.StopWatch; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogIOException; +import org.opencb.opencga.catalog.io.IOManager; +import org.opencb.opencga.catalog.managers.CatalogManager; +import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.common.UriUtils; +import org.opencb.opencga.core.models.file.File; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; +import java.util.zip.GZIPInputStream; + +/** + * Contains two methods mainly for testing purposes. One to create a dump of the current testing OpenCGA installation and a second one to + * restore it. + */ +public class MongoBackupUtils { + + private static final String TEMPORAL_FOLDER_HERE = "TEMPORAL_FOLDER_HERE"; + private static Logger logger = LoggerFactory.getLogger(MongoBackupUtils.class); + + public static void dump(CatalogManager catalogManager, Path opencgaHome) throws CatalogDBException { + StopWatch stopWatch = StopWatch.createStarted(); + try (MongoDBAdaptorFactory dbAdaptorFactory = new MongoDBAdaptorFactory(catalogManager.getConfiguration(), + catalogManager.getIoManagerFactory())) { + MongoClient mongoClient = dbAdaptorFactory.getOrganizationMongoDBAdaptorFactory(dbAdaptorFactory.getOrganizationIds().get(0)) + .getMongoDataStore().getMongoClient(); + MongoDatabase dumpDatabase = mongoClient.getDatabase("test_dump"); + dumpDatabase.drop(); + + Bson emptyBsonQuery = new Document(); + Document databaseSummary = new Document(); + for (String organizationId : dbAdaptorFactory.getOrganizationIds()) { + String databaseName = dbAdaptorFactory.getOrganizationMongoDBAdaptorFactory(organizationId).getMongoDataStore() + .getDatabaseName(); + databaseSummary.put(organizationId, databaseName); + + MongoDatabase database = mongoClient.getDatabase(databaseName); + for (String collection : OrganizationMongoDBAdaptorFactory.COLLECTIONS_LIST) { + MongoCollection dbCollection = database.getCollection(collection); + MongoCollection dumpCollection = dumpDatabase.getCollection(organizationId + "__" + collection); + + try (MongoCursor iterator = dbCollection.find(emptyBsonQuery).noCursorTimeout(true).iterator()) { + List documentList = new LinkedList<>(); + while (iterator.hasNext()) { + Document document = iterator.next(); + if (OrganizationMongoDBAdaptorFactory.FILE_COLLECTION.equals(collection) + || OrganizationMongoDBAdaptorFactory.DELETED_FILE_COLLECTION.equals(collection) + || OrganizationMongoDBAdaptorFactory.STUDY_COLLECTION.equals(collection) + || OrganizationMongoDBAdaptorFactory.DELETED_STUDY_COLLECTION.equals(collection)) { + // Store uri differently + String uri = document.getString("uri"); + String temporalFolder = opencgaHome.getFileName().toString(); + String replacedUri = uri.replace(temporalFolder, "TEMPORAL_FOLDER_HERE"); + document.put("uri", replacedUri); + } + documentList.add(document); + } + if (!documentList.isEmpty()) { + dumpCollection.insertMany(documentList); + } + } + } + } + dumpDatabase.getCollection("summary").insertOne(databaseSummary); + } + logger.info("Database dump created in {} milliseconds.", stopWatch.getTime(TimeUnit.MILLISECONDS)); + } + + public static void restore(CatalogManager catalogManager, Path opencgaHome) + throws Exception { + try (MongoDBAdaptorFactory dbAdaptorFactory = new MongoDBAdaptorFactory(catalogManager.getConfiguration(), + catalogManager.getIoManagerFactory())) { + MongoClient mongoClient = dbAdaptorFactory.getOrganizationMongoDBAdaptorFactory(ParamConstants.ADMIN_ORGANIZATION) + .getMongoDataStore().getMongoClient(); + + MongoDatabase dumpDatabase = mongoClient.getDatabase("test_dump"); + Map databaseNames = new HashMap<>(); + try (MongoCursor mongoIterator = dumpDatabase.getCollection("summary") + .find(new Document()).projection(Projections.exclude("_id")).iterator()) { + while (mongoIterator.hasNext()) { + Document document = mongoIterator.next(); + for (String s : document.keySet()) { + databaseNames.put(s, document.getString(s)); + } + } + } + + restore(catalogManager, opencgaHome, dumpDatabase, databaseNames.keySet()); + } + } + + public static void restore(CatalogManager catalogManager, Path opencgaHome, URL restoreFolder) + throws Exception { + /* + dataset=v3.0.0 + mongo test_dump --eval 'db.getCollectionNames()' --quiet | jq .[] -r | grep -v summary | while read i ; do + org=$(echo $i | cut -d "_" -f 1) ; + collection=$(echo $i | cut -d "_" -f 3) ; + mkdir -p opencga-app/src/test/resources/datasets/catalog/$dataset/mongodb/$org ; + mongo test_dump --quiet --eval "db.getCollection(\"$i\").find().forEach(function(d){ print(tojsononeline(d)); })" | gzip > opencga-app/src/test/resources/datasets/opencga/$dataset/mongodb/$org/$collection.json.gz ; + done + */ + if (restoreFolder.getProtocol().equals("file")) { + restore(catalogManager, opencgaHome, Paths.get(restoreFolder.toURI())); + } else if (restoreFolder.getProtocol().equals("jar")) { + throw new UnsupportedOperationException("Cannot restore from a jar file"); + } + } + + public static void restore(CatalogManager catalogManager, Path opencgaHome, Path restoreFolder) + throws Exception { + + List organizationIds = new ArrayList<>(); + try (Stream stream = Files.list(restoreFolder)) { + stream.forEach(file -> organizationIds.add(file.getFileName().toString())); + } + if (organizationIds.isEmpty()) { + throw new CatalogDBException("No organization found in the restore folder '" + restoreFolder + "'"); + } + restore(catalogManager, opencgaHome, restoreFolder, organizationIds); + } + + private static void restore(CatalogManager catalogManager, Path opencgaHome, Object source, Collection organizationIds) + throws Exception { + logger.info("Restore opencga from source " + source + " for organizations " + organizationIds); + StopWatch stopWatch = StopWatch.createStarted(); + try (MongoDBAdaptorFactory dbAdaptorFactory = new MongoDBAdaptorFactory(catalogManager.getConfiguration(), + catalogManager.getIoManagerFactory())) { + + for (String existingOrganizationId : dbAdaptorFactory.getOrganizationIds()) { + // We need to completely remove databases that were not backed up so tests that attempt to create them again don't fail + if (!organizationIds.contains(existingOrganizationId)) { + logger.info("Completely removing database for organization '{}'", existingOrganizationId); + dbAdaptorFactory.getOrganizationMongoDBAdaptorFactory(existingOrganizationId).deleteCatalogDB(); + } + } + + // First restore the main admin database + String adminDBName = dbAdaptorFactory.getOrganizationMongoDBAdaptorFactory(ParamConstants.ADMIN_ORGANIZATION) + .getMongoDataStore().getDatabaseName(); + logger.info("Restoring database for organization '{}'", ParamConstants.ADMIN_ORGANIZATION); + restoreDatabase(catalogManager, opencgaHome, ParamConstants.ADMIN_ORGANIZATION, adminDBName, dbAdaptorFactory, source); + + for (String organizationId : organizationIds) { + if (!ParamConstants.ADMIN_ORGANIZATION.equals(organizationId)) { + logger.info("Restoring database for organization '{}'", organizationId); + String databaseName = catalogManager.getCatalogDatabase(organizationId); + restoreDatabase(catalogManager, opencgaHome, organizationId, databaseName, dbAdaptorFactory, source); + } + } + } + logger.info("Databases restored in {} milliseconds.", stopWatch.getTime(TimeUnit.MILLISECONDS)); + } + + private static void restoreDatabase(CatalogManager catalogManager, Path opencgaHome, String organizationId, String databaseName, + MongoDBAdaptorFactory dbAdaptorFactory, Object source) + throws Exception { + MongoClient mongoClient = dbAdaptorFactory.getOrganizationMongoDBAdaptorFactory(ParamConstants.ADMIN_ORGANIZATION).getMongoDataStore().getMongoClient(); + Bson emptyBsonQuery = new Document(); + + MongoDatabase database = mongoClient.getDatabase(databaseName); + boolean databaseExists = true; + if (database.getCollection("organization").countDocuments() == 0) { + databaseExists = false; + } + + // Write projects folder in disk + IOManager ioManager = catalogManager.getIoManagerFactory().getDefault(); + Path projectsFolder = opencgaHome.resolve("sessions").resolve("orgs").resolve(organizationId).resolve("projects"); + ioManager.createDirectory(projectsFolder.toUri(), true); + + for (String collection : OrganizationMongoDBAdaptorFactory.COLLECTIONS_LIST) { + MongoCollection dbCollection = database.getCollection(collection); + + dbCollection.deleteMany(emptyBsonQuery); + try (CloseableIterator iterator = documentIterator(source, organizationId, collection)) { + List documentList = new LinkedList<>(); + while (iterator.hasNext()) { + Document document = iterator.next(); + if (OrganizationMongoDBAdaptorFactory.PROJECT_COLLECTION.equals(collection)) { + URI projectFolderUri = projectsFolder.resolve(document.getLong("uid").toString()).toUri(); + if (!ioManager.exists(projectFolderUri)) { + ioManager.createDirectory(projectsFolder.resolve(document.getLong("uid").toString()).toUri()); + } + } else if (OrganizationMongoDBAdaptorFactory.FILE_COLLECTION.equals(collection) + || OrganizationMongoDBAdaptorFactory.DELETED_FILE_COLLECTION.equals(collection) + || OrganizationMongoDBAdaptorFactory.STUDY_COLLECTION.equals(collection) + || OrganizationMongoDBAdaptorFactory.DELETED_STUDY_COLLECTION.equals(collection)) { + + // Write actual temporal folder in database + String uri = document.getString("uri"); + String uriPath = uri.substring(uri.indexOf(TEMPORAL_FOLDER_HERE) + TEMPORAL_FOLDER_HERE.length() + 1); + String replacedUri = opencgaHome.resolve(uriPath).toUri().toString(); + document.put("uri", replacedUri); + + if (OrganizationMongoDBAdaptorFactory.FILE_COLLECTION.equals(collection)) { + createFile(ioManager, document); + } else if (OrganizationMongoDBAdaptorFactory.STUDY_COLLECTION.equals(collection)) { + // Create temporal study folder + ioManager.createDirectory(UriUtils.createUri(replacedUri), true); + } + } + documentList.add(document); + } + if (!documentList.isEmpty()) { + dbCollection.insertMany(documentList); + } + } + } + + if (!databaseExists) { + // Database was completely wiped so we need to regenerate non-existing collections and indexes + logger.info("Database for organization '{}' was wiped. Restoring all indexes again.", organizationId); + OrganizationMongoDBAdaptorFactory orgFactory = dbAdaptorFactory.getOrganizationMongoDBAdaptorFactory(organizationId); + orgFactory.createIndexes(); + } + } + + private static CloseableIterator documentIterator(Object source, String organizationId, String collection) + throws IOException { + if (source instanceof MongoDatabase) { + MongoDatabase dumpDatabase = (MongoDatabase) source; + + MongoCollection dumpCollection = dumpDatabase.getCollection(organizationId + "__" + collection); + logger.info("Restoring {}:{} from database - {}", organizationId, collection, dumpCollection.getNamespace().getFullName()); + return new CloseableIterator<>(dumpCollection.find(new Document()).noCursorTimeout(true).iterator()); + } else if (source instanceof Path) { + Path dir = (Path) source; + java.io.File file = dir.resolve(organizationId).resolve(collection + ".json.gz").toFile(); + if (!file.exists()) { +// logger.info("File {} not found", file); + return new CloseableIterator<>(null, Collections.emptyIterator()); + } + logger.info("Restoring {}:{} from file - {}", organizationId, collection, file); + // Read file lines + Stream stream = new BufferedReader( + new InputStreamReader( + new GZIPInputStream( + Files.newInputStream(file.toPath())))).lines(); + + Iterator iterator = stream + .filter(s -> !s.isEmpty()) +// .peek(System.out::println) + .map(Document::parse) + .iterator(); + return new CloseableIterator<>(stream, iterator); + } else { + throw new IllegalArgumentException("Unknown restore source type " + source.getClass()); + } + } + + private static class CloseableIterator implements Iterator, AutoCloseable { + + private final AutoCloseable closeable; + private final Iterator iterator; + + public CloseableIterator(AutoCloseable closeable, Iterator iterator) { + this.closeable = closeable; + this.iterator = iterator; + } + + public & Closeable> CloseableIterator(CI c) { + this.closeable = c; + this.iterator = c; + } + + @Override + public void close() throws Exception { + if (closeable != null) { + closeable.close(); + } + } + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public T next() { + return iterator.next(); + } + } + + private static void createFile(IOManager ioManager, Document document) throws IOException, CatalogIOException { + String type = document.getString("type"); + if (File.Type.FILE.name().equals(type)) { + String uri = document.getString("uri"); + Path uriPath; + try { + uriPath = Paths.get(UriUtils.createUri(uri)); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + URI directoryUri = uriPath.getParent().toUri(); + // Ensure directories are created + if (!ioManager.exists(directoryUri)) { + ioManager.createDirectory(directoryUri, true); + logger.info("{} directory created", directoryUri); + } + // Create dummy file + createDebugFile(uriPath.toAbsolutePath().toString(), 1); + logger.info("{} file created", uri); + } + } + + /* TYPE_FILE UTILS */ + public static java.io.File createDebugFile() throws IOException { + String fileTestName = "/tmp/fileTest_" + RandomStringUtils.randomAlphanumeric(5); + return createDebugFile(fileTestName); + } + + public static java.io.File createDebugFile(String fileTestName) throws IOException { + return createDebugFile(fileTestName, 200); + } + + public static java.io.File createDebugFile(String fileTestName, int lines) throws IOException { + DataOutputStream os = new DataOutputStream(new FileOutputStream(fileTestName)); + + os.writeBytes("Debug file name: " + fileTestName + "\n"); + for (int i = 0; i < 100; i++) { + os.writeBytes(i + ", "); + } + for (int i = 0; i < lines; i++) { + os.writeBytes(RandomStringUtils.randomAlphanumeric(500)); + os.write('\n'); + } + os.close(); + + return Paths.get(fileTestName).toFile(); + } + + public static String createRandomString(int lines) { + StringBuilder stringBuilder = new StringBuilder(lines); + for (int i = 0; i < 100; i++) { + stringBuilder.append(i + ", "); + } + for (int i = 0; i < lines; i++) { + stringBuilder.append(RandomStringUtils.randomAlphanumeric(500)); + stringBuilder.append("\n"); + } + return stringBuilder.toString(); + } + + public static String getDummyVCFContent() { + return "##fileformat=VCFv4.0\n" + + "##fileDate=20090805\n" + + "##source=myImputationProgramV3.1\n" + + "##reference=1000GenomesPilot-NCBI36\n" + + "##phasing=partial\n" + + "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tNA00001\tNA00002\tNA00003\n" + + "20\t14370\trs6054257\tG\tA\t29\tPASS\tNS=3;DP=14;AF=0.5;DB;H2\tGT:GQ:DP:HQ\t0|0:48:1:51,51\t1|0:48:8:51,51\t1/1:43:5:.,.\n" + + "20\t17330\t.\tT\tA\t3\tq10\tNS=3;DP=11;AF=0.017\tGT:GQ:DP:HQ\t0|0:49:3:58,50\t0|1:3:5:65,3\t0/0:41:3\n" + + "20\t1110696\trs6040355\tA\tG,T\t67\tPASS\tNS=2;DP=10;AF=0.333,0.667;AA=T;DB\tGT:GQ:DP:HQ\t1|2:21:6:23,27\t2|1:2:0:18,2\t2/2:35:4\n" + + "20\t1230237\t.\tT\t.\t47\tPASS\tNS=3;DP=13;AA=T\tGT:GQ:DP:HQ\t0|0:54:7:56,60\t0|0:48:4:51,51\t0/0:61:2\n" + + "20\t1234567\tmicrosat1\tGTCT\tG,GTACT\t50\tPASS\tNS=3;DP=9;AA=G\tGT:GQ:DP\t0/1:35:4\t0/2:17:2\t1/1:40:3"; + } +} diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorTest.java deleted file mode 100644 index e88aeb2a0af..00000000000 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorTest.java +++ /dev/null @@ -1,263 +0,0 @@ -/* - * Copyright 2015-2020 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.catalog.db.mongodb; - -import org.apache.commons.lang3.StringUtils; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.experimental.categories.Category; -import org.junit.rules.ExpectedException; -import org.opencb.biodata.models.common.Status; -import org.opencb.commons.datastore.core.DataStoreServerAddress; -import org.opencb.commons.datastore.core.Query; -import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.commons.datastore.mongodb.MongoDataStore; -import org.opencb.commons.datastore.mongodb.MongoDataStoreManager; -import org.opencb.commons.test.GenericTest; -import org.opencb.opencga.catalog.auth.authentication.JwtManager; -import org.opencb.opencga.catalog.db.api.IndividualDBAdaptor; -import org.opencb.opencga.catalog.db.api.JobDBAdaptor; -import org.opencb.opencga.catalog.db.api.ProjectDBAdaptor; -import org.opencb.opencga.catalog.db.api.SampleDBAdaptor; -import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; -import org.opencb.opencga.catalog.exceptions.CatalogDBException; -import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.exceptions.CatalogParameterException; -import org.opencb.opencga.catalog.io.IOManagerFactory; -import org.opencb.opencga.core.common.PasswordUtils; -import org.opencb.opencga.core.common.TimeUtils; -import org.opencb.opencga.core.config.Admin; -import org.opencb.opencga.core.config.Configuration; -import org.opencb.opencga.core.models.file.File; -import org.opencb.opencga.core.models.file.FileInternal; -import org.opencb.opencga.core.models.individual.Individual; -import org.opencb.opencga.core.models.job.Job; -import org.opencb.opencga.core.models.project.Project; -import org.opencb.opencga.core.models.project.ProjectInternal; -import org.opencb.opencga.core.models.sample.Sample; -import org.opencb.opencga.core.models.study.Group; -import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.models.study.StudyInternal; -import org.opencb.opencga.core.models.user.User; -import org.opencb.opencga.core.models.user.UserInternal; -import org.opencb.opencga.core.models.user.UserQuota; -import org.opencb.opencga.core.models.user.UserStatus; -import org.opencb.opencga.core.testclassification.duration.MediumTests; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedList; - -import static org.junit.Assert.assertTrue; - -@Category(MediumTests.class) -public class MongoDBAdaptorTest extends GenericTest { - - public MongoDBAdaptorFactory catalogDBAdaptor; - - static User user1; - static User user2; - static User user3; - static User user4; - - @Rule - public ExpectedException thrown = ExpectedException.none(); - - UserMongoDBAdaptor catalogUserDBAdaptor; - ProjectMongoDBAdaptor catalogProjectDBAdaptor; - FileMongoDBAdaptor catalogFileDBAdaptor; - JobMongoDBAdaptor catalogJobDBAdaptor; - StudyMongoDBAdaptor catalogStudyDBAdaptor; - IndividualMongoDBAdaptor catalogIndividualDBAdaptor; - PanelMongoDBAdaptor catalogPanelDBAdaptor; - - private Configuration configuration; - - @After - public void after() { - catalogDBAdaptor.close(); - } - - @Before - public void before() throws IOException, CatalogException { - configuration = Configuration.load(getClass().getResource("/configuration-test.yml") - .openStream()); - - DataStoreServerAddress dataStoreServerAddress = new DataStoreServerAddress( - configuration.getCatalog().getDatabase().getHosts().get(0).split(":")[0], 27017); - - String database; - if (StringUtils.isNotEmpty(configuration.getDatabasePrefix())) { - if (!configuration.getDatabasePrefix().endsWith("_")) { - database = configuration.getDatabasePrefix() + "_catalog"; - } else { - database = configuration.getDatabasePrefix() + "catalog"; - } - } else { - database = "opencga_test_catalog"; - } - - /** - * Database is cleared before each execution - */ -// clearDB(dataStoreServerAddress, mongoCredentials); - MongoDataStoreManager mongoManager = new MongoDataStoreManager(dataStoreServerAddress.getHost(), dataStoreServerAddress.getPort()); - MongoDataStore db = mongoManager.get(database); - db.getDb().drop(); - mongoManager.close(); - - configuration.setAdmin(new Admin() - .setAlgorithm("HS256") - .setSecretKey(PasswordUtils.getStrongRandomPassword(JwtManager.SECRET_KEY_MIN_LENGTH)) - ); - catalogDBAdaptor = new MongoDBAdaptorFactory(configuration, new IOManagerFactory()); - catalogUserDBAdaptor = catalogDBAdaptor.getCatalogUserDBAdaptor(); - catalogStudyDBAdaptor = catalogDBAdaptor.getCatalogStudyDBAdaptor(); - catalogProjectDBAdaptor = catalogDBAdaptor.getCatalogProjectDbAdaptor(); - catalogFileDBAdaptor = catalogDBAdaptor.getCatalogFileDBAdaptor(); - catalogJobDBAdaptor = catalogDBAdaptor.getCatalogJobDBAdaptor(); - catalogIndividualDBAdaptor = catalogDBAdaptor.getCatalogIndividualDBAdaptor(); - catalogPanelDBAdaptor = catalogDBAdaptor.getCatalogPanelDBAdaptor(); - initDefaultCatalogDB(); - } - - Sample getSample(long studyUid, String sampleId) throws CatalogDBException { - Query query = new Query() - .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) - .append(SampleDBAdaptor.QueryParams.ID.key(), sampleId); - return catalogDBAdaptor.getCatalogSampleDBAdaptor().get(query, QueryOptions.empty()).first(); - } - - Individual getIndividual(long studyUid, String individualId) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { - Query query = new Query() - .append(IndividualDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) - .append(IndividualDBAdaptor.QueryParams.ID.key(), individualId); - return catalogIndividualDBAdaptor.get(query, QueryOptions.empty()).first(); - } - - Job getJob(long studyUid, String jobId) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { - Query query = new Query() - .append(JobDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) - .append(JobDBAdaptor.QueryParams.ID.key(), jobId); - return catalogJobDBAdaptor.get(query, QueryOptions.empty()).first(); - } - - Project getProject(String userId, String projectId) throws CatalogDBException { - Query query = new Query() - .append(ProjectDBAdaptor.QueryParams.USER_ID.key(), userId) - .append(ProjectDBAdaptor.QueryParams.ID.key(), projectId); - return catalogProjectDBAdaptor.get(query, QueryOptions.empty()).first(); - } - - public void initDefaultCatalogDB() throws CatalogException { - - assertTrue(!catalogDBAdaptor.isCatalogDBReady()); - catalogDBAdaptor.createAllCollections(configuration); - catalogDBAdaptor.initialiseMetaCollection(configuration.getAdmin()); - catalogDBAdaptor.getCatalogMetaDBAdaptor().createIndexes(); -// catalogDBAdaptor.initializeCatalogDB(new Admin()); - - /** - * Let's init the database with some basic data to perform each of the tests - */ - user1 = new User("jcoll", "Jacobo Coll", "jcoll@ebi", "", null, new UserInternal(new UserStatus()), new UserQuota(-1, -1, -1, -1), - Collections.emptyList(), Collections.emptyList(), new HashMap<>(), new LinkedList<>(), new HashMap<>()); - catalogUserDBAdaptor.insert(user1, "1234", null); - catalogProjectDBAdaptor.insert(new Project("P1", "project", "", null, 1, ProjectInternal.init()), "jcoll", null); - catalogProjectDBAdaptor.insert(new Project("P2", "project", "", null, 1, ProjectInternal.init()), "jcoll", null); - catalogProjectDBAdaptor.insert(new Project("P3", "project", "", null, 1, ProjectInternal.init()), "jcoll", null); - - user2 = new User("jmmut", "Jose Miguel", "jmmut@ebi", "ACME", new UserInternal(new UserStatus())); - catalogUserDBAdaptor.insert(user2, "1111", null); - - user3 = new User("imedina", "Nacho", "nacho@gmail", "SPAIN", null, new UserInternal(new UserStatus()), new UserQuota(-1, -1, -1, -1), - Collections.emptyList(), Collections.emptyList(), new HashMap<>(), new LinkedList<>(), new HashMap<>()); - catalogUserDBAdaptor.insert(user3, "2222", null); - catalogProjectDBAdaptor.insert(new Project("pr1", "90 GigaGenomes", TimeUtils.getTime(), TimeUtils.getTime(), "very long description", null, null, - Collections.emptyList(), 1, ProjectInternal.init(), Collections.emptyMap() - ), "imedina", null); - catalogStudyDBAdaptor.insert(catalogProjectDBAdaptor.get(new Query(ProjectDBAdaptor.QueryParams.ID.key(), "pr1"), null).first(), - new Study("name", "Study name", "ph1", TimeUtils.getTime(), TimeUtils.getTime(), "", null, null, null, 0, - Arrays.asList(new Group("@members", Collections.emptyList())), Arrays.asList( - new File("data/", File.Type.DIRECTORY, File.Format.PLAIN, File.Bioformat.NONE, "data/", null, "", - FileInternal.init(), 1000, 1), - new File("file.vcf", File.Type.FILE, File.Format.PLAIN, File.Bioformat.NONE, "data/file.vcf", null, "", - FileInternal.init(), 1000, 1) - ), Collections.emptyList(), new LinkedList<>(), new LinkedList<>(), new LinkedList<>(), - new LinkedList<>(), Collections.emptyList(), new LinkedList<>(), new LinkedList<>(), - null, null, 1, new Status(), StudyInternal.init(), new LinkedList<>(), Collections.emptyMap()), null); - - user4 = new User("pfurio", "Pedro", "pfurio@blabla", "Organization", null, new UserInternal(new UserStatus()), - new UserQuota(-1, -1, -1, -1), Collections.emptyList(), Collections.emptyList(), new HashMap<>(), new LinkedList<>(), - new HashMap<>()); - - catalogUserDBAdaptor.insert(user4, "pfuriopass", null); - catalogProjectDBAdaptor.insert(new Project("pr", "lncRNAs", TimeUtils.getTime(), TimeUtils.getTime(), "My description", null, null, - Collections.emptyList(), 1, ProjectInternal.init(), Collections.emptyMap()), "pfurio", null); - catalogStudyDBAdaptor.insert(catalogProjectDBAdaptor.get(new Query(ProjectDBAdaptor.QueryParams.ID.key(), "pr"), null).first(), - new Study("spongeScan", "spongeScan", "sponges", TimeUtils.getTime(), TimeUtils.getTime(), "", null, null, null, - 0, Arrays.asList(new Group("@members", Collections.emptyList())), Arrays.asList( - new File("data/", File.Type.DIRECTORY, File.Format.UNKNOWN, File.Bioformat.NONE, "data/", - null, "Description", FileInternal.init(), 10, 1), - new File("file1.txt", File.Type.FILE, File.Format.COMMA_SEPARATED_VALUES, - File.Bioformat.NONE, "data/file1.txt", null, "Description", FileInternal.init(), 100, 1), - new File("file2.txt", File.Type.FILE, File.Format.COMMA_SEPARATED_VALUES, - File.Bioformat.NONE, "data/file2.txt", null, "Description2", FileInternal.init(), 100, 1), - new File("alignment.bam", File.Type.FILE, File.Format.BAM, File.Bioformat.ALIGNMENT, - "data/alignment.bam", null, "Tophat alignment file", FileInternal.init(), 5000, 1) - ), Collections.emptyList(), new LinkedList<>(), new LinkedList<>(), new LinkedList<>(), new LinkedList<>(), - Collections.emptyList(), new LinkedList<>(), new LinkedList<>(), null, null, 1, new Status(), - StudyInternal.init(), new LinkedList<>(), Collections.emptyMap()), null); - catalogStudyDBAdaptor.insert(catalogProjectDBAdaptor.get(new Query(ProjectDBAdaptor.QueryParams.ID.key(), "pr"), null).first(), - new Study("mineco", "MINECO", "mineco", TimeUtils.getTime(), TimeUtils.getTime(), "", null, null, null, 0, - Arrays.asList(new Group("@members", Collections.emptyList())), Arrays.asList( - new File("data/", File.Type.DIRECTORY, File.Format.UNKNOWN, File.Bioformat.NONE, "data/", - null, "Description", FileInternal.init(), 10, 1), - new File("m_file1.txt", File.Type.FILE, File.Format.COMMA_SEPARATED_VALUES, - File.Bioformat.NONE, "data/file1.txt", null, "Description", FileInternal.init(), 100, 1), - new File("m_alignment.bam", File.Type.FILE, File.Format.BAM, File.Bioformat.ALIGNMENT, - "data/alignment.bam", null, "Tophat alignment file", FileInternal.init(), 5000, 1) - ), Collections.emptyList(), new LinkedList<>(), new LinkedList<>(), new LinkedList<>(), new LinkedList<>(), - Collections.emptyList(), new LinkedList<>(), new LinkedList<>(), null, null, 1, new Status(), - StudyInternal.init(), new LinkedList<>(), Collections.emptyMap()), null); - - QueryOptions options = new QueryOptions("includeStudies", true); - options.put("includeFiles", true); - options.put("includeJobs", true); - options.put("includeSamples", true); - user1 = catalogUserDBAdaptor.get(MongoDBAdaptorTest.user1.getId(), options).first(); - user2 = catalogUserDBAdaptor.get(MongoDBAdaptorTest.user2.getId(), options).first(); - user3 = catalogUserDBAdaptor.get(MongoDBAdaptorTest.user3.getId(), options).first(); - user4 = catalogUserDBAdaptor.get(MongoDBAdaptorTest.user4.getId(), options).first(); - - } - - public Configuration getConfiguration() { - return configuration; - } -/* - @Test - public void initializeInitializedDB() throws CatalogDBException { - assertTrue(catalogDBAdaptor.isCatalogDBReady()); - thrown.expect(CatalogDBException.class); - catalogDBAdaptor.initializeCatalogDB(); - } -*/ -} diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MongoDBUtilsTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MongoDBUtilsTest.java index 9d8729eb9b0..86f0185e8ed 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MongoDBUtilsTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MongoDBUtilsTest.java @@ -20,6 +20,7 @@ import org.junit.Assert; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.opencb.commons.test.GenericTest; import org.opencb.opencga.core.testclassification.duration.MediumTests; import java.util.Arrays; @@ -28,7 +29,7 @@ * Created by pfurio on 3/3/16. */ @Category(MediumTests.class) -public class MongoDBUtilsTest extends MongoDBAdaptorTest { +public class MongoDBUtilsTest extends GenericTest { /////////// Other tests @Test public void replaceDots() { diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/PanelMongoDBAdaptorTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/PanelMongoDBAdaptorTest.java deleted file mode 100644 index be5cb21765a..00000000000 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/PanelMongoDBAdaptorTest.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2015-2020 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.catalog.db.mongodb; - -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.opencb.opencga.catalog.exceptions.CatalogDBException; -import org.opencb.opencga.core.testclassification.duration.MediumTests; - -import static org.junit.Assert.assertEquals; - -/** - * Created by pfurio on 01/06/16. - */ -@Category(MediumTests.class) -public class PanelMongoDBAdaptorTest extends MongoDBAdaptorTest { - - @Test - public void createPanel() throws CatalogDBException { - long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); - -// DiseasePanel diseasePanel = new DiseasePanel("panel1", "Panel 1", 1, 1, "author", null, "description", Collections.emptyList(), -// Arrays.asList(new VariantPanel().setId("variant1"), new VariantPanel().setId("variant2")), -// Collections.emptyList(), Collections.emptyList(), null, Collections.emptyMap()); - -// QueryResult panel = catalogPanelDBAdaptor.insert(studyId, diseasePanel, new QueryOptions()); -// assertEquals(1, panel.getNumResults()); - } - - @Test - public void getPanel() throws CatalogDBException { - long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); -// DiseasePanel diseasePanel = new DiseasePanel("panel1", "Panel 1", 1, 1, "author", null, "description", Collections.emptyList(), Arrays.asList(new VariantPanel().setId("variant1"), -// new VariantPanel().setId("variant2")), Collections.emptyList(), Collections.emptyList(), null, -// Collections.emptyMap()); -// QueryResult panel = catalogPanelDBAdaptor.insert(studyId, diseasePanel, new QueryOptions()); - -// QueryResult panel1 = catalogPanelDBAdaptor.get(panel.first().getUid(), new QueryOptions()); -// assertEquals(1, panel1.getNumResults()); - } -} diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptorTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptorTest.java index b9c0fb05755..50f710ae3eb 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptorTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptorTest.java @@ -21,13 +21,17 @@ import org.junit.experimental.categories.Category; import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.catalog.db.api.ProjectDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.utils.FqnUtils; import org.opencb.opencga.core.config.storage.CellBaseConfiguration; import org.opencb.opencga.core.models.project.Project; import org.opencb.opencga.core.models.project.ProjectInternal; +import org.opencb.opencga.core.models.study.Study; +import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.MediumTests; import static org.junit.Assert.*; @@ -36,54 +40,52 @@ * Created by pfurio on 3/2/16. */ @Category(MediumTests.class) -public class ProjectMongoDBAdaptorTest extends MongoDBAdaptorTest { +public class ProjectMongoDBAdaptorTest extends AbstractMongoDBAdaptorTest { @Test public void createProjectTest() throws CatalogException, JsonProcessingException { - Project p = new Project("1000G", "Project about some genomes", "", "", "Cool", null, 1, ProjectInternal.init()); - System.out.println(catalogProjectDBAdaptor.insert(p, user1.getId(), null)); - p = new Project("2000G", "Project about some more genomes", "", "", "Cool", null, 1, ProjectInternal.init()); - System.out.println(catalogProjectDBAdaptor.insert(p, user1.getId(), null)); - p = new Project("pmp", "Project management project", "", "", "it is a system", null, 1, ProjectInternal.init()); - System.out.println(catalogProjectDBAdaptor.insert(p, user2.getId(), null)); - System.out.println(catalogProjectDBAdaptor.insert(p, user1.getId(), null)); - - try { - System.out.println(catalogProjectDBAdaptor.insert(p, user1.getId(), null)); - fail("Expected \"projectAlias already exists\" exception"); - } catch (CatalogDBException e) { - System.out.println(e); - } - } - - @Test - public void getProjectIdTest() throws CatalogDBException { - assertTrue(catalogProjectDBAdaptor.getId(user3.getId(), user3.getProjects().get(0).getId()) != -1); - assertTrue(catalogProjectDBAdaptor.getId(user3.getId(), "nonExistingProject") == -1); + Project p = new Project("project1", "Project about some genomes", "", "", "Cool", null, 1, ProjectInternal.init()) + .setFqn(organizationId + ":project1"); + OpenCGAResult insert = catalogProjectDBAdaptor.insert(p, null); + assertEquals(1, insert.getNumInserted()); + p = new Project("project2", "Project about some more genomes", "", "", "Cool", null, 1, ProjectInternal.init()) + .setFqn(organizationId + ":project2"); + insert = catalogProjectDBAdaptor.insert(p, null); + assertEquals(1, insert.getNumInserted()); + p = new Project("project3", "Project management project", "", "", "it is a system", null, 1, ProjectInternal.init()) + .setFqn(organizationId + ":project3"); + insert = catalogProjectDBAdaptor.insert(p, null); + assertEquals(1, insert.getNumInserted()); + + thrown.expect(CatalogDBException.class); + thrown.expectMessage("already exists"); + catalogProjectDBAdaptor.insert(p, null); } @Test public void incrementCurrentRelease() throws CatalogDBException { - long projectId = catalogProjectDBAdaptor.getId(user3.getId(), user3.getProjects().get(0).getId()); - DataResult projectDataResult = catalogProjectDBAdaptor.get(projectId, new QueryOptions(QueryOptions.INCLUDE, + long projectUid = catalogProjectDBAdaptor.get(new Query(ProjectDBAdaptor.QueryParams.ID.key(), project1), + QueryOptions.empty()).first().getUid(); + DataResult projectDataResult = catalogProjectDBAdaptor.get(projectUid, new QueryOptions(QueryOptions.INCLUDE, ProjectDBAdaptor.QueryParams.CURRENT_RELEASE.key())); assertEquals(1, projectDataResult.first().getCurrentRelease()); - catalogProjectDBAdaptor.incrementCurrentRelease(projectId); - int currentRelease = catalogProjectDBAdaptor.get(projectId, QueryOptions.empty()).first().getCurrentRelease(); + catalogProjectDBAdaptor.incrementCurrentRelease(projectUid); + int currentRelease = catalogProjectDBAdaptor.get(projectUid, QueryOptions.empty()).first().getCurrentRelease(); assertEquals(2, currentRelease); - catalogProjectDBAdaptor.incrementCurrentRelease(projectId); - currentRelease = catalogProjectDBAdaptor.get(projectId, QueryOptions.empty()).first().getCurrentRelease(); + catalogProjectDBAdaptor.incrementCurrentRelease(projectUid); + currentRelease = catalogProjectDBAdaptor.get(projectUid, QueryOptions.empty()).first().getCurrentRelease(); assertEquals(3, currentRelease); } @Test public void getProjectTest() throws CatalogDBException { - long projectId = catalogProjectDBAdaptor.getId(user3.getId(), user3.getProjects().get(0).getId()); - System.out.println("projectId = " + projectId); - DataResult project = catalogProjectDBAdaptor.get(projectId, null); + long projectUid = catalogProjectDBAdaptor.get(new Query(ProjectDBAdaptor.QueryParams.ID.key(), project1), + QueryOptions.empty()).first().getUid(); + System.out.println("projectUid = " + projectUid); + DataResult project = catalogProjectDBAdaptor.get(projectUid, null); System.out.println(project); assertNotNull(project.first()); @@ -115,9 +117,8 @@ public void getProjectTest() throws CatalogDBException { @Test public void getAllProjects() throws CatalogDBException { - DataResult allProjects = catalogProjectDBAdaptor.get(user3.getId(), null); - System.out.println(allProjects); - assertTrue(!allProjects.getResults().isEmpty()); + DataResult allProjects = catalogProjectDBAdaptor.get(new Query(), QueryOptions.empty()); + assertFalse(allProjects.getResults().isEmpty()); } /** @@ -131,12 +132,19 @@ public void getAllProjects() throws CatalogDBException { */ @Test public void renameProjectTest() throws CatalogException { - catalogProjectDBAdaptor.insert(new Project("p1", "project1", null, null, "Cool", null, 1, ProjectInternal.init()), user1.getId(), null); - Project p1 = getProject(user1.getId(), "p1"); - catalogProjectDBAdaptor.insert(new Project("p2", "project2", null, null, "Cool", null, 1, ProjectInternal.init()), user1.getId(), null); - Project p2 = getProject(user1.getId(), "p2"); + catalogProjectDBAdaptor.insert(new Project("myp1", "project1", null, null, "Cool", null, 1, ProjectInternal.init()).setFqn(FqnUtils.buildFqn(organizationId, "myp1")), null); + Project p1 = getProject("myp1"); + catalogProjectDBAdaptor.insert(new Project("myp2", "project2", null, null, "Cool", null, 1, ProjectInternal.init()).setFqn(FqnUtils.buildFqn(organizationId, "myp2")), null); + Project p2 = getProject("myp2"); + + // Add study + catalogStudyDBAdaptor.insert(p1, new Study().setId("study").setFqn(FqnUtils.buildFqn(organizationId, "myp1", "study")), null, null); catalogProjectDBAdaptor.update(p1.getUid(), new ObjectMap(ProjectDBAdaptor.QueryParams.ID.key(), "newpmp"), QueryOptions.empty()); + Project project = getProject("newpmp"); + assertEquals(FqnUtils.buildFqn(organizationId, "newpmp"), project.getFqn()); + assertEquals(1, project.getStudies().size()); + assertEquals(FqnUtils.buildFqn(organizationId, "newpmp", "study"), project.getStudies().get(0).getFqn()); try { catalogProjectDBAdaptor.update(-1, new ObjectMap(ProjectDBAdaptor.QueryParams.ID.key(), "falseProject"), QueryOptions.empty()); @@ -153,11 +161,12 @@ public void renameProjectTest() throws CatalogException { } @Test - public void test() throws Exception { - catalogProjectDBAdaptor.insert(new Project("p1", "project1", null, null, "Cool", null, 1, ProjectInternal.init()), user1.getId(), null); - Project p1 = getProject(user1.getId(), "p1"); - catalogProjectDBAdaptor.insert(new Project("p2", "project2", null, null, "Cool", null, 1, ProjectInternal.init()), user1.getId(), null); - Project p2 = getProject(user1.getId(), "p2"); + public void cellbaseUpdateTest() throws Exception { + catalogProjectDBAdaptor.insert(new Project("myp1", "project1", null, null, "Cool", null, 1, ProjectInternal.init()) + .setFqn(organizationId + ":project1"), null); + Project p1 = getProject("myp1"); + catalogProjectDBAdaptor.insert(new Project("myp2", "project2", null, null, "Cool", null, 1, ProjectInternal.init()) + .setFqn(organizationId + ":project2"), null); catalogProjectDBAdaptor.update(p1.getUid(), new ObjectMap(ProjectDBAdaptor.QueryParams.CELLBASE.key(), new CellBaseConfiguration("url", "v")), QueryOptions.empty()); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptorTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptorTest.java index 4f4d67cb179..fa908630690 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptorTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptorTest.java @@ -16,23 +16,19 @@ package org.opencb.opencga.catalog.db.mongodb; -import org.junit.*; +import org.junit.Ignore; +import org.junit.Test; import org.junit.experimental.categories.Category; -import org.junit.rules.ExpectedException; import org.opencb.biodata.models.clinical.Phenotype; -import org.opencb.biodata.models.common.Status; import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.catalog.db.DBAdaptorFactory; import org.opencb.opencga.catalog.db.api.CohortDBAdaptor; import org.opencb.opencga.catalog.db.api.FileDBAdaptor; import org.opencb.opencga.catalog.db.api.SampleDBAdaptor; -import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.managers.SampleManager; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.models.cohort.Cohort; @@ -42,10 +38,8 @@ import org.opencb.opencga.core.models.individual.IndividualInternal; import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.sample.SampleInternal; -import org.opencb.opencga.core.models.user.User; import org.opencb.opencga.core.testclassification.duration.MediumTests; -import java.io.IOException; import java.util.*; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; @@ -60,49 +54,7 @@ * @author Jacobo Coll <jacobo167@gmail.com> */ @Category(MediumTests.class) -public class SampleMongoDBAdaptorTest { - - @Rule - public ExpectedException thrown = ExpectedException.none(); - private DBAdaptorFactory dbAdaptorFactory; - private SampleDBAdaptor catalogSampleDBAdaptor; - private User user1; - private User user2; - private User user3; - private User user4; - private long studyId; - - @After - public void after() { - dbAdaptorFactory.close(); - } - - @Before - public void before() throws IOException, CatalogException { - MongoDBAdaptorTest dbAdaptorTest = new MongoDBAdaptorTest(); - dbAdaptorTest.before(); - - user1 = MongoDBAdaptorTest.user1; - user2 = MongoDBAdaptorTest.user2; - user3 = MongoDBAdaptorTest.user3; - user4 = MongoDBAdaptorTest.user4; - dbAdaptorFactory = dbAdaptorTest.catalogDBAdaptor; - catalogSampleDBAdaptor = dbAdaptorFactory.getCatalogSampleDBAdaptor(); - - studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); - catalogSampleDBAdaptor.insert(studyId, new Sample("s1", null, null, null, null, null, null, 1, 1, "", false, - Collections.emptyList(), new ArrayList<>(), new Status(), SampleInternal.init(), Collections.emptyMap()), Collections.emptyList(), null); - catalogSampleDBAdaptor.insert(studyId, new Sample("s2", null, null, null, null, null, null, 1, 1, "", false, - Collections.emptyList(), new ArrayList<>(), new Status(), SampleInternal.init(), Collections.emptyMap()), Collections.emptyList(), null); - } - - DataResult getSample(long studyUid, String sampleId, QueryOptions options) throws CatalogDBException, - CatalogParameterException, CatalogAuthorizationException { - Query query = new Query() - .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) - .append(SampleDBAdaptor.QueryParams.ID.key(), sampleId); - return catalogSampleDBAdaptor.get(query, options); - } +public class SampleMongoDBAdaptorTest extends AbstractMongoDBAdaptorTest { // @Test // public void testAnnotateSample() throws Exception { @@ -221,7 +173,7 @@ DataResult getSample(long studyUid, String sampleId, QueryOptions option // } @Test - public void searchByOntology() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public void searchByOntology() throws CatalogException { List ontologyList = Arrays.asList( new Phenotype("hpo:123", "One hpo term", "hpo", Phenotype.Status.UNKNOWN), new Phenotype("hpo:456", "Another hpo term", "hpo", Phenotype.Status.UNKNOWN), @@ -239,8 +191,8 @@ public void searchByOntology() throws CatalogDBException, CatalogParameterExcept new InternalStatus(); Sample sample2 = new Sample().setId("sample2").setPhenotypes(ontologyList).setInternal(SampleInternal.init()); - catalogSampleDBAdaptor.insert(studyId, sample1, Collections.emptyList(), new QueryOptions()); - catalogSampleDBAdaptor.insert(studyId, sample2, Collections.emptyList(), new QueryOptions()); + catalogSampleDBAdaptor.insert(studyUid, sample1, Collections.emptyList(), new QueryOptions()); + catalogSampleDBAdaptor.insert(studyUid, sample2, Collections.emptyList(), new QueryOptions()); // Start the search Query query = new Query() @@ -275,21 +227,19 @@ public void searchByOntology() throws CatalogDBException, CatalogParameterExcept @Test public void createSampleCohortTest() throws Exception { - long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); - Sample sample = new Sample("test1", null, "A description", 1); - dbAdaptorFactory.getCatalogSampleDBAdaptor().insert(studyId, sample, Collections.emptyList(), null); + dbAdaptorFactory.getCatalogSampleDBAdaptor(organizationId).insert(studyUid, sample, Collections.emptyList(), null); List samples = new ArrayList<>(); samples.add(sample); Cohort cohort = new Cohort("id_cohort_1", "", Enums.CohortType.CASE_CONTROL, "20210702", "20210702", "Test cohort", samples, 5, Collections.emptyMap()); - dbAdaptorFactory.getCatalogCohortDBAdaptor().insert(studyId, cohort, Collections.emptyList(), null); + dbAdaptorFactory.getCatalogCohortDBAdaptor(organizationId).insert(studyUid, cohort, Collections.emptyList(), null); Query query = new Query() .append(SampleDBAdaptor.QueryParams.ID.key(), "test1") - .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), studyId); + .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), studyUid); DataResult result = catalogSampleDBAdaptor.get(query, QueryOptions.empty()); assertEquals(sample.getId(), result.first().getId()); @@ -300,14 +250,12 @@ public void createSampleCohortTest() throws Exception { @Test public void createSampleTest() throws Exception { - long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); - Sample hg0097 = new Sample("HG0097", null, "A description", 1); - dbAdaptorFactory.getCatalogSampleDBAdaptor().insert(studyId, hg0097, Collections.emptyList(), null); + dbAdaptorFactory.getCatalogSampleDBAdaptor(organizationId).insert(studyUid, hg0097, Collections.emptyList(), null); Query query = new Query() .append(SampleDBAdaptor.QueryParams.ID.key(), "HG0097") - .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), studyId); + .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), studyUid); DataResult result = catalogSampleDBAdaptor.get(query, QueryOptions.empty()); assertEquals(hg0097.getId(), result.first().getId()); @@ -317,8 +265,6 @@ public void createSampleTest() throws Exception { @Test public void searchSampleByDateTest() throws Exception { - long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); - // We create two samples "created" in different years Calendar calendar = Calendar.getInstance(); calendar.set(2004, Calendar.MARCH, 24); @@ -329,111 +275,109 @@ public void searchSampleByDateTest() throws Exception { Sample sample2 = new Sample("sample2", null, "A description", 1) .setCreationDate(TimeUtils.getTime(calendar.getTime())); - dbAdaptorFactory.getCatalogSampleDBAdaptor().insert(studyId, sample1, null, null); - dbAdaptorFactory.getCatalogSampleDBAdaptor().insert(studyId, sample2, null, null); + dbAdaptorFactory.getCatalogSampleDBAdaptor(organizationId).insert(studyUid, sample1, null, null); + dbAdaptorFactory.getCatalogSampleDBAdaptor(organizationId).insert(studyUid, sample2, null, null); QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.ID.key()); Query query = new Query() - .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), studyId) + .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(SampleDBAdaptor.QueryParams.ID.key(), "~^sample"); query.put(SampleDBAdaptor.QueryParams.CREATION_DATE.key(), ">2005"); - DataResult sampleDataResult = dbAdaptorFactory.getCatalogSampleDBAdaptor().get(query, options); + DataResult sampleDataResult = dbAdaptorFactory.getCatalogSampleDBAdaptor(organizationId).get(query, options); assertEquals(1, sampleDataResult.getNumResults()); assertEquals("sample2", sampleDataResult.first().getId()); query.put(SampleDBAdaptor.QueryParams.CREATION_DATE.key(), ">200401"); - sampleDataResult = dbAdaptorFactory.getCatalogSampleDBAdaptor().get(query, options); + sampleDataResult = dbAdaptorFactory.getCatalogSampleDBAdaptor(organizationId).get(query, options); assertEquals(2, sampleDataResult.getNumResults()); query.put(SampleDBAdaptor.QueryParams.CREATION_DATE.key(), "2003-2005"); - sampleDataResult = dbAdaptorFactory.getCatalogSampleDBAdaptor().get(query, options); + sampleDataResult = dbAdaptorFactory.getCatalogSampleDBAdaptor(organizationId).get(query, options); assertEquals(1, sampleDataResult.getNumResults()); assertEquals("sample1", sampleDataResult.first().getId()); query.put(SampleDBAdaptor.QueryParams.CREATION_DATE.key(), "2003-2018"); - sampleDataResult = dbAdaptorFactory.getCatalogSampleDBAdaptor().get(query, options); + sampleDataResult = dbAdaptorFactory.getCatalogSampleDBAdaptor(organizationId).get(query, options); assertEquals(2, sampleDataResult.getNumResults()); query.put(SampleDBAdaptor.QueryParams.CREATION_DATE.key(), "<201611"); - sampleDataResult = dbAdaptorFactory.getCatalogSampleDBAdaptor().get(query, options); + sampleDataResult = dbAdaptorFactory.getCatalogSampleDBAdaptor(organizationId).get(query, options); assertEquals(1, sampleDataResult.getNumResults()); assertEquals("sample1", sampleDataResult.first().getId()); } @Test public void caseInsensitiveSearchTest() throws Exception { - long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); - Sample sample1 = new Sample("sample1", null, "A description", 1); Sample sample2 = new Sample("sample2", null, "A description", 1); Sample sample2Ext = new Sample("sample2Ext", null, "A description", 1); - dbAdaptorFactory.getCatalogSampleDBAdaptor().insert(studyId, sample1, null, null); - dbAdaptorFactory.getCatalogSampleDBAdaptor().insert(studyId, sample2, null, null); - dbAdaptorFactory.getCatalogSampleDBAdaptor().insert(studyId, sample2Ext, null, null); + dbAdaptorFactory.getCatalogSampleDBAdaptor(organizationId).insert(studyUid, sample1, null, null); + dbAdaptorFactory.getCatalogSampleDBAdaptor(organizationId).insert(studyUid, sample2, null, null); + dbAdaptorFactory.getCatalogSampleDBAdaptor(organizationId).insert(studyUid, sample2Ext, null, null); QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.ID.key()); Query query = new Query() - .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), studyId) + .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(SampleDBAdaptor.QueryParams.ID.key(), "~/SAMpl*/i"); - DataResult sampleDataResult = dbAdaptorFactory.getCatalogSampleDBAdaptor().get(query, options); + DataResult sampleDataResult = dbAdaptorFactory.getCatalogSampleDBAdaptor(organizationId).get(query, options); assertEquals(3, sampleDataResult.getNumResults()); assertTrue(Arrays.asList("sample1", "sample2", "sample2Ext") .containsAll(sampleDataResult.getResults().stream().map(Sample::getId).collect(Collectors.toList()))); query = new Query() - .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), studyId) + .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(SampleDBAdaptor.QueryParams.ID.key(), "~/SAMpl*/"); - sampleDataResult = dbAdaptorFactory.getCatalogSampleDBAdaptor().get(query, options); + sampleDataResult = dbAdaptorFactory.getCatalogSampleDBAdaptor(organizationId).get(query, options); assertEquals(0, sampleDataResult.getNumResults()); query = new Query() - .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), studyId) + .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(SampleDBAdaptor.QueryParams.ID.key(), "~/SAMple2ext/"); - sampleDataResult = dbAdaptorFactory.getCatalogSampleDBAdaptor().get(query, options); + sampleDataResult = dbAdaptorFactory.getCatalogSampleDBAdaptor(organizationId).get(query, options); assertEquals(0, sampleDataResult.getNumResults()); query = new Query() - .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), studyId) + .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(SampleDBAdaptor.QueryParams.ID.key(), "~/SAMple2ext/i"); - sampleDataResult = dbAdaptorFactory.getCatalogSampleDBAdaptor().get(query, options); + sampleDataResult = dbAdaptorFactory.getCatalogSampleDBAdaptor(organizationId).get(query, options); assertEquals(1, sampleDataResult.getNumResults()); assertEquals("sample2Ext", sampleDataResult.first().getId()); query = new Query() - .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), studyId) + .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(SampleDBAdaptor.QueryParams.ID.key(), "~/^SAMple/i"); - sampleDataResult = dbAdaptorFactory.getCatalogSampleDBAdaptor().get(query, options); + sampleDataResult = dbAdaptorFactory.getCatalogSampleDBAdaptor(organizationId).get(query, options); assertEquals(3, sampleDataResult.getNumResults()); assertTrue(Arrays.asList("sample1", "sample2", "sample2Ext") .containsAll(sampleDataResult.getResults().stream().map(Sample::getId).collect(Collectors.toList()))); query = new Query() - .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), studyId) + .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(SampleDBAdaptor.QueryParams.ID.key(), "~/^SAMple/"); - sampleDataResult = dbAdaptorFactory.getCatalogSampleDBAdaptor().get(query, options); + sampleDataResult = dbAdaptorFactory.getCatalogSampleDBAdaptor(organizationId).get(query, options); assertEquals(0, sampleDataResult.getNumResults()); query = new Query() - .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), studyId) + .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(SampleDBAdaptor.QueryParams.ID.key(), "~/^AMple/i"); - sampleDataResult = dbAdaptorFactory.getCatalogSampleDBAdaptor().get(query, options); + sampleDataResult = dbAdaptorFactory.getCatalogSampleDBAdaptor(organizationId).get(query, options); assertEquals(0, sampleDataResult.getNumResults()); query = new Query() - .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), studyId) + .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(SampleDBAdaptor.QueryParams.ID.key(), "~/eXt$/i"); - sampleDataResult = dbAdaptorFactory.getCatalogSampleDBAdaptor().get(query, options); + sampleDataResult = dbAdaptorFactory.getCatalogSampleDBAdaptor(organizationId).get(query, options); assertEquals(1, sampleDataResult.getNumResults()); assertEquals("sample2Ext", sampleDataResult.first().getId()); query = new Query() - .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), studyId) + .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(SampleDBAdaptor.QueryParams.ID.key(), "~/^Sam\\w+eXt$/i"); - sampleDataResult = dbAdaptorFactory.getCatalogSampleDBAdaptor().get(query, options); + sampleDataResult = dbAdaptorFactory.getCatalogSampleDBAdaptor(organizationId).get(query, options); assertEquals(1, sampleDataResult.getNumResults()); assertEquals("sample2Ext", sampleDataResult.first().getId()); @@ -441,15 +385,14 @@ public void caseInsensitiveSearchTest() throws Exception { // Test if we can search for samples of an individual @Test - public void getSampleWithIndividual() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { - long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); + public void getSampleWithIndividual() throws CatalogException { QueryOptions queryOptions = new QueryOptions(); // We create a new sample with the individual new InternalStatus(); Sample sample = new Sample().setId("sample1").setInternal(SampleInternal.init()); - catalogSampleDBAdaptor.insert(studyId, sample, Collections.emptyList(), queryOptions); - long sampleId = getSample(studyId, "sample1", queryOptions).first().getUid(); + catalogSampleDBAdaptor.insert(studyUid, sample, Collections.emptyList(), queryOptions); + long sampleId = getSample(studyUid, "sample1", queryOptions).getUid(); // We create an individual String individualName = "individualName"; @@ -458,17 +401,17 @@ public void getSampleWithIndividual() throws CatalogDBException, CatalogParamete .setId(individualName) .setInternal(IndividualInternal.init()) .setSamples(Arrays.asList(new Sample().setUid(sampleId).setVersion(1).setInternal(SampleInternal.init()))); - dbAdaptorFactory.getCatalogIndividualDBAdaptor().insert(studyId, individual, null, queryOptions); + dbAdaptorFactory.getCatalogIndividualDBAdaptor(organizationId).insert(studyUid, individual, null, queryOptions); // Get the sample Query query = new Query() - .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), studyId) + .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(SampleDBAdaptor.QueryParams.INDIVIDUAL_ID.key(), "individualName"); DataResult individualQuery = catalogSampleDBAdaptor.get(query, queryOptions); assertEquals("sample1", individualQuery.first().getId()); query = new Query() - .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), studyId) + .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(SampleDBAdaptor.QueryParams.INDIVIDUAL_ID.key(), "non-existing"); DataResult inexistentIndividualQuery = catalogSampleDBAdaptor.get(query, queryOptions); assertEquals(0, inexistentIndividualQuery.getNumResults()); @@ -476,17 +419,15 @@ public void getSampleWithIndividual() throws CatalogDBException, CatalogParamete @Test public void deleteSampleTest() throws Exception { - long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); - Sample hg0097 = new Sample("HG0097", null, "A description", 1) .setInternal(SampleInternal.init()); - dbAdaptorFactory.getCatalogSampleDBAdaptor().insert(studyId, hg0097, Collections.emptyList(), null); + dbAdaptorFactory.getCatalogSampleDBAdaptor(organizationId).insert(studyUid, hg0097, Collections.emptyList(), null); QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(SampleDBAdaptor.QueryParams.ID.key(), SampleDBAdaptor.QueryParams.UID.key(), SampleDBAdaptor.QueryParams.VERSION.key(), SampleDBAdaptor.QueryParams.STUDY_UID.key())); - Sample sample = getSample(studyId, "HG0097", options).first(); - dbAdaptorFactory.getCatalogSampleDBAdaptor().delete(sample); + Sample sample = getSample(studyUid, "HG0097", options); + dbAdaptorFactory.getCatalogSampleDBAdaptor(organizationId).delete(sample); Query query = new Query() .append(SampleDBAdaptor.QueryParams.UID.key(), sample.getUid()) @@ -503,47 +444,41 @@ public void deleteSampleTest() throws Exception { public void deleteSampleFail1Test() throws Exception { Sample sample = new Sample().setId("non-existing").setUid(55555555); thrown.expect(CatalogDBException.class); - dbAdaptorFactory.getCatalogSampleDBAdaptor().delete(sample); + dbAdaptorFactory.getCatalogSampleDBAdaptor(organizationId).delete(sample); } @Test @Ignore public void deleteSampleFail2Test() throws Exception { - long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); - long fileId = dbAdaptorFactory.getCatalogFileDBAdaptor().getId(user3.getProjects().get(0).getStudies().get(0).getUid(), - "data/file.vcf"); + long fileId = catalogFileDBAdaptor.getId(studyUid, "data/file.vcf"); Sample hg0097 = new Sample("HG0097", null, "A description", 1); - dbAdaptorFactory.getCatalogSampleDBAdaptor().insert(studyId, hg0097, Collections.emptyList(), null); + dbAdaptorFactory.getCatalogSampleDBAdaptor(organizationId).insert(studyUid, hg0097, Collections.emptyList(), null); - Sample sample = getSample(studyId, "HG0097", SampleManager.INCLUDE_SAMPLE_IDS).first(); - dbAdaptorFactory.getCatalogFileDBAdaptor().update(fileId, new ObjectMap(FileDBAdaptor.QueryParams.SAMPLE_IDS.key(), sample.getId()), + Sample sample = getSample(studyUid, "HG0097", SampleManager.INCLUDE_SAMPLE_IDS); + dbAdaptorFactory.getCatalogFileDBAdaptor(organizationId).update(fileId, new ObjectMap(FileDBAdaptor.QueryParams.SAMPLE_IDS.key(), sample.getId()), QueryOptions.empty()); - dbAdaptorFactory.getCatalogSampleDBAdaptor().delete(sample); + dbAdaptorFactory.getCatalogSampleDBAdaptor(organizationId).delete(sample); } @Test @Ignore // TODO: This should be tested in the sample manager, not here !!! public void deleteSampleFail3Test() throws Exception { - long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); - Sample hg0097 = new Sample("HG0097", null, "A description", 1); - dbAdaptorFactory.getCatalogSampleDBAdaptor().insert(studyId, hg0097, Collections.emptyList(), null); - Sample sample = getSample(studyId, "HG0097", SampleManager.INCLUDE_SAMPLE_IDS).first(); + dbAdaptorFactory.getCatalogSampleDBAdaptor(organizationId).insert(studyUid, hg0097, Collections.emptyList(), null); + Sample sample = getSample(studyUid, "HG0097", SampleManager.INCLUDE_SAMPLE_IDS); - dbAdaptorFactory.getCatalogCohortDBAdaptor().insert(studyId, new Cohort("Cohort", "", Enums.CohortType.COLLECTION, "", "", + dbAdaptorFactory.getCatalogCohortDBAdaptor(organizationId).insert(studyUid, new Cohort("Cohort", "", Enums.CohortType.COLLECTION, "", "", "", Collections.singletonList(sample), 1, null), null, null); thrown.expect(CatalogDBException.class); - dbAdaptorFactory.getCatalogSampleDBAdaptor().delete(sample); + dbAdaptorFactory.getCatalogSampleDBAdaptor(organizationId).delete(sample); } @Test public void createMultipleCohorts() throws Exception { - long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); - AtomicInteger numFailures = new AtomicInteger(); Function getCohortName = c -> "Cohort_" + c; int numThreads = 10; @@ -554,7 +489,7 @@ public void createMultipleCohorts() throws Exception { for (int i = 0; i < numThreads; i++) { threads.add(new Thread(() -> { try { - dbAdaptorFactory.getCatalogCohortDBAdaptor().insert(studyId, new Cohort(cohortName, "", + dbAdaptorFactory.getCatalogCohortDBAdaptor(organizationId).insert(studyUid2, new Cohort(cohortName, "", Enums.CohortType.COLLECTION, "", "", "", Collections.emptyList(), 1, null), null, null); } catch (CatalogException ignore) { numFailures.incrementAndGet(); @@ -572,8 +507,8 @@ public void createMultipleCohorts() throws Exception { } assertEquals(numCohorts * numThreads - numCohorts, numFailures.intValue()); - List cohorts = dbAdaptorFactory.getCatalogCohortDBAdaptor().get( - new Query(CohortDBAdaptor.QueryParams.STUDY_UID.key(), studyId), null).getResults(); + List cohorts = dbAdaptorFactory.getCatalogCohortDBAdaptor(organizationId).get( + new Query(CohortDBAdaptor.QueryParams.STUDY_UID.key(), studyUid2), null).getResults(); assertEquals(numCohorts, cohorts.size()); Set names = cohorts.stream().map(Cohort::getId).collect(Collectors.toSet()); for (int c = 0; c < numCohorts; c++) { diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptorTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptorTest.java index 480866110ae..f81f28b37a6 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptorTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptorTest.java @@ -19,15 +19,18 @@ import org.junit.Test; import org.junit.experimental.categories.Category; import org.opencb.commons.datastore.core.DataResult; -import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; -import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.exceptions.CatalogParameterException; +import org.opencb.opencga.catalog.utils.FqnUtils; import org.opencb.opencga.catalog.utils.ParamUtils; -import org.opencb.opencga.core.models.study.*; +import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.models.project.Project; +import org.opencb.opencga.core.models.study.Group; +import org.opencb.opencga.core.models.study.Study; +import org.opencb.opencga.core.models.study.Variable; +import org.opencb.opencga.core.models.study.VariableSet; +import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.MediumTests; import java.util.Arrays; @@ -36,39 +39,46 @@ import java.util.Set; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; /** * Created by pfurio on 19/01/16. */ @Category(MediumTests.class) -public class StudyMongoDBAdaptorTest extends MongoDBAdaptorTest { - - Study getStudy(long projectUid, String studyId) throws CatalogDBException { - Query query = new Query() - .append(StudyDBAdaptor.QueryParams.PROJECT_UID.key(), projectUid) - .append(StudyDBAdaptor.QueryParams.ID.key(), studyId); - return catalogStudyDBAdaptor.get(query, QueryOptions.empty()).first(); - } +public class StudyMongoDBAdaptorTest extends AbstractMongoDBAdaptorTest { @Test public void updateDiskUsage() throws Exception { - catalogDBAdaptor.getCatalogStudyDBAdaptor().updateDiskUsage(null, 5, 100); - assertEquals(2100, catalogStudyDBAdaptor.get(5, null).getResults().get(0).getSize()); - catalogDBAdaptor.getCatalogStudyDBAdaptor().updateDiskUsage(null, 5, -200); - assertEquals(1900, catalogStudyDBAdaptor.get(5, null).getResults().get(0).getSize()); + long size = catalogStudyDBAdaptor.get(studyUid, null).getResults().get(0).getSize(); + catalogStudyDBAdaptor.updateDiskUsage(null, studyUid, 100); + assertEquals(size + 100, catalogStudyDBAdaptor.get(studyUid, null).getResults().get(0).getSize()); + catalogStudyDBAdaptor.updateDiskUsage(null, studyUid, -200); + assertEquals(size - 100, catalogStudyDBAdaptor.get(studyUid, null).getResults().get(0).getSize()); } + private Study getMinimalStudyInstance(String projectId, String id) { + return new Study() + .setId(id) + .setFqn(FqnUtils.buildFqn(organizationId, projectId, id)); + } + /*** - * The test will check whether it is possible to create a new study using an alias that is already being used, but on a different + * The test will check whether it is possible to create a new study using an id that is already being used, but on a different * project. */ @Test - public void createStudySameAliasDifferentProject() throws CatalogException { - catalogStudyDBAdaptor.insert(user1.getProjects().get(0), new Study("Phase 1", "ph1", "", null, StudyInternal.init(), null, 1), null); - Study ph1 = getStudy(user1.getProjects().get(0).getUid(), "ph1"); - assertTrue("It is impossible creating an study with an existing alias on a different project.", ph1.getUid() > 0); + public void createStudyWithSameIdInDifferentProject() throws CatalogException { + Project project = getProject(project1); + OpenCGAResult create = catalogStudyDBAdaptor.insert(project, getMinimalStudyInstance(project1, "studyId"), null, null); + assertEquals(1, create.getNumInserted()); + Study ph1 = getStudy(project.getUid(), "studyId"); + assertNotNull(ph1); + + project = getProject(project2); + create = catalogStudyDBAdaptor.insert(project, getMinimalStudyInstance(project2, "studyId"), null, null); + assertEquals(1, create.getNumInserted()); + ph1 = getStudy(project.getUid(), "studyId"); + assertNotNull(ph1); } private DataResult createExampleVariableSet(String name, boolean confidential) throws CatalogDBException { @@ -87,9 +97,9 @@ private DataResult createExampleVariableSet(String name, boolean co )); VariableSet variableSet = new VariableSet(name, name, false, confidential, false, "My description", variables, Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), 1, Collections.emptyMap()); - catalogStudyDBAdaptor.createVariableSet(5L, variableSet); + catalogStudyDBAdaptor.createVariableSet(studyUid, variableSet); - return catalogStudyDBAdaptor.getVariableSet(5L, name, QueryOptions.empty()); + return catalogStudyDBAdaptor.getVariableSet(studyUid, name, QueryOptions.empty()); } @Test @@ -100,10 +110,10 @@ public void createVariableSetTest() throws CatalogDBException { } @Test - public void testRemoveFieldFromVariableSet() throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException { + public void testRemoveFieldFromVariableSet() throws CatalogException { DataResult variableSetDataResult = createExampleVariableSet("VARSET_1", false); - DataResult result = - catalogStudyDBAdaptor.removeFieldFromVariableSet(5L, variableSetDataResult.first().getUid(), "NAME", user3.getId()); + DataResult result = catalogStudyDBAdaptor.removeFieldFromVariableSet(5L, variableSetDataResult.first().getUid(), "NAME", + orgAdminUserId1); assertEquals(1, result.getNumUpdated()); VariableSet variableSet = catalogStudyDBAdaptor.getVariableSet(variableSetDataResult.first().getUid(), QueryOptions.empty()).first(); @@ -119,7 +129,7 @@ public void testRemoveFieldFromVariableSet() throws CatalogDBException, CatalogA // public void testRenameFieldInVariableSet() throws CatalogDBException, CatalogAuthorizationException { // DataResult variableSetDataResult = createExampleVariableSet("VARSET_1", false); // catalogStudyDBAdaptor.renameFieldVariableSet(variableSetDataResult.first().getId(), "NAME", "NEW_NAME", -// user3.getId()); +// normalUserId1); // } // // @Test @@ -128,7 +138,7 @@ public void testRemoveFieldFromVariableSet() throws CatalogDBException, CatalogA // thrown.expect(CatalogDBException.class); // thrown.expectMessage("NAM} does not exist."); // catalogStudyDBAdaptor.renameFieldVariableSet(variableSetDataResult.first().getId(), "NAM", "NEW_NAME", -// user3.getId()); +// normalUserId1); // } // // @Test @@ -136,7 +146,7 @@ public void testRemoveFieldFromVariableSet() throws CatalogDBException, CatalogA // DataResult variableSetDataResult = createExampleVariableSet("VARSET_1", false); // thrown.expect(CatalogDBException.class); // thrown.expectMessage("The variable {id: AGE} already exists."); -// catalogStudyDBAdaptor.renameFieldVariableSet(variableSetDataResult.first().getId(), "NAME", "AGE", user3.getId()); +// catalogStudyDBAdaptor.renameFieldVariableSet(variableSetDataResult.first().getId(), "NAME", "AGE", normalUserId1); // } // // @Test @@ -144,7 +154,7 @@ public void testRemoveFieldFromVariableSet() throws CatalogDBException, CatalogA // createExampleVariableSet("VARSET_1", false); // thrown.expect(CatalogDBException.class); // thrown.expectMessage("not found"); -// catalogStudyDBAdaptor.renameFieldVariableSet(-1, "NAME", "NEW_NAME", user3.getId()); +// catalogStudyDBAdaptor.renameFieldVariableSet(-1, "NAME", "NEW_NAME", normalUserId1); // } /** @@ -153,15 +163,15 @@ public void testRemoveFieldFromVariableSet() throws CatalogDBException, CatalogA * @throws CatalogDBException */ @Test - public void addFieldToVariableSetTest1() throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException { - createExampleVariableSet("VARSET_1", false); + public void addFieldToVariableSetTest1() throws CatalogException { + DataResult varset1 = createExampleVariableSet("VARSET_1", false); createExampleVariableSet("VARSET_2", true); Variable variable = new Variable("NAM", "", Variable.VariableType.STRING, "", true, false, Collections.emptyList(), null, 0, "", "", null, Collections.emptyMap()); - DataResult result = catalogStudyDBAdaptor.addFieldToVariableSet(5L, 18, variable, user3.getId()); + DataResult result = catalogStudyDBAdaptor.addFieldToVariableSet(5L, varset1.first().getUid(), variable, orgAdminUserId1); assertEquals(1, result.getNumUpdated()); - DataResult queryResult = catalogStudyDBAdaptor.getVariableSet(18L, QueryOptions.empty()); + DataResult queryResult = catalogStudyDBAdaptor.getVariableSet(varset1.first().getUid(), QueryOptions.empty()); // Check that the new variable has been inserted in the variableSet assertTrue(queryResult.first().getVariables().stream().filter(variable1 -> variable.getId().equals(variable1.getId())).findAny() @@ -170,7 +180,7 @@ public void addFieldToVariableSetTest1() throws CatalogDBException, CatalogAutho // We try to insert the same one again. thrown.expect(CatalogDBException.class); thrown.expectMessage("already exist"); - catalogStudyDBAdaptor.addFieldToVariableSet(5L, 18, variable, user3.getId()); + catalogStudyDBAdaptor.addFieldToVariableSet(5L, varset1.first().getUid(), variable, orgAdminUserId1); } /** @@ -179,171 +189,175 @@ public void addFieldToVariableSetTest1() throws CatalogDBException, CatalogAutho * @throws CatalogDBException */ @Test - public void addFieldToVariableSetTest2() throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException { - Variable variable = new Variable("NAM", "", Variable.VariableType.STRING, "", true, false, Collections.emptyList(), null, 0, "", "", null, - Collections.emptyMap()); + public void addFieldToVariableSetTest2() throws CatalogException { + Variable variable = new Variable("NAM", "", Variable.VariableType.STRING, "", true, false, Collections.emptyList(), null, 0, "", "", + null, Collections.emptyMap()); thrown.expect(CatalogDBException.class); thrown.expectMessage("not found"); - catalogStudyDBAdaptor.addFieldToVariableSet(5L, 18, variable, user3.getId()); + catalogStudyDBAdaptor.addFieldToVariableSet(5L, 2L, variable, orgAdminUserId1); } @Test public void createGroup() throws CatalogDBException { - catalogStudyDBAdaptor.createGroup(5L, new Group("name", Arrays.asList("user1", "user2"))); + catalogStudyDBAdaptor.createGroup(studyUid, new Group("name", Arrays.asList(normalUserId1, normalUserId2))); thrown.expect(CatalogDBException.class); thrown.expectMessage("Group already existed"); - catalogStudyDBAdaptor.createGroup(5L, new Group("name", Arrays.asList("user1", "user2"))); + catalogStudyDBAdaptor.createGroup(studyUid, new Group("name", Arrays.asList(normalUserId1, normalUserId2))); } @Test - public void removeUsersFromAllGroups() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { - catalogStudyDBAdaptor.createGroup(5L, new Group("name1", Arrays.asList("user1", "user2"))); - catalogStudyDBAdaptor.createGroup(5L, new Group("name2", Arrays.asList("user1", "user2", "user3"))); - catalogStudyDBAdaptor.createGroup(5L, new Group("name3", Arrays.asList("user1", "user3"))); - - DataResult group = catalogStudyDBAdaptor.getGroup(5L, null, Arrays.asList("user1", "user3")); - assertEquals(3, group.getNumResults()); - catalogStudyDBAdaptor.removeUsersFromAllGroups(5L, Arrays.asList("user1", "user3")); - group = catalogStudyDBAdaptor.getGroup(5L, null, Arrays.asList("user1", "user3")); + public void removeUsersFromAllGroups() throws CatalogException { + catalogStudyDBAdaptor.createGroup(studyUid, new Group("name1", Arrays.asList(normalUserId1, normalUserId2))); + catalogStudyDBAdaptor.createGroup(studyUid, new Group("name2", Arrays.asList(normalUserId1, normalUserId2, normalUserId3))); + catalogStudyDBAdaptor.createGroup(studyUid, new Group("name3", Arrays.asList(normalUserId1, normalUserId3))); + + DataResult group = catalogStudyDBAdaptor.getGroup(studyUid, null, Arrays.asList(normalUserId1, normalUserId3)); + assertEquals(5, group.getNumResults()); + catalogStudyDBAdaptor.removeUsersFromAllGroups(studyUid, Arrays.asList(normalUserId1, normalUserId3)); + group = catalogStudyDBAdaptor.getGroup(studyUid, null, Arrays.asList(normalUserId1, normalUserId3)); assertEquals(0, group.getNumResults()); } @Test - public void resyncUserWithSyncedGroups() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { - // We create synced groups and not synced groups in study 5 - Group group = new Group("@notSyncedGroup", Arrays.asList("user1", "user2", "user3")); - catalogStudyDBAdaptor.createGroup(5L, group); + public void resyncUserWithSyncedGroups() throws CatalogException { + // We create synced groups and not synced groups in study studyUid + Group group = new Group("@notSyncedGroup", Arrays.asList(normalUserId1, normalUserId2, normalUserId3)); + catalogStudyDBAdaptor.createGroup(studyUid, group); group.setId("@syncedGroup1"); group.setSyncedFrom(new Group.Sync("origin1", "@syncedGroup1")); - catalogStudyDBAdaptor.createGroup(5L, group); + catalogStudyDBAdaptor.createGroup(studyUid, group); group.setId("@syncedGroup2"); group.setSyncedFrom(new Group.Sync("origin1", "@syncedGroup2")); - catalogStudyDBAdaptor.createGroup(5L, group); + catalogStudyDBAdaptor.createGroup(studyUid, group); group.setId("@syncedGroup3"); group.setSyncedFrom(new Group.Sync("otherOrigin", "@syncedGroup3")); - catalogStudyDBAdaptor.createGroup(5L, group); - group = new Group("@otherNotSyncedGroup", Arrays.asList("user1", "user3")); - catalogStudyDBAdaptor.createGroup(5L, group); + catalogStudyDBAdaptor.createGroup(studyUid, group); + group = new Group("@otherNotSyncedGroup", Arrays.asList(normalUserId1, normalUserId3)); + catalogStudyDBAdaptor.createGroup(studyUid, group); - // We create the same synced groups and not synced groups in study 9 - group = new Group("@notSyncedGroup", Arrays.asList("user1", "user2", "user3")); - catalogStudyDBAdaptor.createGroup(9L, group); + // We create the same synced groups and not synced groups in study studyUid2 + group = new Group("@notSyncedGroup", Arrays.asList(normalUserId1, normalUserId2, normalUserId3)); + catalogStudyDBAdaptor.createGroup(studyUid2, group); group.setId("@syncedGroup1"); group.setSyncedFrom(new Group.Sync("origin1", "@syncedGroup1")); - catalogStudyDBAdaptor.createGroup(9L, group); + catalogStudyDBAdaptor.createGroup(studyUid2, group); group.setId("@syncedGroup2"); group.setSyncedFrom(new Group.Sync("origin1", "@syncedGroup2")); - catalogStudyDBAdaptor.createGroup(9L, group); + catalogStudyDBAdaptor.createGroup(studyUid2, group); group.setId("@syncedGroup3"); group.setSyncedFrom(new Group.Sync("otherOrigin", "@syncedGroup3")); - catalogStudyDBAdaptor.createGroup(9L, group); - group = new Group("@otherNotSyncedGroup", Arrays.asList("user1", "user3")); - catalogStudyDBAdaptor.createGroup(9L, group); + catalogStudyDBAdaptor.createGroup(studyUid2, group); + group = new Group("@otherNotSyncedGroup", Arrays.asList(normalUserId1, normalUserId3)); + catalogStudyDBAdaptor.createGroup(studyUid2, group); - catalogStudyDBAdaptor.resyncUserWithSyncedGroups("user2", Collections.emptyList(), "origin1"); - DataResult groupsStudy1 = catalogStudyDBAdaptor.getGroup(5L, null, Arrays.asList("user2")); - DataResult groupsStudy2 = catalogStudyDBAdaptor.getGroup(9L, null, Arrays.asList("user2")); - assertEquals(groupsStudy1.getNumResults(), groupsStudy2.getNumResults()); - assertEquals(2, groupsStudy1.getNumResults()); + catalogStudyDBAdaptor.resyncUserWithSyncedGroups(normalUserId2, Collections.emptyList(), "origin1"); + DataResult groupsStudy1 = catalogStudyDBAdaptor.getGroup(studyUid, null, Arrays.asList(normalUserId2)); + DataResult groupsStudy2 = catalogStudyDBAdaptor.getGroup(studyUid2, null, Arrays.asList(normalUserId2)); + assertEquals(4, groupsStudy1.getNumResults()); + assertEquals(2, groupsStudy2.getNumResults()); assertTrue(groupsStudy1.getResults().stream().map(Group::getId).collect(Collectors.toList()) .containsAll(Arrays.asList("@notSyncedGroup", "@syncedGroup3"))); + assertTrue(groupsStudy2.getResults().stream().map(Group::getId).collect(Collectors.toList()) + .containsAll(Arrays.asList("@notSyncedGroup", "@syncedGroup3"))); // Nothing should change with this resync. Group1 doesn't exist and syncedGroup3 is not from origin1. // But because this time it will try to insert users to groups, user2 will be automatically added to group @members - catalogStudyDBAdaptor.resyncUserWithSyncedGroups("user2", Arrays.asList("@group1", "@syncedGroup3"), "origin1"); - groupsStudy1 = catalogStudyDBAdaptor.getGroup(5L, null, Arrays.asList("user2")); - groupsStudy2 = catalogStudyDBAdaptor.getGroup(9L, null, Arrays.asList("user2")); - assertEquals(groupsStudy1.getNumResults(), groupsStudy2.getNumResults()); - assertEquals(3, groupsStudy1.getNumResults()); - assertTrue(groupsStudy1.getResults().stream().map(Group::getId).collect(Collectors.toList()) - .containsAll(Arrays.asList("@notSyncedGroup", "@syncedGroup3", "@members"))); + catalogStudyDBAdaptor.resyncUserWithSyncedGroups(normalUserId2, Arrays.asList("@group1", "@syncedGroup3"), "origin1"); +// groupsStudy1 = catalogStudyDBAdaptor.getGroup(studyUid, null, Arrays.asList(normalUserId2)); + groupsStudy2 = catalogStudyDBAdaptor.getGroup(studyUid2, null, Arrays.asList(normalUserId2)); +// assertEquals(groupsStudy1.getNumResults(), groupsStudy2.getNumResults()); + assertEquals(3, groupsStudy2.getNumResults()); + assertTrue(groupsStudy2.getResults().stream().map(Group::getId).collect(Collectors.toList()) + .containsAll(Arrays.asList("@notSyncedGroup", "@syncedGroup3", ParamConstants.MEMBERS_GROUP))); // Now we add one new user that will have to be added to @syncedGroup3 only. It didn't still exist there catalogStudyDBAdaptor.resyncUserWithSyncedGroups("user5", Arrays.asList("@group1", "@syncedGroup3"), "otherOrigin"); - groupsStudy1 = catalogStudyDBAdaptor.getGroup(5L, null, Arrays.asList("user5")); - groupsStudy2 = catalogStudyDBAdaptor.getGroup(9L, null, Arrays.asList("user5")); + groupsStudy1 = catalogStudyDBAdaptor.getGroup(studyUid, null, Arrays.asList("user5")); + groupsStudy2 = catalogStudyDBAdaptor.getGroup(studyUid2, null, Arrays.asList("user5")); assertEquals(groupsStudy1.getNumResults(), groupsStudy2.getNumResults()); assertEquals(2, groupsStudy1.getNumResults()); assertTrue(groupsStudy1.getResults().stream().map(Group::getId).collect(Collectors.toList()) - .containsAll(Arrays.asList("@syncedGroup3", "@members"))); + .containsAll(Arrays.asList("@syncedGroup3", ParamConstants.MEMBERS_GROUP))); - catalogStudyDBAdaptor.resyncUserWithSyncedGroups("user2", Arrays.asList("@group1", "@syncedGroup2", "@syncedGroup3"), "origin1"); - groupsStudy1 = catalogStudyDBAdaptor.getGroup(5L, null, Arrays.asList("user2")); - groupsStudy2 = catalogStudyDBAdaptor.getGroup(9L, null, Arrays.asList("user2")); - assertEquals(groupsStudy1.getNumResults(), groupsStudy2.getNumResults()); - assertEquals(4, groupsStudy1.getNumResults()); + catalogStudyDBAdaptor.resyncUserWithSyncedGroups(normalUserId2, Arrays.asList("@group1", "@syncedGroup2", "@syncedGroup3"), "origin1"); + groupsStudy1 = catalogStudyDBAdaptor.getGroup(studyUid, null, Arrays.asList(normalUserId2)); + groupsStudy2 = catalogStudyDBAdaptor.getGroup(studyUid2, null, Arrays.asList(normalUserId2)); + assertEquals(5, groupsStudy1.getNumResults()); + assertEquals(4, groupsStudy2.getNumResults()); assertTrue(groupsStudy1.getResults().stream().map(Group::getId).collect(Collectors.toList()) - .containsAll(Arrays.asList("@notSyncedGroup", "@syncedGroup2", "@syncedGroup3", "@members"))); + .containsAll(Arrays.asList("@notSyncedGroup", "@syncedGroup2", "@syncedGroup3", ParamConstants.MEMBERS_GROUP))); + assertTrue(groupsStudy2.getResults().stream().map(Group::getId).collect(Collectors.toList()) + .containsAll(Arrays.asList("@notSyncedGroup", "@syncedGroup2", "@syncedGroup3", ParamConstants.MEMBERS_GROUP))); } @Test - public void updateUserToGroups() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { - // We create synced groups and not synced groups in study 5 + public void updateUserToGroups() throws CatalogException { + // We create synced groups and not synced groups in study studyUid Group group = new Group("@notSyncedGroup", Collections.emptyList()); - catalogStudyDBAdaptor.createGroup(5L, group); + catalogStudyDBAdaptor.createGroup(studyUid, group); group.setId("@syncedGroup1"); group.setSyncedFrom(new Group.Sync("origin1", "@syncedGroup1")); - catalogStudyDBAdaptor.createGroup(5L, group); + catalogStudyDBAdaptor.createGroup(studyUid, group); group.setId("@syncedGroup2"); group.setSyncedFrom(new Group.Sync("origin1", "@syncedGroup2")); - catalogStudyDBAdaptor.createGroup(5L, group); + catalogStudyDBAdaptor.createGroup(studyUid, group); group.setId("@syncedGroup3"); group.setSyncedFrom(new Group.Sync("otherOrigin", "@syncedGroup3")); - catalogStudyDBAdaptor.createGroup(5L, group); + catalogStudyDBAdaptor.createGroup(studyUid, group); - // We create the same synced groups and not synced groups in study 9 + // We create the same synced groups and not synced groups in study studyUid2 group = new Group("@notSyncedGroup", Collections.emptyList()); - catalogStudyDBAdaptor.createGroup(9L, group); + catalogStudyDBAdaptor.createGroup(studyUid2, group); group.setId("@syncedGroup1"); group.setSyncedFrom(new Group.Sync("origin1", "@syncedGroup1")); - catalogStudyDBAdaptor.createGroup(9L, group); + catalogStudyDBAdaptor.createGroup(studyUid2, group); group.setId("@syncedGroup2"); group.setSyncedFrom(new Group.Sync("origin1", "@syncedGroup2")); - catalogStudyDBAdaptor.createGroup(9L, group); + catalogStudyDBAdaptor.createGroup(studyUid2, group); group.setId("@syncedGroup3"); group.setSyncedFrom(new Group.Sync("otherOrigin", "@syncedGroup3")); - catalogStudyDBAdaptor.createGroup(9L, group); + catalogStudyDBAdaptor.createGroup(studyUid2, group); group = new Group("@otherNotSyncedGroup", Collections.emptyList()); - catalogStudyDBAdaptor.createGroup(9L, group); + catalogStudyDBAdaptor.createGroup(studyUid2, group); - catalogStudyDBAdaptor.updateUserFromGroups("user2", null, Arrays.asList("syncedGroup1", "notSyncedGroup"), ParamUtils.AddRemoveAction.ADD); - DataResult groupsStudy1 = catalogStudyDBAdaptor.getGroup(5L, null, Arrays.asList("user2")); - DataResult groupsStudy2 = catalogStudyDBAdaptor.getGroup(9L, null, Arrays.asList("user2")); - assertEquals(groupsStudy1.getNumResults(), groupsStudy2.getNumResults()); - assertEquals(3, groupsStudy1.getNumResults()); + catalogStudyDBAdaptor.updateUserFromGroups(normalUserId2, null, Arrays.asList("syncedGroup1", "notSyncedGroup"), ParamUtils.AddRemoveAction.ADD); + DataResult groupsStudy1 = catalogStudyDBAdaptor.getGroup(studyUid, null, Arrays.asList(normalUserId2)); + DataResult groupsStudy2 = catalogStudyDBAdaptor.getGroup(studyUid2, null, Arrays.asList(normalUserId2)); + assertEquals(4, groupsStudy1.getNumResults()); assertTrue(groupsStudy1.getResults().stream().map(Group::getId).collect(Collectors.toList()) - .containsAll(Arrays.asList("@notSyncedGroup", "@syncedGroup1", "@members"))); + .containsAll(Arrays.asList(restrictedGroup, "@notSyncedGroup", "@syncedGroup1", ParamConstants.MEMBERS_GROUP))); assertEquals(3, groupsStudy2.getNumResults()); assertTrue(groupsStudy2.getResults().stream().map(Group::getId).collect(Collectors.toList()) - .containsAll(Arrays.asList("@notSyncedGroup", "@syncedGroup1", "@members"))); + .containsAll(Arrays.asList("@notSyncedGroup", "@syncedGroup1", ParamConstants.MEMBERS_GROUP))); - catalogStudyDBAdaptor.updateUserFromGroups("user2", null, Arrays.asList("syncedGroup1", "notSyncedGroup"), ParamUtils.AddRemoveAction.REMOVE); - groupsStudy1 = catalogStudyDBAdaptor.getGroup(5L, null, Arrays.asList("user2")); - groupsStudy2 = catalogStudyDBAdaptor.getGroup(9L, null, Arrays.asList("user2")); - assertEquals(1, groupsStudy1.getNumResults()); + catalogStudyDBAdaptor.updateUserFromGroups(normalUserId2, null, Arrays.asList("syncedGroup1", "notSyncedGroup"), ParamUtils.AddRemoveAction.REMOVE); + groupsStudy1 = catalogStudyDBAdaptor.getGroup(studyUid, null, Arrays.asList(normalUserId2)); + groupsStudy2 = catalogStudyDBAdaptor.getGroup(studyUid2, null, Arrays.asList(normalUserId2)); + assertEquals(2, groupsStudy1.getNumResults()); assertEquals(1, groupsStudy2.getNumResults()); - assertEquals("@members", groupsStudy1.first().getId()); - assertEquals("@members", groupsStudy2.first().getId()); + assertTrue(groupsStudy1.getResults().stream().map(Group::getId).collect(Collectors.toList()) + .containsAll(Arrays.asList(restrictedGroup, ParamConstants.MEMBERS_GROUP))); + assertEquals(ParamConstants.MEMBERS_GROUP, groupsStudy2.first().getId()); - catalogStudyDBAdaptor.updateUserFromGroups("user2", Arrays.asList(5L, 9L), Arrays.asList("syncedGroup1", "notSyncedGroup"), ParamUtils.AddRemoveAction.ADD); - groupsStudy1 = catalogStudyDBAdaptor.getGroup(5L, null, Arrays.asList("user2")); - groupsStudy2 = catalogStudyDBAdaptor.getGroup(9L, null, Arrays.asList("user2")); - assertEquals(groupsStudy1.getNumResults(), groupsStudy2.getNumResults()); - assertEquals(3, groupsStudy1.getNumResults()); + catalogStudyDBAdaptor.updateUserFromGroups(normalUserId2, Arrays.asList(studyUid, studyUid2), Arrays.asList("syncedGroup1", "notSyncedGroup"), ParamUtils.AddRemoveAction.ADD); + groupsStudy1 = catalogStudyDBAdaptor.getGroup(studyUid, null, Arrays.asList(normalUserId2)); + groupsStudy2 = catalogStudyDBAdaptor.getGroup(studyUid2, null, Arrays.asList(normalUserId2)); + assertEquals(4, groupsStudy1.getNumResults()); assertTrue(groupsStudy1.getResults().stream().map(Group::getId).collect(Collectors.toList()) - .containsAll(Arrays.asList("@notSyncedGroup", "@syncedGroup1", "@members"))); + .containsAll(Arrays.asList(restrictedGroup, "@notSyncedGroup", "@syncedGroup1", ParamConstants.MEMBERS_GROUP))); assertEquals(3, groupsStudy2.getNumResults()); assertTrue(groupsStudy2.getResults().stream().map(Group::getId).collect(Collectors.toList()) - .containsAll(Arrays.asList("@notSyncedGroup", "@syncedGroup1", "@members"))); + .containsAll(Arrays.asList("@notSyncedGroup", "@syncedGroup1", ParamConstants.MEMBERS_GROUP))); - catalogStudyDBAdaptor.updateUserFromGroups("user2", Collections.singletonList(5L), Arrays.asList("syncedGroup1", "notSyncedGroup"), ParamUtils.AddRemoveAction.REMOVE); - groupsStudy1 = catalogStudyDBAdaptor.getGroup(5L, null, Arrays.asList("user2")); - groupsStudy2 = catalogStudyDBAdaptor.getGroup(9L, null, Arrays.asList("user2")); - assertEquals(1, groupsStudy1.getNumResults()); - assertEquals("@members", groupsStudy1.first().getId()); + catalogStudyDBAdaptor.updateUserFromGroups(normalUserId2, Collections.singletonList(studyUid), Arrays.asList("syncedGroup1", "notSyncedGroup"), ParamUtils.AddRemoveAction.REMOVE); + groupsStudy1 = catalogStudyDBAdaptor.getGroup(studyUid, null, Arrays.asList(normalUserId2)); + groupsStudy2 = catalogStudyDBAdaptor.getGroup(studyUid2, null, Arrays.asList(normalUserId2)); + assertEquals(2, groupsStudy1.getNumResults()); + assertTrue(groupsStudy1.getResults().stream().map(Group::getId).collect(Collectors.toList()) + .containsAll(Arrays.asList(restrictedGroup, ParamConstants.MEMBERS_GROUP))); assertEquals(3, groupsStudy2.getNumResults()); assertTrue(groupsStudy2.getResults().stream().map(Group::getId).collect(Collectors.toList()) - .containsAll(Arrays.asList("@notSyncedGroup", "@syncedGroup1", "@members"))); + .containsAll(Arrays.asList("@notSyncedGroup", "@syncedGroup1", ParamConstants.MEMBERS_GROUP))); } } diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptorTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptorTest.java index 34d0328b8dc..8e4571d86ea 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptorTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptorTest.java @@ -16,12 +16,14 @@ package org.opencb.opencga.catalog.db.mongodb; +import org.bson.Document; import org.junit.Test; import org.junit.experimental.categories.Category; import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.TestParamConstants; import org.opencb.opencga.catalog.db.api.UserDBAdaptor; import org.opencb.opencga.catalog.exceptions.*; import org.opencb.opencga.core.models.common.Enums; @@ -44,20 +46,24 @@ * Created by pfurio on 19/01/16. */ @Category(MediumTests.class) -public class UserMongoDBAdaptorTest extends MongoDBAdaptorTest { +public class UserMongoDBAdaptorTest extends AbstractMongoDBAdaptorTest { @Test public void nativeGet() throws Exception { - Query query = new Query("id", "imedina"); - DataResult queryResult = catalogUserDBAdaptor.nativeGet(query, null); + Query query = new Query(UserDBAdaptor.QueryParams.ID.key(), normalUserId1); + DataResult queryResult = catalogUserDBAdaptor.nativeGet(query, null); + assertEquals(1, queryResult.getNumResults()); + assertEquals(normalUserId1, queryResult.first().getString(UserDBAdaptor.QueryParams.ID.key())); } @Test public void createUserTest() throws CatalogException { User user = new User("NewUser", "", "", "", new UserInternal(new UserStatus())); - catalogUserDBAdaptor.insert(user, "", null); + OpenCGAResult insert = catalogUserDBAdaptor.insert(user, "", null); + assertEquals(1, insert.getNumInserted()); thrown.expect(CatalogDBException.class); + thrown.expectMessage("already exists"); catalogUserDBAdaptor.insert(user, "", null); } @@ -67,9 +73,8 @@ public void deleteUserTest() throws CatalogException { catalogUserDBAdaptor.insert(deletable1, "1234", null); Query query = new Query(UserDBAdaptor.QueryParams.ID.key(), "deletable1"); DataResult userResult = catalogUserDBAdaptor.get(query, QueryOptions.empty()); - assertFalse(userResult.getResults().isEmpty()); - assertNotNull(userResult.first()); - + assertEquals(1, userResult.getNumResults()); + assertEquals("deletable1", userResult.first().getId()); assertEquals(InternalStatus.READY, userResult.first().getInternal().getStatus().getId()); DataResult deleteUser = catalogUserDBAdaptor.delete(deletable1.getId(), new QueryOptions()); @@ -78,55 +83,47 @@ public void deleteUserTest() throws CatalogException { query.append(UserDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), UserStatus.DELETED); DataResult queryResult = catalogUserDBAdaptor.get(query, QueryOptions.empty()); assertEquals(InternalStatus.DELETED, queryResult.first().getInternal().getStatus().getId()); - - - /* - thrown.expect(CatalogDBException.class); - catalogUserDBAdaptor.delete(deletable1.getId()); - */ } @Test public void getUserTest() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { - DataResult user = catalogUserDBAdaptor.get(user1.getId(), null); - assertNotSame(0, user.getResults().size()); - - user = catalogUserDBAdaptor.get(user3.getId(), null); - assertFalse(user.getResults().isEmpty()); - assertFalse(user.first().getProjects().isEmpty()); + DataResult user = catalogUserDBAdaptor.get(normalUserId1, null); + assertEquals(1, user.getNumResults()); + assertEquals(normalUserId1, user.first().getId()); - user = catalogUserDBAdaptor.get(user3.getId(), new QueryOptions("exclude", Arrays.asList("projects"))); - assertEquals(null, user.first().getProjects()); + user = catalogUserDBAdaptor.get(normalUserId3, null); + assertEquals(1, user.getNumResults()); + assertEquals(normalUserId3, user.first().getId()); OpenCGAResult nonExistingUser = catalogUserDBAdaptor.get("NonExistingUser", null); assertEquals(0, nonExistingUser.getNumResults()); } @Test - public void changePasswordTest() throws CatalogDBException, CatalogAuthenticationException { - DataResult result = catalogUserDBAdaptor.changePassword(user2.getId(), "1111", "1234"); + public void changePasswordTest() throws CatalogException { + DataResult result = catalogUserDBAdaptor.changePassword(normalUserId1, TestParamConstants.PASSWORD, "1234"); assertEquals(1, result.getNumUpdated()); thrown.expect(CatalogAuthenticationException.class); - catalogUserDBAdaptor.changePassword(user2.getId(), "BAD_PASSWORD", "asdf"); + catalogUserDBAdaptor.changePassword(normalUserId1, "BAD_PASSWORD", "asdf"); } @Test public void modifyUserTest() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { ObjectMap genomeMapsConfig = new ObjectMap("lastPosition", "4:1222222:1333333"); genomeMapsConfig.put("otherConf", Arrays.asList(1, 2, 3, 4, 5)); - catalogUserDBAdaptor.setConfig(user1.getId(), "genomemaps", genomeMapsConfig); - catalogUserDBAdaptor.setConfig(user1.getId(), "genomemaps2", genomeMapsConfig); + catalogUserDBAdaptor.setConfig(normalUserId1, "genomemaps", genomeMapsConfig); + catalogUserDBAdaptor.setConfig(normalUserId1, "genomemaps2", genomeMapsConfig); - User user = catalogUserDBAdaptor.get(user1.getId(), null).first(); + User user = catalogUserDBAdaptor.get(normalUserId1, null).first(); assertNotNull(user.getConfigs().get("genomemaps")); assertNotNull(user.getConfigs().get("genomemaps2")); Map genomemaps = user.getConfigs().get("genomemaps"); assertNotNull(genomemaps.get("otherConf")); assertNotNull(genomemaps.get("lastPosition")); - catalogUserDBAdaptor.deleteConfig(user1.getId(), "genomemaps"); - user = catalogUserDBAdaptor.get(user1.getId(), null).first(); + catalogUserDBAdaptor.deleteConfig(normalUserId1, "genomemaps"); + user = catalogUserDBAdaptor.get(normalUserId1, null).first(); assertNull(user.getConfigs().get("genomemaps")); assertNotNull(user.getConfigs().get("genomemaps2")); } @@ -137,8 +134,8 @@ public void addFilterTest() throws CatalogDBException, IOException, CatalogParam QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList("key1", "key2")); UserFilter filter = new UserFilter("filter1", "Description of filter 1", Enums.Resource.ALIGNMENT, query, options); - catalogUserDBAdaptor.addFilter(user4.getId(), filter); - DataResult userDataResult = catalogUserDBAdaptor.get(user4.getId(), new QueryOptions()); + catalogUserDBAdaptor.addFilter(normalUserId1, filter); + DataResult userDataResult = catalogUserDBAdaptor.get(normalUserId1, new QueryOptions()); UserFilter filterResult = userDataResult.first().getFilters().get(0); @@ -154,16 +151,16 @@ public void updateFilterTest() throws CatalogDBException, CatalogParameterExcept Query query = new Query("key1", "value1").append("key2", "value2"); QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList("key1", "key2")); UserFilter filter = new UserFilter("filter1", "Description of filter 1", Enums.Resource.ALIGNMENT, query, options); - catalogUserDBAdaptor.addFilter(user4.getId(), filter); + catalogUserDBAdaptor.addFilter(normalUserId1, filter); ObjectMap params = new ObjectMap() .append(UserDBAdaptor.FilterParams.DESCRIPTION.key(), "The description has changed") .append(UserDBAdaptor.FilterParams.RESOURCE.key(), Enums.Resource.VARIANT) .append(UserDBAdaptor.FilterParams.QUERY.key(), new Query("key3", "whatever")) .append(UserDBAdaptor.FilterParams.OPTIONS.key(), new QueryOptions("options", "optionsValue")); - catalogUserDBAdaptor.updateFilter(user4.getId(), filter.getId(), params); + catalogUserDBAdaptor.updateFilter(normalUserId1, filter.getId(), params); - DataResult userDataResult = catalogUserDBAdaptor.get(user4.getId(), new QueryOptions()); + DataResult userDataResult = catalogUserDBAdaptor.get(normalUserId1, new QueryOptions()); UserFilter filterResult = userDataResult.first().getFilters().get(0); @@ -181,9 +178,9 @@ public void deleteFilterTest() throws CatalogDBException, IOException, CatalogPa QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList("key1", "key2")); UserFilter filter = new UserFilter("filter1", "Description of filter 1", Enums.Resource.ALIGNMENT, query, options); - catalogUserDBAdaptor.addFilter(user4.getId(), filter); - catalogUserDBAdaptor.deleteFilter(user4.getId(), filter.getId()); - DataResult userDataResult = catalogUserDBAdaptor.get(user4.getId(), new QueryOptions()); + catalogUserDBAdaptor.addFilter(normalUserId1, filter); + catalogUserDBAdaptor.deleteFilter(normalUserId1, filter.getId()); + DataResult userDataResult = catalogUserDBAdaptor.get(normalUserId1, new QueryOptions()); List filters = userDataResult.first().getFilters(); assertTrue(filters.size() == 0); @@ -196,21 +193,21 @@ public void setConfigTest() throws CatalogDBException, CatalogParameterException .append("key1", Arrays.asList(1, 2, 3, 4, 5)) .append("key2", new ObjectMap("key21", 21).append("key22", 22)); - DataResult writeResult = catalogUserDBAdaptor.setConfig(user4.getId(), "config1", objectMap); + DataResult writeResult = catalogUserDBAdaptor.setConfig(normalUserId1, "config1", objectMap); assertEquals(1, writeResult.getNumUpdated()); - DataResult queryResult = catalogUserDBAdaptor.get(user4.getId(), QueryOptions.empty()); + DataResult queryResult = catalogUserDBAdaptor.get(normalUserId1, QueryOptions.empty()); ObjectMap result = queryResult.first().getConfigs().get("config1"); assertTrue(result.get("key1") instanceof List); assertTrue(result.get("key2") instanceof Map); // Update the config objectMap.put("key2", objectMap.get("key1")); - writeResult = catalogUserDBAdaptor.setConfig(user4.getId(), "config1", objectMap); + writeResult = catalogUserDBAdaptor.setConfig(normalUserId1, "config1", objectMap); assertEquals(1, writeResult.getNumUpdated()); - queryResult = catalogUserDBAdaptor.get(user4.getId(), QueryOptions.empty()); + queryResult = catalogUserDBAdaptor.get(normalUserId1, QueryOptions.empty()); result = queryResult.first().getConfigs().get("config1"); assertTrue(result.get("key1") instanceof List); @@ -223,13 +220,13 @@ public void deleteConfigTest() throws CatalogDBException, IOException { .append("key1", Arrays.asList(1, 2, 3, 4, 5)) .append("key2", new ObjectMap("key21", 21).append("key22", 22)); - catalogUserDBAdaptor.setConfig(user4.getId(), "config1", objectMap); + catalogUserDBAdaptor.setConfig(normalUserId1, "config1", objectMap); - catalogUserDBAdaptor.deleteConfig(user4.getId(), "config1"); + catalogUserDBAdaptor.deleteConfig(normalUserId1, "config1"); thrown.expect(CatalogDBException.class); thrown.expectMessage("Could not delete config1 configuration"); - catalogUserDBAdaptor.deleteConfig(user4.getId(), "config1"); + catalogUserDBAdaptor.deleteConfig(normalUserId1, "config1"); } diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/converters/AnnotableConverterTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/converters/AnnotableConverterTest.java deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/converters/CatalogCohortToSolrCohortConverterTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/converters/CatalogCohortToSolrCohortConverterTest.java deleted file mode 100644 index 9b32462fd0f..00000000000 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/converters/CatalogCohortToSolrCohortConverterTest.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright 2015-2020 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.catalog.db.mongodb.converters; - -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.opencb.opencga.catalog.stats.solr.CohortSolrModel; -import org.opencb.opencga.catalog.stats.solr.converters.CatalogCohortToSolrCohortConverter; -import org.opencb.opencga.core.common.TimeUtils; -import org.opencb.opencga.core.models.cohort.Cohort; -import org.opencb.opencga.core.models.cohort.CohortInternal; -import org.opencb.opencga.core.models.cohort.CohortStatus; -import org.opencb.opencga.core.models.common.Enums; -import org.opencb.opencga.core.models.sample.Sample; -import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.testclassification.duration.ShortTests; - -import java.time.LocalDate; -import java.time.ZoneId; -import java.util.Arrays; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; - -/** - * Created by wasim on 13/08/18. - */ -@Category(ShortTests.class) -public class CatalogCohortToSolrCohortConverterTest { - - @Test - public void CohortToSolrTest() { - Study study = new Study().setFqn("user@project:study").setAttributes(new HashMap<>()) - .setVariableSets(Collections.singletonList(AnnotationHelper.createVariableSet())); - Cohort cohort = new Cohort("id", "", Enums.CohortType.CASE_SET, TimeUtils.getTime(), TimeUtils.getTime(), - "test", Arrays.asList(new Sample().setId("1"), new Sample().setId("2")), 2, null) - .setAttributes(new HashMap<>()); - cohort.setUid(200) - .setInternal(new CohortInternal(TimeUtils.getTime(), TimeUtils.getTime(), new CohortStatus("CALCULATING"))) - .setAnnotationSets(AnnotationHelper.createAnnotation()); - CohortSolrModel cohortSolrModel = new CatalogCohortToSolrCohortConverter(study).convertToStorageType(cohort); - - assertEquals(cohortSolrModel.getUid(), cohort.getUid()); - assertEquals(cohortSolrModel.getStatus(), cohort.getInternal().getStatus().getId()); - - Date date = TimeUtils.toDate(cohort.getCreationDate()); - LocalDate localDate = date.toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); - - assertEquals(localDate.getYear(), cohortSolrModel.getCreationYear()); - assertEquals(localDate.getMonth().toString(), cohortSolrModel.getCreationMonth()); - assertEquals(localDate.getDayOfMonth(), cohortSolrModel.getCreationDay()); - assertEquals(localDate.getDayOfMonth(), cohortSolrModel.getCreationDay()); - assertEquals(localDate.getDayOfWeek().toString(), cohortSolrModel.getCreationDayOfWeek()); - cohortSolrModel.setStatus(cohort.getInternal().getStatus().getId()); - - assertEquals(cohortSolrModel.getType(), cohort.getType().name()); - assertEquals(cohortSolrModel.getRelease(), cohort.getRelease()); - assertEquals(cohortSolrModel.getNumSamples(), cohort.getSamples().size()); - - assertEquals(Arrays.asList(1, 2, 3, 4, 11, 12, 13, 14, 21), cohortSolrModel.getAnnotations().get("annotations__im__vsId.a.ab2.ab2c1.ab2c1d1")); - assertEquals(Arrays.asList(true, false, false), cohortSolrModel.getAnnotations().get("annotations__bm__vsId.a.ab1.ab1c1")); - assertEquals("hello world", cohortSolrModel.getAnnotations().get("annotations__s__vsId.a.ab1.ab1c2")); - assertEquals(Arrays.asList("hello ab2c1d2 1", "hello ab2c1d2 2"), cohortSolrModel.getAnnotations().get("annotations__sm__vsId.a.ab2.ab2c1.ab2c1d2")); - assertEquals(Arrays.asList(Arrays.asList("hello"), Arrays.asList("hello2", "bye2"), Arrays.asList("byeee2", "hellooo2")), cohortSolrModel.getAnnotations().get("annotations__sm__vsId.a.ab3.ab3c1.ab3c1d1")); - assertEquals(Arrays.asList(2.0, 4.0, 24.0), cohortSolrModel.getAnnotations().get("annotations__dm__vsId.a.ab3.ab3c1.ab3c1d2")); - assertNull(cohortSolrModel.getAnnotations().get("nothing")); - assertEquals(cohortSolrModel.getAnnotations().keySet().size(), 6); - } -} diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/converters/CatalogFamilyToSolrFamilyConverterTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/converters/CatalogFamilyToSolrFamilyConverterTest.java deleted file mode 100644 index d5aec157ce7..00000000000 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/converters/CatalogFamilyToSolrFamilyConverterTest.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright 2015-2020 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.catalog.db.mongodb.converters; - -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.opencb.opencga.catalog.stats.solr.FamilySolrModel; -import org.opencb.opencga.catalog.stats.solr.converters.CatalogFamilyToSolrFamilyConverter; -import org.opencb.opencga.core.models.family.Family; -import org.opencb.opencga.core.models.family.FamilyInternal; -import org.opencb.opencga.core.models.individual.Individual; -import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.testclassification.duration.ShortTests; - -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; - -/** - * Created by wasim on 13/08/18. - */ -@Category(ShortTests.class) -public class CatalogFamilyToSolrFamilyConverterTest { - - @Test - public void FamilyToSolrTest() { - Study study = new Study().setFqn("user@project:study").setAttributes(new HashMap<>()) - .setVariableSets(Collections.singletonList(AnnotationHelper.createVariableSet())); - Family family = new Family("id", "family", null, null, - Arrays.asList(new Individual().setId("I1"), new Individual().setId("I2")), "test", 1000, AnnotationHelper.createAnnotation(), null); - family.setUid(100).setInternal(FamilyInternal.init()).setRelease(1).setVersion(2); - FamilySolrModel familySolrModel = new CatalogFamilyToSolrFamilyConverter(study).convertToStorageType(family); - - assertEquals(familySolrModel.getUid(), family.getUid()); - assertEquals(familySolrModel.getStatus(), family.getInternal().getStatus().getId()); - assertEquals(familySolrModel.getNumMembers(), family.getMembers().size()); - assertEquals(familySolrModel.getRelease(), family.getRelease()); - assertEquals(familySolrModel.getVersion(), family.getVersion()); - assertEquals(familySolrModel.getPhenotypes().size(), 0); - - assertEquals(Arrays.asList(1, 2, 3, 4, 11, 12, 13, 14, 21), familySolrModel.getAnnotations().get("annotations__im__vsId.a.ab2.ab2c1.ab2c1d1")); - assertEquals(Arrays.asList(true, false, false), familySolrModel.getAnnotations().get("annotations__bm__vsId.a.ab1.ab1c1")); - assertEquals("hello world", familySolrModel.getAnnotations().get("annotations__s__vsId.a.ab1.ab1c2")); - assertEquals(Arrays.asList("hello ab2c1d2 1", "hello ab2c1d2 2"), familySolrModel.getAnnotations().get("annotations__sm__vsId.a.ab2.ab2c1.ab2c1d2")); - assertEquals(Arrays.asList(Arrays.asList("hello"), Arrays.asList("hello2", "bye2"), Arrays.asList("byeee2", "hellooo2")), familySolrModel.getAnnotations().get("annotations__sm__vsId.a.ab3.ab3c1.ab3c1d1")); - assertEquals(Arrays.asList(2.0, 4.0, 24.0), familySolrModel.getAnnotations().get("annotations__dm__vsId.a.ab3.ab3c1.ab3c1d2")); - assertNull(familySolrModel.getAnnotations().get("nothing")); - assertEquals(familySolrModel.getAnnotations().keySet().size(), 6); - - } -} diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/converters/CatalogFileToSolrFileConverterTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/converters/CatalogFileToSolrFileConverterTest.java deleted file mode 100644 index a28d8fb5667..00000000000 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/converters/CatalogFileToSolrFileConverterTest.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright 2015-2020 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.catalog.db.mongodb.converters; - -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.opencb.biodata.models.clinical.interpretation.Software; -import org.opencb.opencga.catalog.stats.solr.FileSolrModel; -import org.opencb.opencga.catalog.stats.solr.converters.CatalogFileToSolrFileConverter; -import org.opencb.opencga.core.common.TimeUtils; -import org.opencb.opencga.core.models.file.File; -import org.opencb.opencga.core.models.file.FileInternal; -import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.testclassification.duration.ShortTests; - -import java.time.LocalDate; -import java.time.ZoneId; -import java.util.Arrays; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; - -/** - * Created by wasim on 13/08/18. - */ -@Category(ShortTests.class) -public class CatalogFileToSolrFileConverterTest { - - @Test - public void FileToSolrTest() { - Study study = new Study().setFqn("user@project:study").setAttributes(new HashMap<>()) - .setVariableSets(Collections.singletonList(AnnotationHelper.createVariableSet())); - File file = new File("name", File.Type.FILE, File.Format.BAM, File.Bioformat.MICROARRAY_EXPRESSION_ONECHANNEL_AGILENT, - "test/base", null, "convertorTest", FileInternal.init(), 1111L, 2); - file.setUid(111) - .setSampleIds(Arrays.asList("1", "2")) - .setSoftware(new Software().setName("Software")); - file.setAnnotationSets(AnnotationHelper.createAnnotation()); - - FileSolrModel fileSolrModel = new CatalogFileToSolrFileConverter(study).convertToStorageType(file); - - assert (fileSolrModel.getUid() == file.getUid()); - assert (fileSolrModel.getName().equals(file.getName())); - assert (fileSolrModel.getType().equals(file.getType().name())); - assert (fileSolrModel.getFormat().equals(file.getFormat().name())); - assert (fileSolrModel.getBioformat().equals(file.getBioformat().name())); - assert (fileSolrModel.getRelease() == file.getRelease()); - - Date date = TimeUtils.toDate(file.getCreationDate()); - LocalDate localDate = date.toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); - - assertEquals(localDate.getYear(), fileSolrModel.getCreationYear()); - assertEquals(localDate.getMonth().toString(), fileSolrModel.getCreationMonth()); - assertEquals(localDate.getDayOfMonth(), fileSolrModel.getCreationDay()); - assertEquals(localDate.getDayOfMonth(), fileSolrModel.getCreationDay()); - assertEquals(localDate.getDayOfWeek().toString(), fileSolrModel.getCreationDayOfWeek()); - - assert (fileSolrModel.getStatus().equals(file.getInternal().getStatus().getId())); - assert (fileSolrModel.isExternal() == file.isExternal()); - assert (fileSolrModel.getSize() == file.getSize()); - assert (fileSolrModel.getSoftwareName().equals(file.getSoftware().getName())); - assert (fileSolrModel.getNumSamples() == file.getSampleIds().size()); - - assertEquals(Arrays.asList(1, 2, 3, 4, 11, 12, 13, 14, 21), fileSolrModel.getAnnotations().get("annotations__im__vsId.a.ab2.ab2c1.ab2c1d1")); - assertEquals(Arrays.asList(true, false, false), fileSolrModel.getAnnotations().get("annotations__bm__vsId.a.ab1.ab1c1")); - assertEquals("hello world", fileSolrModel.getAnnotations().get("annotations__s__vsId.a.ab1.ab1c2")); - assertEquals(Arrays.asList("hello ab2c1d2 1", "hello ab2c1d2 2"), fileSolrModel.getAnnotations().get("annotations__sm__vsId.a.ab2.ab2c1.ab2c1d2")); - assertEquals(Arrays.asList(Arrays.asList("hello"), Arrays.asList("hello2", "bye2"), Arrays.asList("byeee2", "hellooo2")), fileSolrModel.getAnnotations().get("annotations__sm__vsId.a.ab3.ab3c1.ab3c1d1")); - assertEquals(Arrays.asList(2.0, 4.0, 24.0), fileSolrModel.getAnnotations().get("annotations__dm__vsId.a.ab3.ab3c1.ab3c1d2")); - assertNull(fileSolrModel.getAnnotations().get("nothing")); - assertEquals(fileSolrModel.getAnnotations().keySet().size(), 6); - - } - -} diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/converters/CatalogIndividualToSolrIndividualConverterTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/converters/CatalogIndividualToSolrIndividualConverterTest.java deleted file mode 100644 index cf97bfddd61..00000000000 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/converters/CatalogIndividualToSolrIndividualConverterTest.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright 2015-2020 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.catalog.db.mongodb.converters; - -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.opencb.biodata.models.core.OntologyTermAnnotation; -import org.opencb.biodata.models.core.SexOntologyTermAnnotation; -import org.opencb.biodata.models.pedigree.IndividualProperty; -import org.opencb.opencga.catalog.stats.solr.IndividualSolrModel; -import org.opencb.opencga.catalog.stats.solr.converters.CatalogIndividualToSolrIndividualConverter; -import org.opencb.opencga.core.common.TimeUtils; -import org.opencb.opencga.core.models.individual.Individual; -import org.opencb.opencga.core.models.individual.IndividualInternal; -import org.opencb.opencga.core.models.individual.IndividualPopulation; -import org.opencb.opencga.core.models.individual.Location; -import org.opencb.opencga.core.models.sample.Sample; -import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.testclassification.duration.ShortTests; - -import java.time.LocalDate; -import java.time.ZoneId; -import java.util.Arrays; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; - -/** - * Created by wasim on 13/08/18. - */ -@Category(ShortTests.class) -public class CatalogIndividualToSolrIndividualConverterTest { - - @Test - public void IndividualToSolrTest() { - Study study = new Study().setFqn("user@project:study").setAttributes(new HashMap<>()) - .setVariableSets(Collections.singletonList(AnnotationHelper.createVariableSet())); - Individual individual = new Individual("Id", "individual", new Individual(), new Individual(), new Location(), - SexOntologyTermAnnotation.initMale(), null, new OntologyTermAnnotation().setId("Spanish"), - new IndividualPopulation("valencian", "", ""), null, "", Collections.emptyList(), false, 2, - AnnotationHelper.createAnnotation(), Collections.emptyList(), Collections.emptyList(), IndividualInternal.init(), null); - - individual.setUid(300) - .setKaryotypicSex(IndividualProperty.KaryotypicSex.XX).setVersion(4).setInternal(IndividualInternal.init()) - .setLifeStatus(IndividualProperty.LifeStatus.ABORTED) - .setSamples(Arrays.asList(new Sample().setId("1"), new Sample().setId("2"))).setParentalConsanguinity(true); - - IndividualSolrModel individualSolrModel = new CatalogIndividualToSolrIndividualConverter(study).convertToStorageType(individual); - - assertEquals(individualSolrModel.getUid(), individual.getUid()); - assertEquals(individualSolrModel.getSex(), individual.getSex().getSex().name()); - assertEquals(individualSolrModel.getKaryotypicSex(), individual.getKaryotypicSex().name()); - assertEquals(individualSolrModel.getEthnicity(), individual.getEthnicity().getId()); - assertEquals(individualSolrModel.getPopulation(), individual.getPopulation().getName()); - assertEquals(individualSolrModel.getRelease(), individual.getRelease()); - - Date date = TimeUtils.toDate(individual.getCreationDate()); - LocalDate localDate = date.toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); - - assertEquals(localDate.getYear(), individualSolrModel.getCreationYear()); - assertEquals(localDate.getMonth().toString(), individualSolrModel.getCreationMonth()); - assertEquals(localDate.getDayOfMonth(), individualSolrModel.getCreationDay()); - assertEquals(localDate.getDayOfMonth(), individualSolrModel.getCreationDay()); - assertEquals(localDate.getDayOfWeek().toString(), individualSolrModel.getCreationDayOfWeek()); - - assertEquals(individualSolrModel.getVersion(), individual.getVersion()); - assertEquals(individualSolrModel.getStatus(), individual.getInternal().getStatus().getId()); - assertEquals(individualSolrModel.getLifeStatus(), individual.getLifeStatus().name()); - assertEquals(individualSolrModel.getPhenotypes().size(), 0); - assertEquals(individualSolrModel.isParentalConsanguinity(), individual.isParentalConsanguinity()); - - assertEquals(Arrays.asList(1, 2, 3, 4, 11, 12, 13, 14, 21), individualSolrModel.getAnnotations().get("annotations__im__vsId.a.ab2.ab2c1.ab2c1d1")); - assertEquals(Arrays.asList(true, false, false), individualSolrModel.getAnnotations().get("annotations__bm__vsId.a.ab1.ab1c1")); - assertEquals("hello world", individualSolrModel.getAnnotations().get("annotations__s__vsId.a.ab1.ab1c2")); - assertEquals(Arrays.asList("hello ab2c1d2 1", "hello ab2c1d2 2"), individualSolrModel.getAnnotations().get("annotations__sm__vsId.a.ab2.ab2c1.ab2c1d2")); - assertEquals(Arrays.asList(Arrays.asList("hello"), Arrays.asList("hello2", "bye2"), Arrays.asList("byeee2", "hellooo2")), individualSolrModel.getAnnotations().get("annotations__sm__vsId.a.ab3.ab3c1.ab3c1d1")); - assertEquals(Arrays.asList(2.0, 4.0, 24.0), individualSolrModel.getAnnotations().get("annotations__dm__vsId.a.ab3.ab3c1.ab3c1d2")); - assertNull(individualSolrModel.getAnnotations().get("nothing")); - assertEquals(individualSolrModel.getAnnotations().keySet().size(), 6); - - } -} diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/converters/CatalogSampleToSolrSampleConverterTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/converters/CatalogSampleToSolrSampleConverterTest.java deleted file mode 100644 index 3c236743757..00000000000 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/converters/CatalogSampleToSolrSampleConverterTest.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright 2015-2020 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.catalog.db.mongodb.converters; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.opencb.biodata.models.core.OntologyTermAnnotation; -import org.opencb.biodata.models.pedigree.IndividualProperty; -import org.opencb.opencga.catalog.stats.solr.SampleSolrModel; -import org.opencb.opencga.catalog.stats.solr.converters.CatalogSampleToSolrSampleConverter; -import org.opencb.opencga.catalog.stats.solr.converters.SolrConverterUtil; -import org.opencb.opencga.core.common.JacksonUtils; -import org.opencb.opencga.core.common.TimeUtils; -import org.opencb.opencga.core.models.AclEntry; -import org.opencb.opencga.core.models.common.InternalStatus; -import org.opencb.opencga.core.models.individual.Individual; -import org.opencb.opencga.core.models.individual.IndividualPopulation; -import org.opencb.opencga.core.models.sample.Sample; -import org.opencb.opencga.core.models.sample.SampleInternal; -import org.opencb.opencga.core.models.sample.SamplePermissions; -import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.models.study.StudyPermissions; -import org.opencb.opencga.core.testclassification.duration.ShortTests; - -import java.util.*; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; - -/** - * Created by wasim on 13/08/18. - */ -@Category(ShortTests.class) -public class CatalogSampleToSolrSampleConverterTest { - - @Test - public void SampleToSolrTest() throws JsonProcessingException { - ObjectMapper objectMapper = JacksonUtils.getDefaultObjectMapper(); - - Study study = new Study().setFqn("user@project:study").setAttributes(new HashMap<>()) - .setVariableSets(Collections.singletonList(AnnotationHelper.createVariableSet())); - List> studyAclEntry = Arrays.asList( - objectMapper.readValue(objectMapper.writeValueAsString(new AclEntry<>("user1", EnumSet.noneOf(StudyPermissions.Permissions.class))), Map.class), - objectMapper.readValue(objectMapper.writeValueAsString(new AclEntry<>("user2", EnumSet.allOf(StudyPermissions.Permissions.class))), Map.class), - objectMapper.readValue(objectMapper.writeValueAsString(new AclEntry<>("user3", EnumSet.allOf(StudyPermissions.Permissions.class))), Map.class), - objectMapper.readValue(objectMapper.writeValueAsString(new AclEntry<>("user4", EnumSet.noneOf(StudyPermissions.Permissions.class))), Map.class) - ); - Map attributes = new HashMap<>(); - attributes.put("OPENCGA_ACL", SolrConverterUtil.parseInternalOpenCGAAcls(studyAclEntry)); - study.setAttributes(attributes); - - Individual individual = new Individual(); - individual.setUuid("uuid").setEthnicity(new OntologyTermAnnotation().setId("spanish")) - .setKaryotypicSex(IndividualProperty.KaryotypicSex.XX) - .setPopulation(new IndividualPopulation("valencian", "", "")); - - Sample sample = new Sample(); - new InternalStatus("READY"); - sample.setUid(500).setRelease(3).setVersion(2).setInternal(SampleInternal.init()) - .setSomatic(true).setCreationDate(TimeUtils.getTime()) - .setAnnotationSets(AnnotationHelper.createAnnotation()); - - List sampleAclEntry = Arrays.asList( - objectMapper.readValue(objectMapper.writeValueAsString(new AclEntry<>("user1", EnumSet.of(SamplePermissions.VIEW, SamplePermissions.WRITE))), Map.class), - objectMapper.readValue(objectMapper.writeValueAsString(new AclEntry<>("user2", EnumSet.noneOf(SamplePermissions.class))), Map.class) - ); - attributes = new HashMap<>(); - attributes.put("OPENCGA_ACL", sampleAclEntry); - attributes.put("individual", individual); - sample.setAttributes(attributes); - - SampleSolrModel sampleSolrModel = new CatalogSampleToSolrSampleConverter(study).convertToStorageType(sample); - - assertEquals(sampleSolrModel.getUid(), sample.getUid()); - assertEquals(sampleSolrModel.getRelease(), sample.getRelease()); - assertEquals(sampleSolrModel.getVersion(), sample.getVersion()); - assertEquals(sampleSolrModel.getStatus(), sample.getInternal().getStatus().getId()); - assertEquals(sampleSolrModel.isSomatic(), sample.isSomatic()); - assertEquals(sampleSolrModel.getPhenotypes().size(), 0); - - assertEquals(Arrays.asList(1, 2, 3, 4, 11, 12, 13, 14, 21), sampleSolrModel.getAnnotations().get("annotations__im__vsId.a.ab2.ab2c1.ab2c1d1")); - assertEquals(Arrays.asList(true, false, false), sampleSolrModel.getAnnotations().get("annotations__bm__vsId.a.ab1.ab1c1")); - assertEquals("hello world", sampleSolrModel.getAnnotations().get("annotations__s__vsId.a.ab1.ab1c2")); - assertEquals(Arrays.asList("hello ab2c1d2 1", "hello ab2c1d2 2"), sampleSolrModel.getAnnotations().get("annotations__sm__vsId.a.ab2.ab2c1.ab2c1d2")); - assertEquals(Arrays.asList(Arrays.asList("hello"), Arrays.asList("hello2", "bye2"), Arrays.asList("byeee2", "hellooo2")), sampleSolrModel.getAnnotations().get("annotations__sm__vsId.a.ab3.ab3c1.ab3c1d1")); - assertEquals(Arrays.asList(2.0, 4.0, 24.0), sampleSolrModel.getAnnotations().get("annotations__dm__vsId.a.ab3.ab3c1.ab3c1d2")); - assertNull(sampleSolrModel.getAnnotations().get("nothing")); - assertEquals(sampleSolrModel.getAnnotations().keySet().size(), 6); - - } -} diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/io/CatalogIOManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/io/CatalogIOManagerTest.java index d5d400a047d..f39be2bacb7 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/io/CatalogIOManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/io/CatalogIOManagerTest.java @@ -25,12 +25,12 @@ import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.testclassification.duration.ShortTests; -import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; @Category(ShortTests.class) public class CatalogIOManagerTest { @@ -57,32 +57,18 @@ public void before() throws Exception { ioManager.createDefaultOpenCGAFolders(); } - @Test - public void testCreateAccount() throws Exception { - String userId = "imedina"; - URI userUri = ioManager.createUser(userId); - - Path userPath = Paths.get(userUri); - assertTrue(Files.exists(userPath)); - assertEquals(tmpOutdir.resolve("users/" + userId).toAbsolutePath().toString(), userUri.getPath()); - - ioManager.deleteUser(userId); - assertFalse(Files.exists(userPath)); - } - @Test public void testCreateStudy() throws Exception { - String userId = "imedina"; + String organizationId = "opencb"; + ioManager.createOrganization(organizationId); String projectId = "1000g"; - Path userPath = Paths.get(ioManager.createUser(userId)); - - Path projectPath = Paths.get(ioManager.createProject(userId, projectId)); + Path projectPath = Paths.get(ioManager.createProject(organizationId, projectId)); assertTrue(Files.exists(projectPath)); - assertEquals(userPath.toString() + "/projects/" + projectId, projectPath.toString()); + assertEquals(tmpOutdir.resolve("orgs").resolve(organizationId).resolve("projects").resolve(projectId).toAbsolutePath().toString(), projectPath.toString()); - Path studyPath = Paths.get(ioManager.createStudy(userId, projectId, "phase1")); + Path studyPath = Paths.get(ioManager.createStudy(organizationId, projectId, "phase1")); assertTrue(Files.exists(studyPath)); - assertEquals(projectPath.toString() + "/phase1", studyPath.toString()); + assertEquals(projectPath.resolve("phase1").toString(), studyPath.toString()); } } \ No newline at end of file diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractClinicalManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractClinicalManagerTest.java index 9a10ff857f1..2de27beb3da 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractClinicalManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractClinicalManagerTest.java @@ -33,9 +33,10 @@ import org.opencb.opencga.core.models.family.Family; import org.opencb.opencga.core.models.file.File; import org.opencb.opencga.core.models.individual.Individual; +import org.opencb.opencga.core.models.organizations.OrganizationCreateParams; +import org.opencb.opencga.core.models.organizations.OrganizationUpdateParams; import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.models.user.Account; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.MediumTests; @@ -73,6 +74,7 @@ public class AbstractClinicalManagerTest extends GenericTest { public CatalogManagerExternalResource catalogManagerResource = new CatalogManagerExternalResource(); public CatalogManager catalogManager; + protected final String organizationId = "test"; public String token; public String studyFqn; public Family family; @@ -89,9 +91,15 @@ public void setUp() throws IOException, CatalogException, URISyntaxException { public void setUpCatalogManager() throws IOException, CatalogException, URISyntaxException { ClinicalAnalysis auxClinicalAnalysis; - catalogManager.getUserManager().create("user", "User Name", "mail@ebi.ac.uk", PASSWORD, "", null, Account.AccountType.FULL, catalogManagerResource.getAdminToken()); + // Create new organization, owner and admins + catalogManager.getOrganizationManager().create(new OrganizationCreateParams().setId(organizationId).setName("Test"), QueryOptions.empty(), catalogManagerResource.getAdminToken()); + catalogManager.getUserManager().create("user", "User Name", "mail@ebi.ac.uk", PASSWORD, organizationId, null, catalogManagerResource.getAdminToken()); - token = catalogManager.getUserManager().login("user", PASSWORD).getToken(); + catalogManager.getOrganizationManager().update(organizationId, + new OrganizationUpdateParams().setOwner("user"), + null, catalogManagerResource.getAdminToken()); + + token = catalogManager.getUserManager().login(organizationId, "user", PASSWORD).getToken(); catalogManager.getProjectManager().create("1000G", "Project about some genomes", "", "Homo sapiens", null, "GRCh38", new QueryOptions(), token); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractManagerTest.java index fbce0a4576f..dc857e15965 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractManagerTest.java @@ -18,35 +18,51 @@ import org.apache.commons.lang3.RandomStringUtils; import org.junit.Before; +import org.junit.ClassRule; import org.junit.Rule; import org.junit.experimental.categories.Category; import org.junit.rules.ExpectedException; +import org.junit.rules.TestName; +import org.mockito.Mockito; import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.test.GenericTest; import org.opencb.opencga.TestParamConstants; +import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager; +import org.opencb.opencga.catalog.db.api.UserDBAdaptor; +import org.opencb.opencga.catalog.db.mongodb.MongoBackupUtils; +import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; +import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.utils.FqnUtils; +import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.models.cohort.Cohort; import org.opencb.opencga.core.models.common.AnnotationSet; -import org.opencb.opencga.core.models.file.File; -import org.opencb.opencga.core.models.file.FileCreateParams; -import org.opencb.opencga.core.models.file.FileUpdateParams; +import org.opencb.opencga.core.models.file.*; +import org.opencb.opencga.core.models.individual.Individual; +import org.opencb.opencga.core.models.individual.IndividualAclParams; +import org.opencb.opencga.core.models.individual.IndividualPermissions; +import org.opencb.opencga.core.models.organizations.OrganizationCreateParams; +import org.opencb.opencga.core.models.organizations.OrganizationUpdateParams; import org.opencb.opencga.core.models.sample.Sample; -import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.models.study.Variable; -import org.opencb.opencga.core.models.study.VariableSet; -import org.opencb.opencga.core.models.user.Account; +import org.opencb.opencga.core.models.sample.SampleAclParams; +import org.opencb.opencga.core.models.sample.SamplePermissions; +import org.opencb.opencga.core.models.study.*; +import org.opencb.opencga.core.models.user.User; +import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.MediumTests; -import java.io.DataOutputStream; -import java.io.FileOutputStream; -import java.io.IOException; +import java.nio.file.Path; import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.apache.commons.lang3.StringUtils.join; +import static org.opencb.opencga.catalog.utils.ParamUtils.AclAction.ADD; +import static org.opencb.opencga.catalog.utils.ParamUtils.AclAction.SET; @Category(MediumTests.class) public class AbstractManagerTest extends GenericTest { @@ -55,32 +71,110 @@ public class AbstractManagerTest extends GenericTest { public ExpectedException thrown = ExpectedException.none(); @Rule - public CatalogManagerExternalResource catalogManagerResource = new CatalogManagerExternalResource(); + public TestName testName = new TestName(); + + @ClassRule + public static CatalogManagerExternalResource catalogManagerResource = new CatalogManagerExternalResource(); + public static Path mongoDumpFolder; protected CatalogManager catalogManager; + protected final static String organizationId = "test"; protected String opencgaToken; - protected String token; - protected String sessionIdUser2; - protected String sessionIdUser3; - protected File testFolder; - protected String project1; - protected String project2; + protected final static String orgOwnerUserId = "owner"; + protected String ownerToken; + protected final static String orgAdminUserId1 = "orgAdminUser1"; + protected String orgAdminToken1; + protected final static String orgAdminUserId2 = "orgAdminUser2"; + protected String orgAdminToken2; + protected final static String studyAdminUserId1 = "studyAdminUser1"; + protected String studyAdminToken1; + protected final static String normalUserId1 = "normalUser1"; + protected String normalToken1; + protected final static String normalUserId2 = "normalUser2"; + protected String normalToken2; + protected final static String normalUserId3 = "normalUser3"; + protected String normalToken3; + protected final static String noAccessUserId1 = "noAccessUserId1"; + protected String noAccessToken1; + + protected String restrictedGroup = "@restrictedGroup"; // normalUserId2, normalUserId3 + + protected final String project1 = "1000G"; + protected final String projectFqn1 = organizationId + "@1000G"; + protected final String project2 = "pmp"; + protected final String projectFqn2 = organizationId + "@pmp"; + protected final String project3 = "p1"; + protected final String projectFqn3 = organizationId + "@p1"; + + protected final String studyId = "phase1"; protected long studyUid; protected String studyFqn; + protected final String studyId2 = "phase3"; protected long studyUid2; protected String studyFqn2; + protected final String studyId3 = "s1"; protected String studyFqn3; - protected String s_1; - protected String s_2; - protected String s_3; - protected String s_4; - protected String s_5; - protected String s_6; - protected String s_7; - protected String s_8; - protected String s_9; - protected String testFile1; - protected String testFile2; + protected final String s_1Id = "s_1"; // shared with @restrictedGroup + protected long s_1Uid; + protected final String s_2Id = "s_2"; // shared with * + protected long s_2Uid; + protected final String s_3Id = "s_3"; // shared with @restrictedGroup + protected long s_3Uid; + protected final String s_4Id = "s_4"; + protected long s_4Uid; + protected final String s_5Id = "s_5"; // shared with @restrictedGroup + protected long s_5Uid; + protected final String s_6Id = "s_6"; // shared with @members + protected long s_6Uid; + protected final String s_7Id = "s_7"; + protected long s_7Uid; + protected final String s_8Id = "s_8"; + protected long s_8Uid; + protected final String s_9Id = "s_9"; + protected long s_9Uid; + + protected final String ind1 = "ind1"; + protected final String ind2 = "ind2"; + + protected final String testFile1 = "data/test/folder/test_1K.txt.gz";; + protected final String testFile2 = "data/test/folder/test_0.5K.txt";; + + protected String data = "data/"; // + protected String data_d1 = "data/d1/"; // Shared with normalUser1 + protected String data_d1_d2 = "data/d1/d2/"; // + protected String data_d1_d2_d3 = "data/d1/d2/d3/"; // Forbidden for normalUser1 + protected String data_d1_d2_d3_d4 = "data/d1/d2/d3/d4/"; // + protected String data_d1_d2_d3_d4_txt = "data/d1/d2/d3/d4/my.txt"; // Shared for normalUser1 + + protected final String ALL_FILE_PERMISSIONS = join( + EnumSet.allOf(FilePermissions.class) + .stream() + .map(FilePermissions::name) + .collect(Collectors.toList()), + ","); + protected final String DENY_FILE_PERMISSIONS = ""; + + protected final String ALL_SAMPLE_PERMISSIONS = join( + EnumSet.allOf(SamplePermissions.class) + .stream() + .map(SamplePermissions::name) + .collect(Collectors.toList()), + ","); + protected final String DENY_SAMPLE_PERMISSIONS = ""; + + protected final String ALL_INDIVIDUAL_PERMISSIONS = join( + EnumSet.allOf(IndividualPermissions.class) + .stream() + .map(IndividualPermissions::name) + .collect(Collectors.toList()), + ","); + + protected final SampleAclParams allSamplePermissions = new SampleAclParams(null, null, null, null, ALL_SAMPLE_PERMISSIONS); + protected final SampleAclParams noSamplePermissions = new SampleAclParams(null, null, null, null, DENY_SAMPLE_PERMISSIONS); + + protected final IndividualAclParams allIndividualPermissions = new IndividualAclParams(null, ALL_INDIVIDUAL_PERMISSIONS); + + private static boolean firstExecutionFinished = false; protected static final QueryOptions INCLUDE_RESULT = new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true); @@ -90,80 +184,152 @@ public void setUp() throws Exception { setUpCatalogManager(catalogManager); } - public void setUpCatalogManager(CatalogManager catalogManager) throws IOException, CatalogException { - opencgaToken = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); + public void setUpCatalogManager(CatalogManager catalogManager) throws Exception { + if (!firstExecutionFinished) { + createDummyData(catalogManager); + MongoBackupUtils.dump(catalogManager, catalogManagerResource.getOpencgaHome()); + firstExecutionFinished = true; + } else { + // Clear sessions folder + catalogManagerResource.clearOpenCGAHome(testName.getMethodName()); + MongoBackupUtils.restore(catalogManager, catalogManagerResource.getOpencgaHome()); + initVariables(); + } + } - catalogManager.getUserManager().create("user", "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, "", null, Account.AccountType.FULL, opencgaToken); - catalogManager.getUserManager().create("user2", "User2 Name", "mail2@ebi.ac.uk", TestParamConstants.PASSWORD, "", null, Account.AccountType.FULL, opencgaToken); - catalogManager.getUserManager().create("user3", "User3 Name", "user.2@e.mail", TestParamConstants.PASSWORD, "ACME", null, Account.AccountType.FULL, opencgaToken); + private void initVariables() throws CatalogException { + catalogManager = catalogManagerResource.resetCatalogManager(); + opencgaToken = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); + ownerToken = catalogManager.getUserManager().login(organizationId, orgOwnerUserId, TestParamConstants.PASSWORD).getToken(); + orgAdminToken1 = catalogManager.getUserManager().login(organizationId, orgAdminUserId1, TestParamConstants.PASSWORD).getToken(); + orgAdminToken2 = catalogManager.getUserManager().login(organizationId, orgAdminUserId2, TestParamConstants.PASSWORD).getToken(); + studyAdminToken1 = catalogManager.getUserManager().login(organizationId, studyAdminUserId1, TestParamConstants.PASSWORD).getToken(); + normalToken1 = catalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD).getToken(); + normalToken2 = catalogManager.getUserManager().login(organizationId, normalUserId2, TestParamConstants.PASSWORD).getToken(); + normalToken3 = catalogManager.getUserManager().login(organizationId, normalUserId3, TestParamConstants.PASSWORD).getToken(); + noAccessToken1 = catalogManager.getUserManager().login(organizationId, noAccessUserId1, TestParamConstants.PASSWORD).getToken(); + + Study study = catalogManager.getStudyManager().get(studyId, StudyManager.INCLUDE_STUDY_IDS, ownerToken).first(); + studyUid = study.getUid(); + studyFqn = study.getFqn(); - token = catalogManager.getUserManager().login("user", TestParamConstants.PASSWORD).getToken(); - sessionIdUser2 = catalogManager.getUserManager().login("user2", TestParamConstants.PASSWORD).getToken(); - sessionIdUser3 = catalogManager.getUserManager().login("user3", TestParamConstants.PASSWORD).getToken(); + study = catalogManager.getStudyManager().get(studyId2, StudyManager.INCLUDE_STUDY_IDS, ownerToken).first(); + studyUid2 = study.getUid(); + studyFqn2 = study.getFqn(); - project1 = catalogManager.getProjectManager().create("1000G", "Project about some genomes", "", "Homo sapiens", - null, "GRCh38", INCLUDE_RESULT, token).first().getId(); - project2 = catalogManager.getProjectManager().create("pmp", "Project Management Project", "life art intelligent system", - "Homo sapiens", null, "GRCh38", INCLUDE_RESULT, sessionIdUser2).first().getId(); - catalogManager.getProjectManager().create("p1", "project 1", "", "Homo sapiens", null, "GRCh38", INCLUDE_RESULT, - sessionIdUser3).first(); + studyFqn3 = FqnUtils.buildFqn(organizationId, project2, studyId3); + + OpenCGAResult result = catalogManager.getSampleManager().get(studyFqn, + Arrays.asList(s_1Id, s_2Id, s_3Id, s_4Id, s_5Id, s_6Id, s_7Id, s_8Id, s_9Id), SampleManager.INCLUDE_SAMPLE_IDS, ownerToken); + s_1Uid = result.getResults().get(0).getUid(); + s_2Uid = result.getResults().get(1).getUid(); + s_3Uid = result.getResults().get(2).getUid(); + s_4Uid = result.getResults().get(3).getUid(); + s_5Uid = result.getResults().get(4).getUid(); + s_6Uid = result.getResults().get(5).getUid(); + s_7Uid = result.getResults().get(6).getUid(); + s_8Uid = result.getResults().get(7).getUid(); + s_9Uid = result.getResults().get(8).getUid(); + } - Study study = catalogManager.getStudyManager().create(project1, "phase1", null, "Phase 1", "Done", null, null, null, null, INCLUDE_RESULT, token).first(); + private void createDummyData(CatalogManager catalogManager) throws CatalogException { + // Create new organization, owner and admins + opencgaToken = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); + catalogManager.getOrganizationManager().create(new OrganizationCreateParams().setId(organizationId).setName("Test"), QueryOptions.empty(), opencgaToken); + catalogManager.getUserManager().create(new User().setId(orgOwnerUserId).setName(orgOwnerUserId).setOrganization(organizationId), TestParamConstants.PASSWORD, opencgaToken); + catalogManager.getUserManager().create(orgAdminUserId1, "User2 Name", "mail2@ebi.ac.uk", TestParamConstants.PASSWORD, organizationId, null, opencgaToken); + catalogManager.getUserManager().create(orgAdminUserId2, "User3 Name", "user.3@e.mail", TestParamConstants.PASSWORD, organizationId, null, opencgaToken); + catalogManager.getUserManager().create(studyAdminUserId1, "User3 Name", "user.3@e.mail", TestParamConstants.PASSWORD, organizationId, null, opencgaToken); + catalogManager.getUserManager().create(normalUserId1, "User4 Name", "user.4@e.mail", TestParamConstants.PASSWORD, organizationId, null, opencgaToken); + catalogManager.getUserManager().create(normalUserId2, "User5 Name", "user.5@e.mail", TestParamConstants.PASSWORD, organizationId, null, opencgaToken); + catalogManager.getUserManager().create(normalUserId3, "User6 Name", "user.6@e.mail", TestParamConstants.PASSWORD, organizationId, null, opencgaToken); + catalogManager.getUserManager().create(noAccessUserId1, "Name", "user.6@e.mail", TestParamConstants.PASSWORD, organizationId, null, opencgaToken); + + catalogManager.getOrganizationManager().update(organizationId, + new OrganizationUpdateParams() + .setOwner(orgOwnerUserId) + .setAdmins(Arrays.asList(orgAdminUserId1, orgAdminUserId2)), + null, opencgaToken); + + ownerToken = catalogManager.getUserManager().login(organizationId, orgOwnerUserId, TestParamConstants.PASSWORD).getToken(); + orgAdminToken1 = catalogManager.getUserManager().login(organizationId, orgAdminUserId1, TestParamConstants.PASSWORD).getToken(); + orgAdminToken2 = catalogManager.getUserManager().login(organizationId, orgAdminUserId2, TestParamConstants.PASSWORD).getToken(); + studyAdminToken1 = catalogManager.getUserManager().login(organizationId, studyAdminUserId1, TestParamConstants.PASSWORD).getToken(); + normalToken1 = catalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD).getToken(); + normalToken2 = catalogManager.getUserManager().login(organizationId, normalUserId2, TestParamConstants.PASSWORD).getToken(); + normalToken3 = catalogManager.getUserManager().login(organizationId, normalUserId3, TestParamConstants.PASSWORD).getToken(); + noAccessToken1 = catalogManager.getUserManager().login(organizationId, noAccessUserId1, TestParamConstants.PASSWORD).getToken(); + + catalogManager.getProjectManager().create(project1, "Project about some genomes", "", "Homo sapiens", + null, "GRCh38", INCLUDE_RESULT, ownerToken); + catalogManager.getProjectManager().create(project2, "Project Management Project", "life art intelligent system", + "Homo sapiens", null, "GRCh38", INCLUDE_RESULT, orgAdminToken1); + catalogManager.getProjectManager().create(project3, "project 1", "", "Homo sapiens", null, "GRCh38", INCLUDE_RESULT, + orgAdminToken2); + + Study study = catalogManager.getStudyManager().create(project1, studyId, null, "Phase 1", "Done", null, null, null, null, + INCLUDE_RESULT, ownerToken).first(); studyUid = study.getUid(); studyFqn = study.getFqn(); + catalogManager.getStudyManager().updateGroup(studyFqn, ParamConstants.ADMINS_GROUP, ParamUtils.BasicUpdateAction.ADD, + new GroupUpdateParams(Collections.singletonList(studyAdminUserId1)), ownerToken); - study = catalogManager.getStudyManager().create(project1, "phase3", null, "Phase 3", "d", null, null, null, null, INCLUDE_RESULT, token).first(); + study = catalogManager.getStudyManager().create(project1, studyId2, null, "Phase 3", "d", null, null, null, null, INCLUDE_RESULT, + ownerToken).first(); studyUid2 = study.getUid(); studyFqn2 = study.getFqn(); + catalogManager.getStudyManager().updateGroup(studyFqn2, ParamConstants.ADMINS_GROUP, ParamUtils.BasicUpdateAction.ADD, + new GroupUpdateParams(Collections.singletonList(studyAdminUserId1)), ownerToken); - study = catalogManager.getStudyManager().create(project2, "s1", null, "Study 1", "", null, null, null, null, INCLUDE_RESULT, sessionIdUser2).first(); + study = catalogManager.getStudyManager().create(project2, studyId3, null, "Study 1", "", null, null, null, null, INCLUDE_RESULT, + orgAdminToken1).first(); studyFqn3 = study.getFqn(); + catalogManager.getStudyManager().updateGroup(studyFqn3, ParamConstants.ADMINS_GROUP, ParamUtils.BasicUpdateAction.ADD, + new GroupUpdateParams(Collections.singletonList(studyAdminUserId1)), ownerToken); catalogManager.getFileManager().createFolder(studyFqn2, Paths.get("data/test/folder/").toString(), true, - null, QueryOptions.empty(), token); + null, QueryOptions.empty(), ownerToken); - testFolder = catalogManager.getFileManager().createFolder(studyFqn, Paths.get("data/test/folder/").toString(), - true, null, INCLUDE_RESULT, token).first(); + File testFolder = catalogManager.getFileManager().createFolder(studyFqn, Paths.get("data/test/folder/").toString(), + true, null, INCLUDE_RESULT, ownerToken).first(); ObjectMap attributes = new ObjectMap(); attributes.put("field", "value"); attributes.put("numValue", 5); catalogManager.getFileManager().update(studyFqn, testFolder.getPath(), - new FileUpdateParams().setAttributes(attributes), new QueryOptions(), token); + new FileUpdateParams().setAttributes(attributes), new QueryOptions(), ownerToken); - testFile1 = testFolder.getPath() + "test_1K.txt.gz"; DataResult queryResult2 = catalogManager.getFileManager().create(studyFqn, new FileCreateParams() .setContent(RandomStringUtils.randomAlphanumeric(1000)) .setPath(testFile1) .setType(File.Type.FILE), - false, token); + false, ownerToken); - File fileTest1k = catalogManager.getFileManager().get(studyFqn, queryResult2.first().getPath(), INCLUDE_RESULT, token).first(); + File fileTest1k = catalogManager.getFileManager().get(studyFqn, queryResult2.first().getPath(), INCLUDE_RESULT, ownerToken).first(); attributes = new ObjectMap(); attributes.put("field", "value"); attributes.put("name", "fileTest1k"); attributes.put("numValue", "10"); attributes.put("boolean", false); catalogManager.getFileManager().update(studyFqn, fileTest1k.getPath(), - new FileUpdateParams().setAttributes(attributes), new QueryOptions(), token); + new FileUpdateParams().setAttributes(attributes), new QueryOptions(), ownerToken); - testFile2 = testFolder.getPath() + "test_0.5K.txt"; DataResult queryResult1 = catalogManager.getFileManager().create(studyFqn, new FileCreateParams() .setContent(RandomStringUtils.randomAlphanumeric(500)) .setPath(testFile2) .setBioformat(File.Bioformat.DATAMATRIX_EXPRESSION) .setType(File.Type.FILE), - false, token); + false, ownerToken); - File fileTest05k = catalogManager.getFileManager().get(studyFqn, queryResult1.first().getPath(), INCLUDE_RESULT, token).first(); + File fileTest05k = catalogManager.getFileManager().get(studyFqn, queryResult1.first().getPath(), INCLUDE_RESULT, ownerToken).first(); attributes = new ObjectMap(); attributes.put("field", "valuable"); attributes.put("name", "fileTest05k"); attributes.put("numValue", 5); attributes.put("boolean", true); catalogManager.getFileManager().update(studyFqn, fileTest05k.getPath(), - new FileUpdateParams().setAttributes(attributes), new QueryOptions(), token); + new FileUpdateParams().setAttributes(attributes), new QueryOptions(), ownerToken); DataResult queryResult = catalogManager.getFileManager().create(studyFqn, new FileCreateParams() @@ -171,134 +337,162 @@ public void setUpCatalogManager(CatalogManager catalogManager) throws IOExceptio .setPath(testFolder.getPath() + "test_0.1K.png") .setFormat(File.Format.IMAGE) .setType(File.Type.FILE), - false, token); + false, ownerToken); - File test01k = catalogManager.getFileManager().get(studyFqn, queryResult.first().getPath(), INCLUDE_RESULT, token).first(); + File test01k = catalogManager.getFileManager().get(studyFqn, queryResult.first().getPath(), INCLUDE_RESULT, ownerToken).first(); attributes = new ObjectMap(); attributes.put("field", "other"); attributes.put("name", "test01k"); attributes.put("numValue", 50); attributes.put("nested", new ObjectMap("num1", 45).append("num2", 33).append("text", "HelloWorld")); catalogManager.getFileManager().update(studyFqn, test01k.getPath(), - new FileUpdateParams().setAttributes(attributes), new QueryOptions(), token); + new FileUpdateParams().setAttributes(attributes), new QueryOptions(), ownerToken); List variables = new ArrayList<>(); variables.addAll(Arrays.asList( - new Variable("NAME", "", "", Variable.VariableType.STRING, "", true, false, Collections.emptyList(), null, 0, "", "", null, - Collections.emptyMap()), + new Variable("NAME", "", "", Variable.VariableType.STRING, "", true, false, Collections.emptyList(), null, 0, "", "", null, + Collections.emptyMap()), new Variable("AGE", "", "", Variable.VariableType.INTEGER, null, true, false, Collections.singletonList("0:130"), null, 1, "", "", - null, Collections.emptyMap()), + null, Collections.emptyMap()), new Variable("HEIGHT", "", "", Variable.VariableType.DOUBLE, "1.5", false, false, Collections.singletonList("0:"), null, 2, "", - "", null, Collections.emptyMap()), - new Variable("ALIVE", "", "", Variable.VariableType.BOOLEAN, "", true, false, Collections.emptyList(), null, 3, "", "", - null, Collections.emptyMap()), + "", null, Collections.emptyMap()), + new Variable("ALIVE", "", "", Variable.VariableType.BOOLEAN, "", true, false, Collections.emptyList(), null, 3, "", "", + null, Collections.emptyMap()), new Variable("PHEN", "", "", Variable.VariableType.CATEGORICAL, "CASE", true, false, Arrays.asList("CASE", "CONTROL"), null, 4, - "", "", null, Collections.emptyMap()), + "", "", null, Collections.emptyMap()), new Variable("EXTRA", "", "", Variable.VariableType.STRING, "", false, false, Collections.emptyList(), null, 5, "", "", null, - Collections.emptyMap()) + Collections.emptyMap()) )); VariableSet vs = catalogManager.getStudyManager().createVariableSet(studyFqn, "vs", "vs", true, false, "", null, variables, - null, token).first(); + null, ownerToken).first(); - Sample sample = new Sample().setId("s_1"); + Sample sample = new Sample().setId(s_1Id); sample.setAnnotationSets(Collections.singletonList(new AnnotationSet("annot1", vs.getId(), - new ObjectMap("NAME", "s_1").append("AGE", 6).append("ALIVE", true).append("PHEN", "CONTROL")))); - s_1 = catalogManager.getSampleManager().create(studyFqn, sample, INCLUDE_RESULT, token).first().getId(); + new ObjectMap("NAME", s_1Id).append("AGE", 6).append("ALIVE", true).append("PHEN", "CONTROL")))); + Sample tmpSample = catalogManager.getSampleManager().create(studyFqn, sample, INCLUDE_RESULT, ownerToken).first(); + s_1Uid = tmpSample.getUid(); - sample.setId("s_2"); + sample.setId(s_2Id); sample.setAnnotationSets(Collections.singletonList(new AnnotationSet("annot1", vs.getId(), - new ObjectMap("NAME", "s_2").append("AGE", 10).append("ALIVE", false).append("PHEN", "CASE")))); - s_2 = catalogManager.getSampleManager().create(studyFqn, sample, INCLUDE_RESULT, token).first().getId(); + new ObjectMap("NAME", s_2Id).append("AGE", 10).append("ALIVE", false).append("PHEN", "CASE")))); + tmpSample = catalogManager.getSampleManager().create(studyFqn, sample, INCLUDE_RESULT, ownerToken).first(); + s_2Uid = tmpSample.getUid(); - sample.setId("s_3"); + sample.setId(s_3Id); sample.setAnnotationSets(Collections.singletonList(new AnnotationSet("annot1", vs.getId(), - new ObjectMap("NAME", "s_3").append("AGE", 15).append("ALIVE", true).append("PHEN", "CONTROL")))); - s_3 = catalogManager.getSampleManager().create(studyFqn, sample, INCLUDE_RESULT, token).first().getId(); + new ObjectMap("NAME", s_3Id).append("AGE", 15).append("ALIVE", true).append("PHEN", "CONTROL")))); + tmpSample = catalogManager.getSampleManager().create(studyFqn, sample, INCLUDE_RESULT, ownerToken).first(); + s_3Uid = tmpSample.getUid(); - sample.setId("s_4"); + sample.setId(s_4Id); sample.setAnnotationSets(Collections.singletonList(new AnnotationSet("annot1", vs.getId(), - new ObjectMap("NAME", "s_4").append("AGE", 22).append("ALIVE", false).append("PHEN", "CONTROL")))); - s_4 = catalogManager.getSampleManager().create(studyFqn, sample, INCLUDE_RESULT, token).first().getId(); + new ObjectMap("NAME", s_4Id).append("AGE", 22).append("ALIVE", false).append("PHEN", "CONTROL")))); + tmpSample = catalogManager.getSampleManager().create(studyFqn, sample, INCLUDE_RESULT, ownerToken).first(); + s_4Uid = tmpSample.getUid(); - sample.setId("s_5"); + sample.setId(s_5Id); sample.setAnnotationSets(Collections.singletonList(new AnnotationSet("annot1", vs.getId(), - new ObjectMap("NAME", "s_5").append("AGE", 29).append("ALIVE", true).append("PHEN", "CASE")))); - s_5 = catalogManager.getSampleManager().create(studyFqn, sample, INCLUDE_RESULT, token).first().getId(); + new ObjectMap("NAME", s_5Id).append("AGE", 29).append("ALIVE", true).append("PHEN", "CASE")))); + tmpSample = catalogManager.getSampleManager().create(studyFqn, sample, INCLUDE_RESULT, ownerToken).first(); + s_5Uid = tmpSample.getUid(); - sample.setId("s_6"); + sample.setId(s_6Id); sample.setAnnotationSets(Collections.singletonList(new AnnotationSet("annot2", vs.getId(), - new ObjectMap("NAME", "s_6").append("AGE", 38).append("ALIVE", true).append("PHEN", "CONTROL")))); - s_6 = catalogManager.getSampleManager().create(studyFqn, sample, INCLUDE_RESULT, token).first().getId(); + new ObjectMap("NAME", s_6Id).append("AGE", 38).append("ALIVE", true).append("PHEN", "CONTROL")))); + tmpSample = catalogManager.getSampleManager().create(studyFqn, sample, INCLUDE_RESULT, ownerToken).first(); + s_6Uid = tmpSample.getUid(); - sample.setId("s_7"); + sample.setId(s_7Id); sample.setAnnotationSets(Collections.singletonList(new AnnotationSet("annot2", vs.getId(), - new ObjectMap("NAME", "s_7").append("AGE", 46).append("ALIVE", false).append("PHEN", "CASE")))); - s_7 = catalogManager.getSampleManager().create(studyFqn, sample, INCLUDE_RESULT, token).first().getId(); + new ObjectMap("NAME", s_7Id).append("AGE", 46).append("ALIVE", false).append("PHEN", "CASE")))); + tmpSample = catalogManager.getSampleManager().create(studyFqn, sample, INCLUDE_RESULT, ownerToken).first(); + s_7Uid = tmpSample.getUid(); - sample.setId("s_8"); + sample.setId(s_8Id); sample.setAnnotationSets(Collections.singletonList(new AnnotationSet("annot2", vs.getId(), - new ObjectMap("NAME", "s_8").append("AGE", 72).append("ALIVE", true).append("PHEN", "CONTROL")))); - s_8 = catalogManager.getSampleManager().create(studyFqn, sample, INCLUDE_RESULT, token).first().getId(); + new ObjectMap("NAME", s_8Id).append("AGE", 72).append("ALIVE", true).append("PHEN", "CONTROL")))); + tmpSample = catalogManager.getSampleManager().create(studyFqn, sample, INCLUDE_RESULT, ownerToken).first(); + s_8Uid = tmpSample.getUid(); - sample.setId("s_9"); + sample.setId(s_9Id); sample.setAnnotationSets(Collections.emptyList()); - s_9 = catalogManager.getSampleManager().create(studyFqn, sample, INCLUDE_RESULT, token).first().getId(); + tmpSample = catalogManager.getSampleManager().create(studyFqn, sample, INCLUDE_RESULT, ownerToken).first(); + s_9Uid = tmpSample.getUid(); catalogManager.getFileManager().update(studyFqn, test01k.getPath(), new FileUpdateParams() - .setSampleIds(Arrays.asList(s_1, s_2, s_3, s_4, s_5)), INCLUDE_RESULT, token); - } - - - /* TYPE_FILE UTILS */ - public static java.io.File createDebugFile() throws IOException { - String fileTestName = "/tmp/fileTest_" + RandomStringUtils.randomAlphanumeric(5); - return createDebugFile(fileTestName); - } + .setSampleIds(Arrays.asList(s_1Id, s_2Id, s_3Id, s_4Id, s_5Id)), INCLUDE_RESULT, ownerToken); - public static java.io.File createDebugFile(String fileTestName) throws IOException { - return createDebugFile(fileTestName, 200); - } + // ================ PERMISSIONS ================= - public static java.io.File createDebugFile(String fileTestName, int lines) throws IOException { - DataOutputStream os = new DataOutputStream(new FileOutputStream(fileTestName)); + catalogManager.getFileManager().createFolder(studyFqn, data_d1, true, null, QueryOptions.empty(), ownerToken); + catalogManager.getFileManager().createFolder(studyFqn, data_d1_d2, false, null, QueryOptions.empty(), ownerToken); + catalogManager.getFileManager().createFolder(studyFqn, data_d1_d2_d3, false, null, QueryOptions.empty(), ownerToken); + catalogManager.getFileManager().createFolder(studyFqn, data_d1_d2_d3_d4, false, null, QueryOptions.empty(), ownerToken); - os.writeBytes("Debug file name: " + fileTestName + "\n"); - for (int i = 0; i < 100; i++) { - os.writeBytes(i + ", "); - } - for (int i = 0; i < lines; i++) { - os.writeBytes(RandomStringUtils.randomAlphanumeric(500)); - os.write('\n'); - } - os.close(); + catalogManager.getFileManager().create(studyFqn, + new FileCreateParams() + .setContent("file content") + .setPath(data_d1_d2_d3_d4_txt) + .setType(File.Type.FILE), + false, ownerToken); + + StudyAclParams aclParams = new StudyAclParams("", AuthorizationManager.ROLE_ANALYST); + catalogManager.getStudyManager().updateAcl(studyFqn, normalUserId1, aclParams, ADD, orgAdminToken1); + catalogManager.getStudyManager().createGroup(studyFqn, new Group(restrictedGroup, Arrays.asList(normalUserId2, normalUserId3)), + orgAdminToken2); + + catalogManager.getFileManager().updateAcl(studyFqn, Collections.singletonList(data_d1), restrictedGroup, + new FileAclParams(null, ALL_FILE_PERMISSIONS), SET, ownerToken); + catalogManager.getFileManager().updateAcl(studyFqn, Collections.singletonList(data_d1_d2_d3), restrictedGroup, + new FileAclParams(null, DENY_FILE_PERMISSIONS), SET, ownerToken); + catalogManager.getFileManager().updateAcl(studyFqn, Collections.singletonList(data_d1_d2_d3_d4_txt), restrictedGroup, + new FileAclParams(null, ALL_FILE_PERMISSIONS), SET, ownerToken); + + catalogManager.getCohortManager().create(studyFqn, new Cohort() + .setId("all") + .setSamples(Stream.of(s_1Id, s_2Id, s_3Id).map(s -> new Sample().setId(s)).collect(Collectors.toList())), + QueryOptions.empty(), ownerToken); + + catalogManager.getIndividualManager().create(studyFqn, new Individual().setId(ind1), Collections.singletonList(s_1Id), + QueryOptions.empty(), ownerToken); + catalogManager.getIndividualManager().create(studyFqn, new Individual().setId(ind2), Collections.singletonList(s_2Id), + QueryOptions.empty(), ownerToken); + + catalogManager.getIndividualManager().updateAcl(studyFqn, Collections.singletonList(ind1), restrictedGroup, + allIndividualPermissions, SET, true, ownerToken); + + catalogManager.getSampleManager().updateAcl(studyFqn, Collections.singletonList(s_1Id), restrictedGroup, allSamplePermissions, + SET, ownerToken); + catalogManager.getSampleManager().updateAcl(studyFqn, Collections.singletonList(s_2Id), ParamConstants.ANONYMOUS_USER_ID, + noSamplePermissions, SET, ownerToken); + catalogManager.getSampleManager().updateAcl(studyFqn, Collections.singletonList(s_3Id), restrictedGroup, noSamplePermissions, + SET, ownerToken); + catalogManager.getSampleManager().updateAcl(studyFqn, Collections.singletonList(s_5Id), restrictedGroup, noSamplePermissions, + SET, ownerToken); + catalogManager.getSampleManager().updateAcl(studyFqn, Collections.singletonList(s_6Id), ParamConstants.MEMBERS_GROUP, + allSamplePermissions, SET, ownerToken); + + // ================ END PERMISSIONS ================= - return Paths.get(fileTestName).toFile(); } - public static String createRandomString(int lines) { - StringBuilder stringBuilder = new StringBuilder(lines); - for (int i = 0; i < 100; i++) { - stringBuilder.append(i + ", "); - } - for (int i = 0; i < lines; i++) { - stringBuilder.append(RandomStringUtils.randomAlphanumeric(500)); - stringBuilder.append("\n"); - } - return stringBuilder.toString(); + protected CatalogManager mockCatalogManager() throws CatalogDBException { + CatalogManager spy = Mockito.spy(catalogManager); + UserManager userManager = spy.getUserManager(); + UserManager userManagerSpy = Mockito.spy(userManager); + Mockito.doReturn(userManagerSpy).when(spy).getUserManager(); + MongoDBAdaptorFactory mongoDBAdaptorFactory = mockMongoDBAdaptorFactory(); + Mockito.doReturn(mongoDBAdaptorFactory).when(userManagerSpy).getCatalogDBAdaptorFactory(); + return spy; } - public static String getDummyVCFContent() { - return "##fileformat=VCFv4.0\n" + - "##fileDate=20090805\n" + - "##source=myImputationProgramV3.1\n" + - "##reference=1000GenomesPilot-NCBI36\n" + - "##phasing=partial\n" + - "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tNA00001\tNA00002\tNA00003\n" + - "20\t14370\trs6054257\tG\tA\t29\tPASS\tNS=3;DP=14;AF=0.5;DB;H2\tGT:GQ:DP:HQ\t0|0:48:1:51,51\t1|0:48:8:51,51\t1/1:43:5:.,.\n" + - "20\t17330\t.\tT\tA\t3\tq10\tNS=3;DP=11;AF=0.017\tGT:GQ:DP:HQ\t0|0:49:3:58,50\t0|1:3:5:65,3\t0/0:41:3\n" + - "20\t1110696\trs6040355\tA\tG,T\t67\tPASS\tNS=2;DP=10;AF=0.333,0.667;AA=T;DB\tGT:GQ:DP:HQ\t1|2:21:6:23,27\t2|1:2:0:18,2\t2/2:35:4\n" + - "20\t1230237\t.\tT\t.\t47\tPASS\tNS=3;DP=13;AA=T\tGT:GQ:DP:HQ\t0|0:54:7:56,60\t0|0:48:4:51,51\t0/0:61:2\n" + - "20\t1234567\tmicrosat1\tGTCT\tG,GTACT\t50\tPASS\tNS=3;DP=9;AA=G\tGT:GQ:DP\t0/1:35:4\t0/2:17:2\t1/1:40:3"; + protected MongoDBAdaptorFactory mockMongoDBAdaptorFactory() throws CatalogDBException { + MongoDBAdaptorFactory catalogDBAdaptorFactory = (MongoDBAdaptorFactory) catalogManager.getUserManager().getCatalogDBAdaptorFactory(); + MongoDBAdaptorFactory dbAdaptorFactorySpy = Mockito.spy(catalogDBAdaptorFactory); + UserDBAdaptor userDBAdaptor = dbAdaptorFactorySpy.getCatalogUserDBAdaptor(organizationId); + UserDBAdaptor userDBAdaptorSpy = Mockito.spy(userDBAdaptor); + Mockito.doReturn(userDBAdaptorSpy).when(dbAdaptorFactorySpy).getCatalogUserDBAdaptor(organizationId); + return dbAdaptorFactorySpy; } } diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerExternalResource.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerExternalResource.java index e717f074fe0..09c94ee33d3 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerExternalResource.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerExternalResource.java @@ -16,13 +16,10 @@ package org.opencb.opencga.catalog.managers; -import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.config.Configurator; -import org.bson.Document; import org.junit.Assert; import org.junit.rules.ExternalResource; -import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.TestParamConstants; import org.opencb.opencga.catalog.auth.authentication.JwtManager; import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; @@ -43,8 +40,6 @@ import java.nio.file.Paths; import java.nio.file.StandardCopyOption; -import static org.opencb.opencga.core.common.JacksonUtils.getDefaultObjectMapper; - /** * Created on 05/05/16 * @@ -52,11 +47,11 @@ */ public class CatalogManagerExternalResource extends ExternalResource { - private static CatalogManager catalogManager; + private CatalogManager catalogManager; private Configuration configuration; private Path opencgaHome; private String adminToken; - + public boolean initialized = false; public CatalogManagerExternalResource() { Configurator.setLevel("org.mongodb.driver.cluster", Level.WARN); @@ -65,9 +60,30 @@ public CatalogManagerExternalResource() { @Override public void before() throws Exception { + initialized = true; + System.out.println("-------------------------------------------------------------------------------"); + System.out.println("Initializing CatalogManagerExternalResource"); + System.out.println("-------------------------------------------------------------------------------"); + if (catalogManager != null) { + catalogManager.close(); + catalogManager = null; + } + clearOpenCGAHome("static"); + + //Catalog database might be already installed. Need to delete it before installing it again. + clearCatalog(configuration); + + catalogManager = new CatalogManager(configuration); + String secretKey = PasswordUtils.getStrongRandomPassword(JwtManager.SECRET_KEY_MIN_LENGTH); + catalogManager.installCatalogDB("HS256", secretKey, TestParamConstants.ADMIN_PASSWORD, "opencga@admin.com", true); + + adminToken = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); + } + + public Path clearOpenCGAHome(String testName) throws IOException { int c = 0; do { - opencgaHome = Paths.get("target/test-data").resolve("junit_opencga_home_" + TimeUtils.getTimeMillis() + (c > 0 ? "_" + c : "")); + opencgaHome = Paths.get("target/test-data").resolve("junit_opencga_home_" + testName + "_" + TimeUtils.getTimeMillis() + (c > 0 ? "_" + c : "")); c++; } while (opencgaHome.toFile().exists()); Files.createDirectories(opencgaHome); @@ -79,21 +95,7 @@ public void before() throws Exception { Path analysisPath = Files.createDirectories(opencgaHome.resolve("analysis/pedigree-graph")).toAbsolutePath(); FileInputStream inputStream = new FileInputStream("../opencga-app/app/analysis/pedigree-graph/ped.R"); Files.copy(inputStream, analysisPath.resolve("ped.R"), StandardCopyOption.REPLACE_EXISTING); - - clearCatalog(configuration); - if (!opencgaHome.toFile().exists()) { - deleteFolderTree(opencgaHome.toFile()); - Files.createDirectory(opencgaHome); - } - configuration.getAdmin().setSecretKey(PasswordUtils.getStrongRandomPassword(JwtManager.SECRET_KEY_MIN_LENGTH)); - catalogManager = new CatalogManager(configuration); - catalogManager.installCatalogDB(configuration.getAdmin().getSecretKey(), TestParamConstants.ADMIN_PASSWORD, "opencga@admin.com", "", - true, true); - catalogManager.close(); - // FIXME!! Should not need to create again the catalogManager - // Have to create again the CatalogManager, as it has a random "secretKey" inside - catalogManager = new CatalogManager(configuration); - adminToken = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); + return opencgaHome; } @Override @@ -103,10 +105,12 @@ public void after() { if (catalogManager != null) { catalogManager.close(); } - adminToken = null; } catch (CatalogException e) { throw new RuntimeException(e); } + System.out.println("-------------------------------------------------------------------------------"); + System.out.println("Shutting down CatalogManagerExternalResource"); + System.out.println("-------------------------------------------------------------------------------"); } public Configuration getConfiguration() { @@ -117,6 +121,13 @@ public CatalogManager getCatalogManager() { return catalogManager; } + public CatalogManager resetCatalogManager() throws CatalogException { + catalogManager.close(); + catalogManager = new CatalogManager(configuration); + adminToken = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); + return catalogManager; + } + public String getAdminToken() { return adminToken; } @@ -125,42 +136,11 @@ public Path getOpencgaHome() { return opencgaHome; } - public ObjectMapper generateNewObjectMapper() { - ObjectMapper jsonObjectMapper = getDefaultObjectMapper(); -// jsonObjectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); -// jsonObjectMapper.configure(MapperFeature.REQUIRE_SETTERS_FOR_GETTERS, true); - return jsonObjectMapper; - } - public static void clearCatalog(Configuration configuration) throws CatalogException, URISyntaxException { try (MongoDBAdaptorFactory dbAdaptorFactory = new MongoDBAdaptorFactory(configuration, new IOManagerFactory())) { - for (String collection : MongoDBAdaptorFactory.COLLECTIONS_LIST) { - dbAdaptorFactory.getMongoDataStore().getCollection(collection).remove(new Document(), QueryOptions.empty()); - } + dbAdaptorFactory.deleteCatalogDB(); } -// List dataStoreServerAddresses = new LinkedList<>(); -// for (String hostPort : configuration.getCatalog().getDatabase().getHosts()) { -// if (hostPort.contains(":")) { -// String[] split = hostPort.split(":"); -// Integer port = Integer.valueOf(split[1]); -// dataStoreServerAddresses.add(new DataStoreServerAddress(split[0], port)); -// } else { -// dataStoreServerAddresses.add(new DataStoreServerAddress(hostPort, 27017)); -// } -// } -// MongoDataStoreManager mongoManager = new MongoDataStoreManager(dataStoreServerAddresses); -// -//// if (catalogManager == null) { -//// catalogManager = new CatalogManager(configuration); -//// } -// -//// MongoDataStore db = mongoManager.get(catalogConfiguration.getDatabase().getDatabase()); -// MongoDataStore db = mongoManager.get(configuration.getDatabasePrefix() + "_catalog"); -// db.getDb().drop(); -//// mongoManager.close(catalogConfiguration.getDatabase().getDatabase()); -// mongoManager.close(configuration.getDatabasePrefix() + "_catalog"); - Path rootdir = Paths.get(UriUtils.createDirectoryUri(configuration.getWorkspace())); deleteFolderTree(rootdir.toFile()); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerTest.java index 890c74709de..8f0bce1401e 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerTest.java @@ -17,24 +17,30 @@ package org.opencb.opencga.catalog.managers; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import com.mongodb.BasicDBObject; import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.StopWatch; -import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; import org.opencb.biodata.models.common.Status; import org.opencb.biodata.models.pedigree.IndividualProperty; -import org.opencb.commons.datastore.core.*; +import org.opencb.commons.datastore.core.DataResult; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.TestParamConstants; import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager; import org.opencb.opencga.catalog.db.api.*; -import org.opencb.opencga.catalog.exceptions.*; +import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; +import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.config.Optimizations; +import org.opencb.opencga.core.models.Acl; import org.opencb.opencga.core.models.AclEntry; import org.opencb.opencga.core.models.AclEntryList; import org.opencb.opencga.core.models.cohort.Cohort; @@ -45,18 +51,19 @@ import org.opencb.opencga.core.models.individual.Individual; import org.opencb.opencga.core.models.individual.IndividualUpdateParams; import org.opencb.opencga.core.models.job.*; +import org.opencb.opencga.core.models.organizations.OrganizationConfiguration; +import org.opencb.opencga.core.models.organizations.OrganizationCreateParams; +import org.opencb.opencga.core.models.organizations.OrganizationUpdateParams; +import org.opencb.opencga.core.models.project.DataStore; import org.opencb.opencga.core.models.project.Project; import org.opencb.opencga.core.models.project.ProjectCreateParams; import org.opencb.opencga.core.models.project.ProjectOrganism; import org.opencb.opencga.core.models.sample.*; import org.opencb.opencga.core.models.study.*; -import org.opencb.opencga.core.models.user.Account; -import org.opencb.opencga.core.models.user.AuthenticationResponse; import org.opencb.opencga.core.models.user.User; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.MediumTests; -import javax.naming.NamingException; import java.io.IOException; import java.util.*; import java.util.concurrent.ExecutorService; @@ -64,7 +71,6 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import static org.hamcrest.CoreMatchers.allOf; import static org.hamcrest.CoreMatchers.containsString; import static org.junit.Assert.*; @@ -73,232 +79,24 @@ public class CatalogManagerTest extends AbstractManagerTest { @Test public void createStudyFailMoreThanOneProject() throws CatalogException { - catalogManager.getProjectManager().incrementRelease(project1, token); + catalogManager.getProjectManager().incrementRelease(project1, ownerToken); catalogManager.getProjectManager().create("1000G2", "Project about some genomes", "", "Homo sapiens", - null, "GRCh38", new QueryOptions(), token); + null, "GRCh38", new QueryOptions(), ownerToken); // Create a new study without providing the project. It should raise an error because the user owns more than one project thrown.expect(CatalogException.class); - thrown.expectMessage("More than one project found"); + thrown.expectMessage("Missing"); catalogManager.getStudyManager().create(null, "phasexx", null, "Phase 1", "Done", null, - null, null, null, null, token); - } - - @Test - public void testAdminUserExists() throws Exception { - String token = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); - assertEquals("opencga", catalogManager.getUserManager().getUserId(token)); - } - - @Test - public void testGetToken() throws Exception { - String token = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); - Map claims = new HashMap<>(); - claims.put("a", "hola"); - claims.put("ab", "byw"); - // Create a token valid for 1 second - String expiringToken = catalogManager.getUserManager().getToken("opencga", claims, 1L, token); - assertEquals("opencga", catalogManager.getUserManager().getUserId(expiringToken)); - - String nonExpiringToken = catalogManager.getUserManager().getNonExpiringToken("opencga", claims, token); - assertEquals("opencga", catalogManager.getUserManager().getUserId(nonExpiringToken)); - - Thread.sleep(1000); - thrown.expect(CatalogAuthenticationException.class); - thrown.expectMessage("expired"); - assertEquals("opencga", catalogManager.getUserManager().getUserId(expiringToken)); - } - @Test - public void testCreateExistingUser() throws Exception { - thrown.expect(CatalogException.class); - thrown.expectMessage(containsString("already exists")); - catalogManager.getUserManager().create("user", "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, "", null, Account.AccountType.FULL, opencgaToken); - } - - @Test - public void testCreateAnonymousUser() throws Exception { - thrown.expect(CatalogParameterException.class); - thrown.expectMessage(containsString("reserved")); - catalogManager.getUserManager().create(ParamConstants.ANONYMOUS_USER_ID, "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, "", null, - Account.AccountType.FULL, opencgaToken); - } - - @Test - public void testCreateRegisteredUser() throws Exception { - thrown.expect(CatalogParameterException.class); - thrown.expectMessage(containsString("reserved")); - catalogManager.getUserManager().create(ParamConstants.REGISTERED_USERS, "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, "", null, - Account.AccountType.FULL, opencgaToken); - } - - @Test - public void testLogin() throws Exception { - catalogManager.getUserManager().login("user", TestParamConstants.PASSWORD); - - thrown.expect(CatalogAuthenticationException.class); - thrown.expectMessage(allOf(containsString("Incorrect"), containsString("password"))); - catalogManager.getUserManager().login("user", "fakePassword"); - } - - @Test - public void testGetUserInfo() throws CatalogException { - DataResult user = catalogManager.getUserManager().get("user", new QueryOptions(), token); - System.out.println("user = " + user); - OpenCGAResult result = catalogManager.getUserManager().get("user2", new QueryOptions(), token); - assertEquals(Event.Type.ERROR, result.getEvents().get(0).getType()); - - catalogManager.getStudyManager().updateGroup(studyFqn, StudyManager.MEMBERS, ParamUtils.BasicUpdateAction.ADD, - new GroupUpdateParams(Collections.singletonList("user2")), token); - result = catalogManager.getUserManager().get("user2", new QueryOptions(), token); - assertTrue(result.getEvents().isEmpty()); - assertTrue(StringUtils.isNotEmpty(result.first().getEmail())); - } - - @Test - public void testUserInfoProjections() throws CatalogException { - QueryOptions options = new QueryOptions() - .append(QueryOptions.INCLUDE, UserDBAdaptor.QueryParams.PROJECTS_ID.key()); - DataResult user = catalogManager.getUserManager().get("user", options, token); - assertNotNull(user.first().getProjects()); - assertTrue(StringUtils.isNotEmpty(user.first().getProjects().get(0).getId())); - assertTrue(StringUtils.isEmpty(user.first().getProjects().get(0).getName())); - assertNull(user.first().getProjects().get(0).getStudies()); - - options = new QueryOptions() - .append(QueryOptions.INCLUDE, UserDBAdaptor.QueryParams.PROJECTS.key() + ".studies." - + StudyDBAdaptor.QueryParams.FQN.key()); - user = catalogManager.getUserManager().get("user", options, token); - assertNotNull(user.first().getProjects()); - assertEquals(2, user.first().getProjects().get(0).getStudies().size()); - assertTrue(StringUtils.isNotEmpty(user.first().getProjects().get(0).getStudies().get(0).getFqn())); - assertTrue(StringUtils.isEmpty(user.first().getProjects().get(0).getStudies().get(0).getName())); - - options = new QueryOptions() - .append(QueryOptions.INCLUDE, UserDBAdaptor.QueryParams.SHARED_PROJECTS.key() + ".studies." - + StudyDBAdaptor.QueryParams.FQN.key()); - user = catalogManager.getUserManager().get("user2", options, sessionIdUser2); - assertEquals(0, user.first().getSharedProjects().size()); - - // Grant permissions to user2 to access study of user1 - catalogManager.getStudyManager().updateGroup(studyFqn, StudyManager.MEMBERS, ParamUtils.BasicUpdateAction.ADD, - new GroupUpdateParams(Collections.singletonList("user2")), token); - - options = new QueryOptions() - .append(QueryOptions.INCLUDE, UserDBAdaptor.QueryParams.SHARED_PROJECTS.key() + ".studies." - + StudyDBAdaptor.QueryParams.FQN.key()); - user = catalogManager.getUserManager().get("user2", options, sessionIdUser2); - assertEquals(1, user.first().getSharedProjects().size()); - assertEquals(studyFqn, user.first().getSharedProjects().get(0).getStudies().get(0).getFqn()); - assertNull(user.first().getSharedProjects().get(0).getStudies().get(0).getId()); - - options = new QueryOptions() - .append(QueryOptions.INCLUDE, UserDBAdaptor.QueryParams.SHARED_PROJECTS.key() + ".studies." - + StudyDBAdaptor.QueryParams.ID.key()); - user = catalogManager.getUserManager().get("user2", options, sessionIdUser2); - assertEquals(1, user.first().getSharedProjects().size()); - assertEquals(studyFqn, user.first().getSharedProjects().get(0).getStudies().get(0).getFqn()); - assertNotNull(user.first().getSharedProjects().get(0).getStudies().get(0).getId()); - } - - @Test - public void testModifyUser() throws CatalogException, InterruptedException, IOException { - ObjectMap params = new ObjectMap(); - String newName = "Changed Name " + RandomStringUtils.randomAlphanumeric(10); - String newPassword = RandomStringUtils.randomAlphanumeric(10); - String newEmail = "new@email.ac.uk"; - - params.put("name", newName); - ObjectMap attributes = new ObjectMap("myBoolean", true); - attributes.put("value", 6); - attributes.put("object", new BasicDBObject("id", 1234)); - params.put("attributes", attributes); - - Thread.sleep(10); - - catalogManager.getUserManager().update("user", params, null, token); - catalogManager.getUserManager().update("user", new ObjectMap("email", newEmail), null, token); - catalogManager.getUserManager().changePassword("user", TestParamConstants.PASSWORD, newPassword); - - List userList = catalogManager.getUserManager().get("user", new QueryOptions(QueryOptions - .INCLUDE, Arrays.asList(UserDBAdaptor.QueryParams.NAME.key(), UserDBAdaptor.QueryParams.EMAIL.key(), - UserDBAdaptor.QueryParams.ATTRIBUTES.key())), token).getResults(); - User userPost = userList.get(0); - System.out.println("userPost = " + userPost); - assertEquals(userPost.getName(), newName); - assertEquals(userPost.getEmail(), newEmail); - - catalogManager.getUserManager().login("user", newPassword); - for (Map.Entry entry : attributes.entrySet()) { - assertEquals(userPost.getAttributes().get(entry.getKey()), entry.getValue()); - } - - catalogManager.getUserManager().changePassword("user", newPassword, TestParamConstants.PASSWORD); - catalogManager.getUserManager().login("user", TestParamConstants.PASSWORD); - - try { - params = new ObjectMap(); - params.put("password", "1234321"); - catalogManager.getUserManager().update("user", params, null, token); - fail("Expected exception"); - } catch (CatalogDBException e) { - System.out.println(e); - } - - try { - catalogManager.getUserManager().update("user", params, null, sessionIdUser2); - fail("Expected exception"); - } catch (CatalogException e) { - System.out.println(e); - } - } - - @Test - public void testUpdateUserConfig() throws CatalogException { - Map map = new HashMap<>(); - map.put("key1", "value1"); - map.put("key2", "value2"); - catalogManager.getUserManager().setConfig("user", "a", map, token); - - Map config = (Map) catalogManager.getUserManager().getConfig("user", "a", token).first(); - assertEquals(2, config.size()); - assertEquals("value1", config.get("key1")); - assertEquals("value2", config.get("key2")); - - map = new HashMap<>(); - map.put("key2", "value3"); - catalogManager.getUserManager().setConfig("user", "a", map, token); - config = (Map) catalogManager.getUserManager().getConfig("user", "a", token).first(); - assertEquals(1, config.size()); - assertEquals("value3", config.get("key2")); - - catalogManager.getUserManager().deleteConfig("user", "a", token); - - thrown.expect(CatalogException.class); - thrown.expectMessage("not found"); - catalogManager.getUserManager().getConfig("user", "a", token); - } - - private String getAdminToken() throws CatalogException, IOException { - return catalogManager.getUserManager().loginAsAdmin("admin").getToken(); - } - - @Test - public void createUserUsingMailAsId() throws CatalogException { - catalogManager.getUserManager().create(new User().setId("hello.mail@mymail.org").setName("Hello") - .setAccount(new Account().setType(Account.AccountType.GUEST)), TestParamConstants.PASSWORD, opencgaToken); - AuthenticationResponse login = catalogManager.getUserManager().login("hello.mail@mymail.org", TestParamConstants.PASSWORD); - assertNotNull(login); - User user = catalogManager.getUserManager().get("hello.mail@mymail.org", new QueryOptions(), login.getToken()).first(); - assertEquals("hello.mail@mymail.org", user.getId()); + null, null, null, null, ownerToken); } @Test public void getGroupsTest() throws CatalogException { - Group group = new Group("groupId", Arrays.asList("user2", "user3")).setSyncedFrom(new Group.Sync("ldap", "bio")); - catalogManager.getStudyManager().createGroup(studyFqn, group, token); + Group group = new Group("groupId", Arrays.asList(normalUserId2, normalUserId3)).setSyncedFrom(new Group.Sync("ldap", "bio")); + catalogManager.getStudyManager().createGroup(studyFqn, group, ownerToken); - OpenCGAResult customGroups = catalogManager.getStudyManager().getCustomGroups(studyFqn, null, token); - assertEquals(3, customGroups.getNumResults()); + OpenCGAResult customGroups = catalogManager.getStudyManager().getCustomGroups(studyFqn, null, ownerToken); + assertEquals(4, customGroups.getNumResults()); for (CustomGroup customGroup : customGroups.getResults()) { if (!customGroup.getUsers().isEmpty()) { @@ -306,129 +104,56 @@ public void getGroupsTest() throws CatalogException { } } - customGroups = catalogManager.getStudyManager().getCustomGroups(studyFqn, group.getId(), token); + customGroups = catalogManager.getStudyManager().getCustomGroups(studyFqn, group.getId(), ownerToken); assertEquals(1, customGroups.getNumResults()); assertEquals(group.getId(), customGroups.first().getId()); assertEquals(2, customGroups.first().getUsers().size()); assertTrue(StringUtils.isNotEmpty(customGroups.first().getUsers().get(0).getName())); - assertNull(customGroups.first().getUsers().get(0).getProjects()); - assertNull(customGroups.first().getUsers().get(0).getSharedProjects()); thrown.expect(CatalogAuthorizationException.class); - thrown.expectMessage("Only owners"); - catalogManager.getStudyManager().getCustomGroups(studyFqn, group.getId(), sessionIdUser2); - } - - @Ignore - @Test - public void importLdapUsers() throws CatalogException, NamingException, IOException { - // Action only for admins - catalogManager.getUserManager().importRemoteEntities("ldap", Arrays.asList("pfurio", "imedina"), false, null, null, - getAdminToken()); - // TODO: Validate the users have been imported - } - - // To make this test work we will need to add a correct user and password to be able to login - @Ignore - @Test - public void loginNotRegisteredUsers() throws CatalogException { - // Action only for admins - Group group = new Group("ldap", Collections.emptyList()).setSyncedFrom(new Group.Sync("ldap", "bio")); - catalogManager.getStudyManager().createGroup(studyFqn, group, token); - catalogManager.getStudyManager().updateAcl(Arrays.asList(studyFqn), "@ldap", new StudyAclParams("", "view_only"), - ParamUtils.AclAction.SET, token); - String token = catalogManager.getUserManager().login("user", "password").getToken(); - - assertEquals(9, catalogManager.getSampleManager().count(studyFqn, new Query(), token).getNumTotalResults()); - - // We remove the permissions for group ldap - catalogManager.getStudyManager().updateAcl(Arrays.asList(studyFqn), "@ldap", new StudyAclParams("", ""), - ParamUtils.AclAction.RESET, this.token); - - assertEquals(0, catalogManager.getSampleManager().count(studyFqn, new Query(), token).getNumTotalResults()); - } - - @Ignore - @Test - public void syncUsers() throws CatalogException { - // Action only for admins - String token = catalogManager.getUserManager().loginAsAdmin("admin").getToken(); - - catalogManager.getUserManager().importRemoteGroupOfUsers("ldap", "bio", "bio", studyFqn, true, token); - DataResult bio = catalogManager.getStudyManager().getGroup(studyFqn, "bio", this.token); - - assertEquals(1, bio.getNumResults()); - assertEquals(0, bio.first().getUserIds().size()); - - catalogManager.getUserManager().syncAllUsersOfExternalGroup(studyFqn, "ldap", token); - bio = catalogManager.getStudyManager().getGroup(studyFqn, "bio", this.token); - - assertEquals(1, bio.getNumResults()); - assertTrue(!bio.first().getUserIds().isEmpty()); - } - - @Ignore - @Test - public void importLdapGroups() throws CatalogException, IOException { - // Action only for admins - String remoteGroup = "bio"; - String internalGroup = "test"; - String study = "user@1000G:phase1"; - catalogManager.getUserManager().importRemoteGroupOfUsers("ldap", remoteGroup, internalGroup, study, true, getAdminToken()); - - DataResult test = catalogManager.getStudyManager().getGroup("user@1000G:phase1", "test", token); - assertEquals(1, test.getNumResults()); - assertEquals("@test", test.first().getId()); - assertTrue(test.first().getUserIds().size() > 0); - -// internalGroup = "test1"; -// try { -// catalogManager.getUserManager().importRemoteGroupOfUsers("ldap", remoteGroup, internalGroup, study, getAdminToken()); -// fail("Should not be possible creating another group containing the same users that belong to a different group"); -// } catch (CatalogException e) { -// System.out.println(e.getMessage()); -// } - - remoteGroup = "bioo"; - internalGroup = "test2"; - thrown.expect(CatalogException.class); - thrown.expectMessage("not found"); - catalogManager.getUserManager().importRemoteGroupOfUsers("ldap", remoteGroup, internalGroup, study, true, getAdminToken()); + thrown.expectMessage("study administrators"); + catalogManager.getStudyManager().getCustomGroups(studyFqn, group.getId(), normalToken2); } @Test public void createEmptyGroup() throws CatalogException { - catalogManager.getUserManager().create("test", "test", "test@mail.com", TestParamConstants.PASSWORD, null, 100L, Account.AccountType.GUEST, opencgaToken); - catalogManager.getStudyManager().createGroup("user@1000G:phase1", "group_cancer_some_thing_else", null, token); - catalogManager.getStudyManager().updateGroup("user@1000G:phase1", "group_cancer_some_thing_else", ParamUtils.BasicUpdateAction.ADD, - new GroupUpdateParams(Collections.singletonList("test")), token); + catalogManager.getUserManager().create("test", "test", "test@mail.com", TestParamConstants.PASSWORD, organizationId, 100L, opencgaToken); + catalogManager.getStudyManager().createGroup(studyFqn, "group_cancer_some_thing_else", null, ownerToken); + catalogManager.getStudyManager().updateGroup(studyFqn, "group_cancer_some_thing_else", ParamUtils.BasicUpdateAction.ADD, + new GroupUpdateParams(Collections.singletonList("test")), ownerToken); } @Test public void testAssignPermissions() throws CatalogException { - catalogManager.getUserManager().create("test", "test", "test@mail.com", TestParamConstants.PASSWORD, null, 100L, Account.AccountType.GUEST, opencgaToken); + catalogManager.getUserManager().create("test", "test", "test@mail.com", TestParamConstants.PASSWORD, organizationId, 100L, opencgaToken); - catalogManager.getStudyManager().createGroup("user@1000G:phase1", "group_cancer_some_thing_else", - Collections.singletonList("test"), token); + catalogManager.getStudyManager().createGroup(studyFqn, "group_cancer_some_thing_else", Collections.singletonList("test"), ownerToken); DataResult> permissions = catalogManager.getStudyManager().updateAcl( - Collections.singletonList("user@1000G:phase1"), "@group_cancer_some_thing_else", - new StudyAclParams("", "view_only"), ParamUtils.AclAction.SET, token); + studyFqn, "@group_cancer_some_thing_else", + new StudyAclParams("", "view_only"), ParamUtils.AclAction.SET, ownerToken); assertEquals("@group_cancer_some_thing_else", permissions.first().getAcl().get(0).getMember()); assertFalse(permissions.first().getAcl().get(0).getPermissions().isEmpty()); - String token = catalogManager.getUserManager().login("test", TestParamConstants.PASSWORD).getToken(); - DataResult studyDataResult = catalogManager.getStudyManager().get("user@1000G:phase1", QueryOptions.empty(), token); + String token = catalogManager.getUserManager().login(organizationId, "test", TestParamConstants.PASSWORD).getToken(); + DataResult studyDataResult = catalogManager.getStudyManager().get(studyFqn, QueryOptions.empty(), token); assertEquals(1, studyDataResult.getNumResults()); assertTrue(studyDataResult.first().getAttributes().isEmpty()); - studyDataResult = catalogManager.getStudyManager().get("user@1000G:phase1", new QueryOptions(DBAdaptor.INCLUDE_ACLS, true), token); + studyDataResult = catalogManager.getStudyManager().get(studyFqn, new QueryOptions(DBAdaptor.INCLUDE_ACLS, true), ownerToken); assertEquals(1, studyDataResult.getNumResults()); assertTrue(!studyDataResult.first().getAttributes().isEmpty()); assertTrue(studyDataResult.first().getAttributes().containsKey("OPENCGA_ACL")); List> acls = (List>) studyDataResult.first().getAttributes().get("OPENCGA_ACL"); - assertEquals(1, acls.size()); - assertEquals("@group_cancer_some_thing_else", acls.get(0).get("member")); - assertTrue(!((List) acls.get(0).get("permissions")).isEmpty()); + assertEquals(2, acls.size()); + assertTrue(acls.stream().map(x -> String.valueOf(x.get("member"))).collect(Collectors.toSet()).contains("@group_cancer_some_thing_else")); + + studyDataResult = catalogManager.getStudyManager().get(studyFqn, new QueryOptions(DBAdaptor.INCLUDE_ACLS, true), token); + assertEquals(1, studyDataResult.getNumResults()); + assertTrue(!studyDataResult.first().getAttributes().isEmpty()); + assertTrue(studyDataResult.first().getAttributes().containsKey("OPENCGA_ACL")); + acls = (List>) studyDataResult.first().getAttributes().get("OPENCGA_ACL"); + assertEquals(2, acls.size()); + assertTrue(acls.stream().map(x -> String.valueOf(x.get("member"))).collect(Collectors.toSet()).contains("@group_cancer_some_thing_else")); } /** @@ -437,12 +162,12 @@ public void testAssignPermissions() throws CatalogException { @Test public void testGetAllProjects() throws Exception { - Query query = new Query(ProjectDBAdaptor.QueryParams.USER_ID.key(), "user"); - DataResult projects = catalogManager.getProjectManager().search(query, null, token); - assertEquals(1, projects.getNumResults()); + Query query = new Query(); + DataResult projects = catalogManager.getProjectManager().search(organizationId, query, null, ownerToken); + assertEquals(3, projects.getNumResults()); - projects = catalogManager.getProjectManager().search(query, null, sessionIdUser2); - assertEquals(0, projects.getNumResults()); + projects = catalogManager.getProjectManager().search(organizationId, query, null, normalToken1); + assertEquals(1, projects.getNumResults()); } @Test @@ -451,20 +176,17 @@ public void testCreateProject() throws Exception { String projectAlias = "projectAlias_ASDFASDF"; catalogManager.getProjectManager().create(projectAlias, "Project", "", "Homo sapiens", null, "GRCh38", new - QueryOptions(), token); + QueryOptions(), ownerToken); thrown.expect(CatalogDBException.class); thrown.expectMessage(containsString("already exists")); catalogManager.getProjectManager().create(projectAlias, "Project", "", "Homo sapiens", - null, "GRCh38", new QueryOptions(), token); + null, "GRCh38", new QueryOptions(), ownerToken); } @Test public void testModifyProject() throws CatalogException { String newProjectName = "ProjectName " + RandomStringUtils.randomAlphanumeric(10); - String projectId = catalogManager.getUserManager().get("user", new QueryOptions(), token).first().getProjects().get(0) - .getId(); - ObjectMap options = new ObjectMap(); options.put("name", newProjectName); ObjectMap attributes = new ObjectMap("myBoolean", true); @@ -472,8 +194,8 @@ public void testModifyProject() throws CatalogException { attributes.put("object", new ObjectMap("id", 1234)); options.put("attributes", attributes); - catalogManager.getProjectManager().update(projectId, options, null, token); - DataResult result = catalogManager.getProjectManager().get(projectId, null, token); + catalogManager.getProjectManager().update(project1, options, null, ownerToken); + DataResult result = catalogManager.getProjectManager().get(project1, null, ownerToken); Project project = result.first(); System.out.println(result); @@ -485,17 +207,103 @@ public void testModifyProject() throws CatalogException { options = new ObjectMap(); options.put(ProjectDBAdaptor.QueryParams.CREATION_DATE.key(), "20180101120000"); - catalogManager.getProjectManager().update(projectId, options, null, token); - project = catalogManager.getProjectManager().get(projectId, null, token).first(); + catalogManager.getProjectManager().update(project1, options, null, ownerToken); + project = catalogManager.getProjectManager().get(project1, null, ownerToken).first(); assertEquals("20180101120000", project.getCreationDate()); options = new ObjectMap(); options.put(ProjectDBAdaptor.QueryParams.ID.key(), "newProjectId"); - catalogManager.getProjectManager().update(projectId, options, null, token); + catalogManager.getProjectManager().update(project1, options, null, ownerToken); thrown.expect(CatalogException.class); - thrown.expectMessage("not found"); - catalogManager.getProjectManager().update(projectId, options, null, token); + thrown.expectMessage("found"); + catalogManager.getProjectManager().update(project1, options, null, ownerToken); + } + + @Test + public void updateProjectPermissionTest() throws CatalogException { + ObjectMap params = new ObjectMap() + .append(ProjectDBAdaptor.QueryParams.DESCRIPTION.key(), "my new description"); + Project project = catalogManager.getProjectManager().update(project1, params, INCLUDE_RESULT, ownerToken).first(); + assertEquals("my new description", project.getDescription()); + + params.put(ProjectDBAdaptor.QueryParams.DESCRIPTION.key(), "my new description 2"); + project = catalogManager.getProjectManager().update(project1, params, INCLUDE_RESULT, orgAdminToken2).first(); + assertEquals("my new description 2", project.getDescription()); + + params.put(ProjectDBAdaptor.QueryParams.DESCRIPTION.key(), "my new description 3"); + project = catalogManager.getProjectManager().update(project1, params, INCLUDE_RESULT, studyAdminToken1).first(); + assertEquals("my new description 3", project.getDescription()); + + // Make normalUser1 admin of first study + catalogManager.getStudyManager().updateGroup(studyFqn, ParamConstants.ADMINS_GROUP, ParamUtils.BasicUpdateAction.ADD, + new GroupUpdateParams(Collections.singletonList(normalUserId1)), ownerToken); + // And remove normalUser1 from the admins group of the second study (just in case) + catalogManager.getStudyManager().updateGroup(studyFqn2, ParamConstants.ADMINS_GROUP, ParamUtils.BasicUpdateAction.REMOVE, + new GroupUpdateParams(Collections.singletonList(normalUserId1)), ownerToken); + + CatalogAuthorizationException catalogAuthorizationException = assertThrows(CatalogAuthorizationException.class, + () -> catalogManager.getProjectManager().update(project1, params, INCLUDE_RESULT, normalToken1)); + assertFalse(catalogAuthorizationException.getCause().getMessage().contains(studyFqn)); + assertTrue(catalogAuthorizationException.getCause().getMessage().contains(studyFqn2)); + + catalogAuthorizationException = assertThrows(CatalogAuthorizationException.class, + () -> catalogManager.getProjectManager().update(project1, params, INCLUDE_RESULT, normalToken2)); + assertTrue(catalogAuthorizationException.getCause().getMessage().contains(studyFqn)); + assertTrue(catalogAuthorizationException.getCause().getMessage().contains(studyFqn2)); + + // Remove orgAdminUser1 from the administrators group of the organization + Map actionMap = new HashMap<>(); + actionMap.put(OrganizationDBAdaptor.QueryParams.ADMINS.key(), ParamUtils.BasicUpdateAction.REMOVE); + QueryOptions options = new QueryOptions(Constants.ACTIONS, actionMap); + catalogManager.getOrganizationManager().update(organizationId, new OrganizationUpdateParams() + .setAdmins(Collections.singletonList(orgAdminUserId1)), options, ownerToken); + + catalogAuthorizationException = assertThrows(CatalogAuthorizationException.class, + () -> catalogManager.getProjectManager().update(project1, params, INCLUDE_RESULT, orgAdminToken1)); + assertTrue(catalogAuthorizationException.getCause().getMessage().contains(studyFqn)); + assertTrue(catalogAuthorizationException.getCause().getMessage().contains(studyFqn2)); + + // Create a third study + catalogManager.getStudyManager().create(project1, new Study().setId("study_3"), null, ownerToken); + catalogAuthorizationException = assertThrows(CatalogAuthorizationException.class, + () -> catalogManager.getProjectManager().update(project1, params, INCLUDE_RESULT, orgAdminToken1)); + assertTrue(catalogAuthorizationException.getCause().getMessage().contains(studyFqn)); + assertTrue(catalogAuthorizationException.getCause().getMessage().contains(studyFqn2)); + assertTrue(catalogAuthorizationException.getCause().getMessage().contains("study_3")); + + // Add orgAdminUser1 to the administrators group of the third study + catalogManager.getStudyManager().updateGroup("study_3", ParamConstants.ADMINS_GROUP, ParamUtils.BasicUpdateAction.ADD, + new GroupUpdateParams(Collections.singletonList(orgAdminUserId1)), ownerToken); + catalogAuthorizationException = assertThrows(CatalogAuthorizationException.class, + () -> catalogManager.getProjectManager().update(project1, params, INCLUDE_RESULT, orgAdminToken1)); + assertTrue(catalogAuthorizationException.getCause().getMessage().contains(studyFqn)); + assertTrue(catalogAuthorizationException.getCause().getMessage().contains(studyFqn2)); + assertFalse(catalogAuthorizationException.getCause().getMessage().contains("study_3")); + + // Add orgAdminUser1 to the administrators group of the second study + catalogManager.getStudyManager().updateGroup(studyFqn2, ParamConstants.ADMINS_GROUP, ParamUtils.BasicUpdateAction.ADD, + new GroupUpdateParams(Collections.singletonList(orgAdminUserId1)), ownerToken); + catalogAuthorizationException = assertThrows(CatalogAuthorizationException.class, + () -> catalogManager.getProjectManager().update(project1, params, INCLUDE_RESULT, orgAdminToken1)); + assertTrue(catalogAuthorizationException.getCause().getMessage().contains(studyFqn)); + assertFalse(catalogAuthorizationException.getCause().getMessage().contains(studyFqn2)); + assertFalse(catalogAuthorizationException.getCause().getMessage().contains("study_3")); + + // Add orgAdminUser1 to the administrators group of the remaining study + catalogManager.getStudyManager().updateGroup(studyFqn, ParamConstants.ADMINS_GROUP, ParamUtils.BasicUpdateAction.ADD, + new GroupUpdateParams(Collections.singletonList(orgAdminUserId1)), ownerToken); + catalogManager.getProjectManager().update(project1, params, INCLUDE_RESULT, orgAdminToken1); + } + + @Test + public void updatePrivateParamsFromProjectTest() throws CatalogException { + catalogManager.getProjectManager().setDatastoreVariant(projectFqn1, new DataStore(), opencgaToken); + catalogManager.getProjectManager().setDatastoreVariant(projectFqn1, new DataStore(), ownerToken); + catalogManager.getProjectManager().setDatastoreVariant(projectFqn1, new DataStore(), orgAdminToken1); + thrown.expect(CatalogAuthorizationException.class); + thrown.expectMessage("administrators"); + catalogManager.getProjectManager().setDatastoreVariant(projectFqn1, new DataStore(), normalToken1); } @Test @@ -503,15 +311,15 @@ public void testLimitProjects() throws CatalogException { for (int i = 0; i < 20; i++) { catalogManager.getProjectManager().create(new ProjectCreateParams() .setId("project_" + i) - .setOrganism(new ProjectOrganism("hsapiens", "grch38")), QueryOptions.empty(), token); + .setOrganism(new ProjectOrganism("hsapiens", "grch38")), QueryOptions.empty(), ownerToken); for (int j = 0; j < 2; j++) { catalogManager.getStudyManager().create("project_" + i, new Study().setId("study_" + i + "_" + j), QueryOptions.empty(), - token); + ownerToken); } } - OpenCGAResult results = catalogManager.getProjectManager().search(new Query(), new QueryOptions(QueryOptions.LIMIT, 10), - token); + OpenCGAResult results = catalogManager.getProjectManager().search(organizationId, new Query(), new QueryOptions(QueryOptions.LIMIT, 10), + ownerToken); assertEquals(10, results.getNumResults()); } @@ -521,9 +329,6 @@ public void testLimitProjects() throws CatalogException { @Test public void testModifyStudy() throws Exception { - Query query = new Query(StudyDBAdaptor.QueryParams.OWNER.key(), "user"); - String studyId = catalogManager.getStudyManager().search(query, null, token).first().getId(); - String newName = "Phase 1 " + RandomStringUtils.randomAlphanumeric(20); String newDescription = RandomStringUtils.randomAlphanumeric(500); @@ -533,9 +338,9 @@ public void testModifyStudy() throws Exception { .setName(newName) .setDescription(newDescription) .setAttributes(attributes); - catalogManager.getStudyManager().update(studyId, updateParams, null, token); + catalogManager.getStudyManager().update(studyFqn, updateParams, null, ownerToken); - DataResult result = catalogManager.getStudyManager().get(studyId, null, token); + DataResult result = catalogManager.getStudyManager().get(studyFqn, null, ownerToken); System.out.println(result); Study study = result.first(); assertEquals(study.getName(), newName); @@ -545,139 +350,151 @@ public void testModifyStudy() throws Exception { } assertNotEquals("20180101120000", study.getCreationDate()); - catalogManager.getStudyManager().update(studyId, new StudyUpdateParams().setCreationDate("20180101120000"), null, token); - study = catalogManager.getStudyManager().get(studyId, null, token).first(); + catalogManager.getStudyManager().update(studyFqn, new StudyUpdateParams().setCreationDate("20180101120000"), null, ownerToken); + study = catalogManager.getStudyManager().get(studyFqn, null, ownerToken).first(); assertEquals("20180101120000", study.getCreationDate()); } @Test public void testGetAllStudies() throws CatalogException { - Query query = new Query(ProjectDBAdaptor.QueryParams.USER_ID.key(), "user"); - String projectId = catalogManager.getProjectManager().search(query, null, token).first().getId(); + Query query = new Query(); + String projectId = catalogManager.getProjectManager().search(organizationId, query, null, ownerToken).first().getFqn(); Study study_1 = catalogManager.getStudyManager().create(projectId, new Study().setId("study_1").setCreationDate("20150101120000") - , null, token).first(); + , null, ownerToken).first(); assertEquals("20150101120000", study_1.getCreationDate()); - catalogManager.getStudyManager().create(projectId, "study_2", null, "study_2", "description", null, null, null, null, null, token); + catalogManager.getStudyManager().create(projectId, "study_2", null, "study_2", "description", null, null, null, null, null, ownerToken); - catalogManager.getStudyManager().create(projectId, "study_3", null, "study_3", "description", null, null, null, null, null, token); + catalogManager.getStudyManager().create(projectId, "study_3", null, "study_3", "description", null, null, null, null, null, ownerToken); String study_4 = catalogManager.getStudyManager().create(projectId, "study_4", null, "study_4", "description", null, null, null, - null, null, token).first().getId(); + null, null, ownerToken).first().getId(); - assertEquals(new HashSet<>(Collections.emptyList()), catalogManager.getStudyManager().search(new Query(StudyDBAdaptor.QueryParams - .GROUP_USER_IDS.key(), "user2"), null, token).getResults().stream().map(Study::getId) - .collect(Collectors.toSet())); + assertEquals(new HashSet<>(Collections.singletonList(studyId)), catalogManager.getStudyManager().searchInOrganization(organizationId, + new Query(StudyDBAdaptor.QueryParams.GROUP_USER_IDS.key(), normalUserId1), null, ownerToken) + .getResults().stream().map(Study::getId).collect(Collectors.toSet())); -// catalogManager.getStudyManager().createGroup(Long.toString(study_4), "admins", "user3", sessionIdUser); - catalogManager.getStudyManager().updateGroup(study_4, "admins", ParamUtils.BasicUpdateAction.SET, - new GroupUpdateParams(Collections.singletonList("user3")), token); - assertEquals(new HashSet<>(Arrays.asList("study_4")), catalogManager.getStudyManager().search(new Query(StudyDBAdaptor.QueryParams - .GROUP_USER_IDS.key(), "user3"), null, token).getResults().stream().map(Study::getId) +// catalogManager.getStudyManager().createGroup(Long.toString(study_4), "admins", normalUserId3, sessionIdUser); + catalogManager.getStudyManager().updateGroup(study_4, "admins", ParamUtils.BasicUpdateAction.ADD, + new GroupUpdateParams(Collections.singletonList(normalUserId1)), ownerToken); + assertEquals(new HashSet<>(Arrays.asList(studyId, "study_4")), catalogManager.getStudyManager().search(projectId, + new Query(StudyDBAdaptor.QueryParams.GROUP_USER_IDS.key(), normalUserId1), null, ownerToken).getResults().stream().map(Study::getId) .collect(Collectors.toSet())); - assertEquals(new HashSet<>(Arrays.asList("phase1", "phase3", "study_1", "study_2", "study_3", "study_4")), - catalogManager.getStudyManager().search(new Query(StudyDBAdaptor.QueryParams.PROJECT_ID.key(), projectId), null, token) + assertEquals(new HashSet<>(Arrays.asList(studyId, studyId2, "study_1", "study_2", "study_3", "study_4")), + catalogManager.getStudyManager().search(projectId, new Query(StudyDBAdaptor.QueryParams.PROJECT_ID.key(), projectId), null, ownerToken) .getResults().stream().map(Study::getId).collect(Collectors.toSet())); - assertEquals(new HashSet<>(Arrays.asList("phase1", "phase3", "study_1", "study_2", "study_3", "study_4")), - catalogManager.getStudyManager().search(new Query(), null, token).getResults().stream().map(Study::getId) + assertEquals(new HashSet<>(Arrays.asList(studyId, studyId2, "study_1", "study_2", "study_3", "study_4")), + catalogManager.getStudyManager().search(projectId, new Query(), null, ownerToken).getResults().stream().map(Study::getId) .collect(Collectors.toSet())); - assertEquals(new HashSet<>(Arrays.asList("study_1", "study_2", "study_3", "study_4")), catalogManager.getStudyManager().search(new - Query(StudyDBAdaptor.QueryParams.ID.key(), "~^study"), null, token).getResults().stream() + assertEquals(new HashSet<>(Arrays.asList("study_1", "study_2", "study_3", "study_4")), catalogManager.getStudyManager().search(projectId, new + Query(StudyDBAdaptor.QueryParams.ID.key(), "~^study"), null, ownerToken).getResults().stream() .map(Study::getId).collect(Collectors.toSet())); - assertEquals(Collections.singleton("s1"), catalogManager.getStudyManager().search(new Query(), null, sessionIdUser2).getResults() + assertEquals(new HashSet<>(Arrays.asList(studyId, studyId2)), catalogManager.getStudyManager().search(projectId, new Query(), null, + studyAdminToken1).getResults() .stream() .map(Study::getId).collect(Collectors.toSet())); } @Test public void testGetId() throws CatalogException { - // Create another study with alias phase3 - catalogManager.getStudyManager().create(project2, "phase3", null, "Phase 3", "d", null, null, null, null, null, sessionIdUser2); + // Create another study with alias study3 + Study study = catalogManager.getStudyManager().create(project1, "study3", null, "Phase 3", "d", null, null, null, null, + INCLUDE_RESULT, orgAdminToken1).first(); - String userId = catalogManager.getUserManager().getUserId(token); - List uids = catalogManager.getStudyManager().resolveIds(Arrays.asList("*"), userId) + List uids = catalogManager.getStudyManager().resolveIds(Arrays.asList("*"), orgOwnerUserId, organizationId) .stream() .map(Study::getUid) .collect(Collectors.toList()); - assertTrue(uids.contains(studyUid) && uids.contains(studyUid2)); + assertTrue(uids.contains(studyUid) && uids.contains(study.getUid())); - uids = catalogManager.getStudyManager().resolveIds(Collections.emptyList(), userId) + uids = catalogManager.getStudyManager().resolveIds(Collections.emptyList(), orgOwnerUserId, organizationId) .stream() .map(Study::getUid) .collect(Collectors.toList()); - assertTrue(uids.contains(studyUid) && uids.contains(studyUid2)); + assertTrue(uids.contains(studyUid) && uids.contains(study.getUid())); - uids = catalogManager.getStudyManager().resolveIds(Collections.emptyList(), userId) + uids = catalogManager.getStudyManager().resolveIds(Collections.emptyList(), orgOwnerUserId, organizationId) .stream() .map(Study::getUid) .collect(Collectors.toList()); - assertTrue(uids.contains(studyUid) && uids.contains(studyUid2)); + assertTrue(uids.contains(studyUid) && uids.contains(study.getUid())); - uids = catalogManager.getStudyManager().resolveIds(Arrays.asList("1000G:*"), userId) + uids = catalogManager.getStudyManager().resolveIds(Arrays.asList("1000G:*"), orgOwnerUserId, organizationId) .stream() .map(Study::getUid) .collect(Collectors.toList()); - assertTrue(uids.contains(studyUid) && uids.contains(studyUid2)); + assertTrue(uids.contains(studyUid) && uids.contains(study.getUid())); - uids = catalogManager.getStudyManager().resolveIds(Arrays.asList(userId + "@1000G:*"), userId) + uids = catalogManager.getStudyManager().resolveIds(Arrays.asList(organizationId + "@1000G:*"), orgOwnerUserId, organizationId) .stream() .map(Study::getUid) .collect(Collectors.toList()); - assertTrue(uids.contains(studyUid) && uids.contains(studyUid2)); + assertTrue(uids.contains(studyUid) && uids.contains(study.getUid())); - uids = catalogManager.getStudyManager().resolveIds(Arrays.asList(userId + "@1000G:phase1", userId + "@1000G:phase3"), userId) + uids = catalogManager.getStudyManager().resolveIds(Arrays.asList(organizationId + "@1000G:phase1", organizationId + "@1000G:study3"), orgOwnerUserId, organizationId) .stream() .map(Study::getUid) .collect(Collectors.toList()); - assertTrue(uids.contains(studyUid) && uids.contains(studyUid2)); + assertTrue(uids.contains(studyUid) && uids.contains(study.getUid())); - uids = catalogManager.getStudyManager().resolveIds(Arrays.asList(userId + "@1000G:phase1", "phase3"), userId) + uids = catalogManager.getStudyManager().resolveIds(Arrays.asList(organizationId + "@1000G:phase1", "study3"), orgOwnerUserId, organizationId) .stream() .map(Study::getUid) .collect(Collectors.toList()); - assertTrue(uids.contains(studyUid) && uids.contains(studyUid2)); + assertTrue(uids.contains(studyUid) && uids.contains(study.getUid())); - uids = catalogManager.getStudyManager().resolveIds(Arrays.asList(userId + "@1000G:phase3", studyFqn), userId) + uids = catalogManager.getStudyManager().resolveIds(Arrays.asList(organizationId + "@1000G:study3", studyFqn), orgOwnerUserId, organizationId) .stream() .map(Study::getUid) .collect(Collectors.toList()); - assertTrue(uids.contains(studyUid) && uids.contains(studyUid2)); + assertTrue(uids.contains(studyUid) && uids.contains(study.getUid())); try { - catalogManager.getStudyManager().resolveId(null, userId); + catalogManager.getStudyManager().resolveId(null, orgOwnerUserId, organizationId); fail("This method should fail because it should find several studies"); } catch (CatalogException e) { assertTrue(e.getMessage().contains("More than one study")); } - - Study study = catalogManager.getStudyManager().resolveId("phase3", userId); - assertEquals(studyUid2, study.getUid()); } @Test public void testGetOnlyStudyUserAnonymousCanSee() throws CatalogException { + String otherOrg = "otherOrg"; + catalogManager.getOrganizationManager().create(new OrganizationCreateParams().setId(otherOrg).setName("Test"), QueryOptions.empty(), + opencgaToken); + catalogManager.getUserManager().create(new User().setId(orgOwnerUserId).setName(orgOwnerUserId).setOrganization(otherOrg), + TestParamConstants.PASSWORD, opencgaToken); + ownerToken = catalogManager.getUserManager().login(otherOrg, orgOwnerUserId, TestParamConstants.PASSWORD).getToken(); + + catalogManager.getOrganizationManager().update(otherOrg, + new OrganizationUpdateParams() + .setOwner(orgOwnerUserId), + null, opencgaToken); + Project project = catalogManager.getProjectManager().create("myProject", "Project about some genomes", "", "Homo sapiens", + null, "GRCh38", INCLUDE_RESULT, ownerToken).first(); + StudyManager studyManager = catalogManager.getStudyManager(); try { - studyManager.resolveIds(Collections.emptyList(), "*"); + studyManager.resolveIds(Collections.emptyList(), ParamConstants.ANONYMOUS_USER_ID, otherOrg); fail("This should throw an exception. No studies should be found for user anonymous"); } catch (CatalogException e) { } // Create another study with alias phase3 - DataResult study = catalogManager.getStudyManager().create(String.valueOf(project2), "phase3", null, "Phase 3", "d", null, - null, null, null, null, sessionIdUser2); + DataResult study = catalogManager.getStudyManager().create(project.getFqn(), "phase3", null, "Phase 3", "d", null, + null, null, null, null, ownerToken); try { - studyManager.resolveIds(Collections.emptyList(), "*"); + studyManager.resolveIds(Collections.emptyList(), ParamConstants.ANONYMOUS_USER_ID, otherOrg); fail("This should throw an exception. No studies should be found for user anonymous"); } catch (CatalogException e) { } - catalogManager.getStudyManager().updateGroup("phase3", "@members", ParamUtils.BasicUpdateAction.ADD, - new GroupUpdateParams(Collections.singletonList("*")), sessionIdUser2); + catalogManager.getStudyManager().updateGroup("phase3", ParamConstants.MEMBERS_GROUP, ParamUtils.BasicUpdateAction.ADD, + new GroupUpdateParams(Collections.singletonList(ParamConstants.ANONYMOUS_USER_ID)), ownerToken); - List studies = studyManager.resolveIds(Collections.emptyList(), "*"); + List studies = studyManager.resolveIds(Collections.emptyList(), ParamConstants.ANONYMOUS_USER_ID, otherOrg); assertEquals(1, studies.size()); assertEquals(study.first().getUid(), studies.get(0).getUid()); } @@ -687,28 +504,28 @@ public void testGetSelectedStudyUserAnonymousCanSee() throws CatalogException { StudyManager studyManager = catalogManager.getStudyManager(); try { - studyManager.resolveIds(Collections.singletonList("phase3"), "*"); + studyManager.resolveIds(Collections.singletonList("phase3"), "*", organizationId); fail("This should throw an exception. No studies should be found for user anonymous"); } catch (CatalogException e) { } // Create another study with alias phase3 - DataResult study = catalogManager.getStudyManager().create(project2, "phase3", null, "Phase 3", "d", null, null, null, - null, null, sessionIdUser2); - catalogManager.getStudyManager().updateGroup("phase3", "@members", ParamUtils.BasicUpdateAction.ADD, - new GroupUpdateParams(Collections.singletonList("*")), sessionIdUser2); + Study study = catalogManager.getStudyManager().create(project2, "phase3", null, "Phase 3", "d", null, null, null, + null, INCLUDE_RESULT, orgAdminToken1).first(); + catalogManager.getStudyManager().updateGroup(study.getFqn(), ParamConstants.MEMBERS_GROUP, ParamUtils.BasicUpdateAction.ADD, + new GroupUpdateParams(Collections.singletonList("*")), orgAdminToken1); - List studies = studyManager.resolveIds(Collections.singletonList("phase3"), "*"); + List studies = studyManager.resolveIds(Collections.singletonList("phase3"), "*", organizationId); assertEquals(1, studies.size()); - assertEquals(study.first().getUid(), studies.get(0).getUid()); + assertEquals(study.getUid(), studies.get(0).getUid()); } @Test public void testCreatePermissionRules() throws CatalogException { - PermissionRule rules = new PermissionRule("rules1", new Query("a", "b"), Arrays.asList("user2", "user3"), + PermissionRule rules = new PermissionRule("rules1", new Query("a", "b"), Arrays.asList(normalUserId1, normalUserId2), Arrays.asList(SamplePermissions.VIEW.name(), SamplePermissions.WRITE.name())); DataResult permissionRulesDataResult = catalogManager.getStudyManager().createPermissionRule( - studyFqn, Enums.Entity.SAMPLES, rules, token); + studyFqn, Enums.Entity.SAMPLES, rules, ownerToken); assertEquals(1, permissionRulesDataResult.getNumResults()); assertEquals("rules1", permissionRulesDataResult.first().getId()); assertEquals(1, permissionRulesDataResult.first().getQuery().size()); @@ -718,36 +535,36 @@ public void testCreatePermissionRules() throws CatalogException { // Add new permission rules object rules.setId("rules2"); permissionRulesDataResult = catalogManager.getStudyManager().createPermissionRule(studyFqn, Enums.Entity.SAMPLES, rules, - token); + ownerToken); assertEquals(1, permissionRulesDataResult.getNumResults()); assertEquals(rules, permissionRulesDataResult.first()); } @Test public void testUpdatePermissionRulesIncorrectPermission() throws CatalogException { - PermissionRule rules = new PermissionRule("rules1", new Query("a", "b"), Arrays.asList("user2", "user3"), + PermissionRule rules = new PermissionRule("rules1", new Query("a", "b"), Arrays.asList(normalUserId2, normalUserId3), Arrays.asList("VV", "UPDATE")); thrown.expect(CatalogException.class); thrown.expectMessage("Detected unsupported"); - catalogManager.getStudyManager().createPermissionRule(studyFqn, Enums.Entity.SAMPLES, rules, token); + catalogManager.getStudyManager().createPermissionRule(studyFqn, Enums.Entity.SAMPLES, rules, ownerToken); } @Test public void testUpdatePermissionRulesNonExistingUser() throws CatalogException { - PermissionRule rules = new PermissionRule("rules1", new Query("a", "b"), Arrays.asList("user2", "user20"), + PermissionRule rules = new PermissionRule("rules1", new Query("a", "b"), Arrays.asList(normalUserId2, "user20"), Arrays.asList(SamplePermissions.VIEW.name(), SamplePermissions.WRITE.name())); thrown.expect(CatalogException.class); thrown.expectMessage("does not exist"); - catalogManager.getStudyManager().createPermissionRule(studyFqn, Enums.Entity.SAMPLES, rules, token); + catalogManager.getStudyManager().createPermissionRule(studyFqn, Enums.Entity.SAMPLES, rules, ownerToken); } @Test public void testUpdatePermissionRulesNonExistingGroup() throws CatalogException { - PermissionRule rules = new PermissionRule("rules1", new Query("a", "b"), Arrays.asList("user2", "@group"), + PermissionRule rules = new PermissionRule("rules1", new Query("a", "b"), Arrays.asList(normalUserId1, "@group"), Arrays.asList(SamplePermissions.VIEW.name(), SamplePermissions.WRITE.name())); thrown.expect(CatalogException.class); thrown.expectMessage("not found"); - catalogManager.getStudyManager().createPermissionRule(studyFqn, Enums.Entity.SAMPLES, rules, token); + catalogManager.getStudyManager().createPermissionRule(studyFqn, Enums.Entity.SAMPLES, rules, ownerToken); } @Test @@ -755,14 +572,13 @@ public void removeAllPermissionsToMember() throws CatalogException { StudyManager studyManager = catalogManager.getStudyManager(); // Assign permissions to study - DataResult groupDataResult = studyManager.updateGroup(studyFqn, "@members", ParamUtils.BasicUpdateAction.ADD, - new GroupUpdateParams(Arrays.asList("user2", "user3")), token); - assertEquals(3, groupDataResult.first().getUserIds().size()); - assertEquals("@members", groupDataResult.first().getId()); + DataResult groupDataResult = studyManager.updateGroup(studyFqn, ParamConstants.MEMBERS_GROUP, + ParamUtils.BasicUpdateAction.ADD, new GroupUpdateParams(Arrays.asList(normalUserId2, normalUserId3)), ownerToken); + assertEquals(8, groupDataResult.first().getUserIds().size()); + assertEquals(ParamConstants.MEMBERS_GROUP, groupDataResult.first().getId()); // Obtain all samples from study - DataResult sampleDataResult = catalogManager.getSampleManager().search(studyFqn, new Query(), QueryOptions - .empty(), token); + DataResult sampleDataResult = catalogManager.getSampleManager().search(studyFqn, new Query(), QueryOptions.empty(), ownerToken); assertTrue(sampleDataResult.getNumResults() > 0); // Assign permissions to all the samples @@ -772,12 +588,12 @@ public void removeAllPermissionsToMember() throws CatalogException { .map(Sample::getId) .collect(Collectors.toList()); DataResult> sampleAclResult = catalogManager.getSampleManager().updateAcl(studyFqn, - sampleIds, "user2,user3", sampleAclParams, ParamUtils.AclAction.SET, token); + sampleIds, normalUserId2 + "," + normalUserId3, sampleAclParams, ParamUtils.AclAction.SET, ownerToken); assertEquals(sampleIds.size(), sampleAclResult.getNumResults()); for (AclEntryList result : sampleAclResult.getResults()) { assertEquals(2, result.getAcl().size()); - assertTrue(result.getAcl().stream().map(AclEntry::getMember).collect(Collectors.toList()).containsAll(Arrays.asList("user2", "user3"))); - assertEquals("user2", result.getAcl().get(0).getMember()); + assertTrue(result.getAcl().stream().map(AclEntry::getMember).collect(Collectors.toList()).containsAll(Arrays.asList(normalUserId2, normalUserId3))); + assertEquals(normalUserId2, result.getAcl().get(0).getMember()); assertTrue(result.getAcl().get(0).getPermissions().containsAll(Arrays.asList(SamplePermissions.VIEW, SamplePermissions.WRITE))); assertTrue(result.getAcl().get(1).getPermissions().containsAll(Arrays.asList(SamplePermissions.VIEW, @@ -785,25 +601,23 @@ public void removeAllPermissionsToMember() throws CatalogException { } // Remove all the permissions to both users in the study. That should also remove the permissions they had in all the samples. - groupDataResult = studyManager.updateGroup(studyFqn, "@members", ParamUtils.BasicUpdateAction.REMOVE, - new GroupUpdateParams(Arrays.asList("user2", "user3")), token); - assertEquals(1, groupDataResult.first().getUserIds().size()); + groupDataResult = studyManager.updateGroup(studyFqn, ParamConstants.MEMBERS_GROUP, ParamUtils.BasicUpdateAction.REMOVE, + new GroupUpdateParams(Arrays.asList(normalUserId2, normalUserId3)), ownerToken); + assertEquals(6, groupDataResult.first().getUserIds().size()); // Get sample permissions for those members for (Sample sample : sampleDataResult.getResults()) { long sampleUid = sample.getUid(); OpenCGAResult> sampleAcl = - catalogManager.getAuthorizationManager().getAcl("user", studyUid, sampleUid, Collections.singletonList("user2"), - Enums.Resource.SAMPLE, SamplePermissions.class); + catalogManager.getAuthorizationManager().getAcl(organizationId, studyUid, sampleUid, Collections.singletonList(normalUserId2), Enums.Resource.SAMPLE, SamplePermissions.class, orgOwnerUserId); assertEquals(1, sampleAcl.getNumResults()); assertEquals(1, sampleAcl.first().getAcl().size()); - assertEquals("user2", sampleAcl.first().getAcl().get(0).getMember()); + assertEquals(normalUserId2, sampleAcl.first().getAcl().get(0).getMember()); assertNull(sampleAcl.first().getAcl().get(0).getPermissions()); - sampleAcl = catalogManager.getAuthorizationManager().getAcl("user", studyUid, sampleUid, Collections.singletonList("user3"), - Enums.Resource.SAMPLE, SamplePermissions.class); + sampleAcl = catalogManager.getAuthorizationManager().getAcl(organizationId, studyUid, sampleUid, Collections.singletonList(normalUserId3), Enums.Resource.SAMPLE, SamplePermissions.class, orgOwnerUserId); assertEquals(1, sampleAcl.getNumResults()); assertEquals(1, sampleAcl.first().getAcl().size()); - assertEquals("user3", sampleAcl.first().getAcl().get(0).getMember()); + assertEquals(normalUserId3, sampleAcl.first().getAcl().get(0).getMember()); assertNull(sampleAcl.first().getAcl().get(0).getPermissions()); } } @@ -813,14 +627,14 @@ public void removeUsersFromStudies() throws CatalogException { StudyManager studyManager = catalogManager.getStudyManager(); // Assign permissions to study - DataResult groupDataResult = studyManager.updateGroup(studyFqn, "@members", ParamUtils.BasicUpdateAction.ADD, - new GroupUpdateParams(Arrays.asList("user2", "user3")), token); - assertEquals(3, groupDataResult.first().getUserIds().size()); - assertEquals("@members", groupDataResult.first().getId()); + DataResult groupDataResult = studyManager.updateGroup(studyFqn, ParamConstants.MEMBERS_GROUP, + ParamUtils.BasicUpdateAction.ADD, new GroupUpdateParams(Arrays.asList(normalUserId2, normalUserId3)), ownerToken); + assertEquals(8, groupDataResult.first().getUserIds().size()); + assertEquals(ParamConstants.MEMBERS_GROUP, groupDataResult.first().getId()); // Obtain all samples from study DataResult sampleDataResult = catalogManager.getSampleManager().search(studyFqn, new Query(), QueryOptions - .empty(), token); + .empty(), ownerToken); assertTrue(sampleDataResult.getNumResults() > 0); // Assign permissions to all the samples @@ -829,57 +643,51 @@ public void removeUsersFromStudies() throws CatalogException { List sampleIds = sampleDataResult.getResults().stream().map(Sample::getId).collect(Collectors.toList()); OpenCGAResult> sampleAclResult = catalogManager.getSampleManager().updateAcl(studyFqn, - sampleIds, "user2,user3", sampleAclParams, ParamUtils.AclAction.SET, token); + sampleIds, normalUserId2 + "," + normalUserId3, sampleAclParams, ParamUtils.AclAction.SET, ownerToken); assertEquals(sampleIds.size(), sampleAclResult.getNumResults()); for (AclEntryList result : sampleAclResult.getResults()) { assertEquals(2, result.getAcl().size()); - assertTrue(result.getAcl().stream().map(AclEntry::getMember).collect(Collectors.toList()).containsAll(Arrays.asList("user2", "user3"))); - assertEquals("user2", result.getAcl().get(0).getMember()); - assertTrue(result.getAcl().get(0).getPermissions().containsAll(Arrays.asList(SamplePermissions.VIEW, - SamplePermissions.WRITE))); - assertTrue(result.getAcl().get(1).getPermissions().containsAll(Arrays.asList(SamplePermissions.VIEW, - SamplePermissions.WRITE))); + assertTrue(result.getAcl().stream().map(AclEntry::getMember).collect(Collectors.toList()).containsAll(Arrays.asList(normalUserId2, normalUserId3))); + assertEquals(normalUserId2, result.getAcl().get(0).getMember()); + assertTrue(result.getAcl().get(0).getPermissions().containsAll(Arrays.asList(SamplePermissions.VIEW, SamplePermissions.WRITE))); + assertTrue(result.getAcl().get(1).getPermissions().containsAll(Arrays.asList(SamplePermissions.VIEW, SamplePermissions.WRITE))); } - catalogManager.getStudyManager().updateGroup(studyFqn, "@members", ParamUtils.BasicUpdateAction.REMOVE, - new GroupUpdateParams(Arrays.asList("user2", "user3")), token); + catalogManager.getStudyManager().updateGroup(studyFqn, ParamConstants.MEMBERS_GROUP, ParamUtils.BasicUpdateAction.REMOVE, + new GroupUpdateParams(Arrays.asList(normalUserId2, normalUserId3)), ownerToken); - String userId1 = catalogManager.getUserManager().getUserId(token); - Study study3 = catalogManager.getStudyManager().resolveId(studyFqn, userId1); + Study study3 = catalogManager.getStudyManager().resolveId(studyFqn, orgOwnerUserId, organizationId); OpenCGAResult> studyAcl = catalogManager.getAuthorizationManager() - .getStudyAcl(userId1, study3.getUid(), "user2"); + .getStudyAcl(organizationId, study3.getUid(), normalUserId2, orgOwnerUserId); assertEquals(1, studyAcl.getNumResults()); assertEquals(1, studyAcl.first().getAcl().size()); - assertEquals("user2", studyAcl.first().getAcl().get(0).getMember()); + assertEquals(normalUserId2, studyAcl.first().getAcl().get(0).getMember()); assertNull(studyAcl.first().getAcl().get(0).getPermissions()); - String userId = catalogManager.getUserManager().getUserId(token); - Study study1 = catalogManager.getStudyManager().resolveId(studyFqn, userId); - studyAcl = catalogManager.getAuthorizationManager().getStudyAcl(userId, study1.getUid(), "user3"); + Study study1 = catalogManager.getStudyManager().resolveId(studyFqn, orgOwnerUserId, organizationId); + studyAcl = catalogManager.getAuthorizationManager().getStudyAcl(organizationId, study1.getUid(), normalUserId3, orgOwnerUserId); assertEquals(1, studyAcl.getNumResults()); assertEquals(1, studyAcl.first().getAcl().size()); - assertEquals("user3", studyAcl.first().getAcl().get(0).getMember()); + assertEquals(normalUserId3, studyAcl.first().getAcl().get(0).getMember()); assertNull(studyAcl.first().getAcl().get(0).getPermissions()); - groupDataResult = catalogManager.getStudyManager().getGroup(studyFqn, null, token); + groupDataResult = catalogManager.getStudyManager().getGroup(studyFqn, null, ownerToken); for (Group group : groupDataResult.getResults()) { - assertTrue(!group.getUserIds().contains("user2")); - assertTrue(!group.getUserIds().contains("user3")); + assertFalse(group.getUserIds().contains(normalUserId2)); + assertFalse(group.getUserIds().contains(normalUserId3)); } for (Sample sample : sampleDataResult.getResults()) { OpenCGAResult> sampleAcl = - catalogManager.getAuthorizationManager().getAcl("user", studyUid, sample.getUid(), Collections.singletonList("user2"), - Enums.Resource.SAMPLE, SamplePermissions.class); + catalogManager.getAuthorizationManager().getAcl(organizationId, studyUid, sample.getUid(), Collections.singletonList(normalUserId2), Enums.Resource.SAMPLE, SamplePermissions.class, orgOwnerUserId); assertEquals(1, sampleAcl.getNumResults()); assertEquals(1, sampleAcl.first().getAcl().size()); - assertEquals("user2", sampleAcl.first().getAcl().get(0).getMember()); + assertEquals(normalUserId2, sampleAcl.first().getAcl().get(0).getMember()); assertNull(sampleAcl.first().getAcl().get(0).getPermissions()); - sampleAcl = catalogManager.getAuthorizationManager().getAcl("user", studyUid, sample.getUid(), Collections.singletonList("user3"), - Enums.Resource.SAMPLE, SamplePermissions.class); + sampleAcl = catalogManager.getAuthorizationManager().getAcl(organizationId, studyUid, sample.getUid(), Collections.singletonList(normalUserId3), Enums.Resource.SAMPLE, SamplePermissions.class, orgOwnerUserId); assertEquals(1, sampleAcl.getNumResults()); assertEquals(1, sampleAcl.first().getAcl().size()); - assertEquals("user3", sampleAcl.first().getAcl().get(0).getMember()); + assertEquals(normalUserId3, sampleAcl.first().getAcl().get(0).getMember()); assertNull(sampleAcl.first().getAcl().get(0).getPermissions()); } } @@ -890,102 +698,120 @@ public void removeUsersFromStudies() throws CatalogException { @Test public void testCreateJob() throws CatalogException { - Query query = new Query(StudyDBAdaptor.QueryParams.OWNER.key(), "user"); - String studyId = catalogManager.getStudyManager().search(query, null, token).first().getId(); + String studyId = catalogManager.getStudyManager().searchInOrganization(organizationId, new Query(), null, ownerToken).first().getId(); - catalogManager.getJobManager().submit(studyId, "command-subcommand", null, Collections.emptyMap(), token); - catalogManager.getJobManager().submit(studyId, "command-subcommand2", null, Collections.emptyMap(), token); + catalogManager.getJobManager().submit(studyId, "command-subcommand", null, Collections.emptyMap(), ownerToken); + catalogManager.getJobManager().submit(studyId, "command-subcommand2", null, Collections.emptyMap(), ownerToken); catalogManager.getJobManager().create(studyId, new Job().setId("job1").setInternal(new JobInternal(new Enums.ExecutionStatus(Enums.ExecutionStatus.DONE))), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); catalogManager.getJobManager().create(studyId, new Job().setId("job2").setInternal(new JobInternal(new Enums.ExecutionStatus(Enums.ExecutionStatus.ERROR))), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); catalogManager.getJobManager().create(studyId, new Job().setId("job3").setInternal(new JobInternal(new Enums.ExecutionStatus(Enums.ExecutionStatus.UNREGISTERED))), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); - query = new Query(JobDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), Enums.ExecutionStatus.PENDING); - DataResult unfinishedJobs = catalogManager.getJobManager().search(String.valueOf(studyId), query, null, token); + Query query = new Query(JobDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), Enums.ExecutionStatus.PENDING); + DataResult unfinishedJobs = catalogManager.getJobManager().search(String.valueOf(studyId), query, null, ownerToken); assertEquals(2, unfinishedJobs.getNumResults()); - DataResult allJobs = catalogManager.getJobManager().search(String.valueOf(studyId), (Query) null, null, token); + DataResult allJobs = catalogManager.getJobManager().search(String.valueOf(studyId), (Query) null, null, ownerToken); assertEquals(5, allJobs.getNumResults()); thrown.expectMessage("status different"); thrown.expect(CatalogException.class); catalogManager.getJobManager().create(studyId, new Job().setId("job5").setInternal(new JobInternal(new Enums.ExecutionStatus(Enums.ExecutionStatus.PENDING))), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); } @Test - public void testCreateJobAndReuse() throws CatalogException { - Query query = new Query(StudyDBAdaptor.QueryParams.OWNER.key(), "user"); + public void testKillJob() throws CatalogException { + Job job = catalogManager.getJobManager().submit(studyId, "command-subcommand", null, Collections.emptyMap(), ownerToken).first(); + assertFalse(job.getInternal().isKillJobRequested()); + catalogManager.getJobManager().kill(studyFqn, job.getId(), ownerToken); + job = catalogManager.getJobManager().get(studyFqn, job.getId(), QueryOptions.empty(), ownerToken).first(); + assertTrue(job.getInternal().isKillJobRequested()); + + for (String status : Arrays.asList(Enums.ExecutionStatus.DONE, Enums.ExecutionStatus.ABORTED, Enums.ExecutionStatus.ERROR)) { + job = catalogManager.getJobManager().submit(studyId, "command-subcommand", null, Collections.emptyMap(), ownerToken).first(); + catalogManager.getJobManager().update(studyFqn, job.getId(), + new ObjectMap(JobDBAdaptor.QueryParams.INTERNAL_STATUS.key(), new Enums.ExecutionStatus(status)), + QueryOptions.empty(), ownerToken); + // Try to kill job that is already finished + String jobId = job.getId(); + CatalogException catalogException = assertThrows(CatalogException.class, () -> catalogManager.getJobManager().kill(studyFqn, jobId, ownerToken)); + assertTrue(catalogException.getMessage().contains("status")); + } + } + + @Test + public void testCreateJobAndReuse() throws CatalogException { String project1 = catalogManager.getProjectManager().create("testCreateJobAndReuse_project1", "", "", "Homo sapiens", - null, "GRCh38", INCLUDE_RESULT, token).first().getId(); + null, "GRCh38", INCLUDE_RESULT, ownerToken).first().getId(); String project2 = catalogManager.getProjectManager().create("testCreateJobAndReuse_project2", "", "", "Homo sapiens", - null, "GRCh38", INCLUDE_RESULT, token).first().getId(); + null, "GRCh38", INCLUDE_RESULT, ownerToken).first().getId(); String study1 = catalogManager.getStudyManager().create(project1, new Study() - .setId("studyWithDuplicatedID"), INCLUDE_RESULT, token).first().getUuid(); + .setId("studyWithDuplicatedID"), INCLUDE_RESULT, ownerToken).first().getUuid(); String study2 = catalogManager.getStudyManager().create(project2, new Study() - .setId("studyWithDuplicatedID"), INCLUDE_RESULT, token).first().getUuid(); + .setId("studyWithDuplicatedID"), INCLUDE_RESULT, ownerToken).first().getUuid(); // catalogManager.getConfiguration().getAnalysis().getExecution().getOptions() // .put("jobs.reuse.tools", "command-subcommand"); // String toolId = "command-subcommand"; String toolId = "variant-index"; - String job1 = catalogManager.getJobManager().submit(study1, toolId, null, new ObjectMap("key", 1).append("key2", 2), token).first().getId(); + String job1 = catalogManager.getJobManager().submit(study1, toolId, null, new ObjectMap("key", 1).append("key2", 2), ownerToken).first().getId(); // Same params, different order, empty jobId OpenCGAResult result = catalogManager.getJobManager().submit(study1, toolId, null, new ObjectMap("key2", 2).append("key", 1), - "", "", Collections.emptyList(), Collections.emptyList(), token); + "", "", Collections.emptyList(), Collections.emptyList(), null, null, false, ownerToken); assertEquals(job1, result.first().getId()); assertEquals(1, result.getEvents().size()); assertEquals("reuse", result.getEvents().get(0).getId()); // Same params, different values - result = catalogManager.getJobManager().submit(study1, toolId, null, new ObjectMap("key2", 2).append("key", 2), token); + result = catalogManager.getJobManager().submit(study1, toolId, null, new ObjectMap("key2", 2).append("key", 2), ownerToken); assertNotEquals(job1, result.first().getId()); // Same params, but with jobId result = catalogManager.getJobManager().submit(study1, toolId, null, new ObjectMap("key2", 2).append("key", 2), "MyJobId", "", - Collections.emptyList(), Collections.emptyList(), token); + Collections.emptyList(), Collections.emptyList(), null, null, false, ownerToken); assertNotEquals(job1, result.first().getId()); assertEquals("MyJobId", result.first().getId()); // Same params, but with dependencies result = catalogManager.getJobManager().submit(study1, toolId, null, new ObjectMap("key2", 2).append("key", 2), "", "", - Collections.singletonList(job1), Collections.emptyList(), token); + Collections.singletonList(job1), Collections.emptyList(), null, null, false, ownerToken); assertNotEquals(job1, result.first().getId()); } @Test public void submitJobWithDependenciesFromDifferentStudies() throws CatalogException { - Job first = catalogManager.getJobManager().submit(studyFqn, "command-subcommand", null, Collections.emptyMap(), token).first(); + Job first = catalogManager.getJobManager().submit(studyFqn, "command-subcommand", null, Collections.emptyMap(), ownerToken).first(); Job second = catalogManager.getJobManager().submit(studyFqn2, "command-subcommand2", null, Collections.emptyMap(), null, "", - Collections.singletonList(first.getUuid()), null, token).first(); + Collections.singletonList(first.getUuid()), null, null, null, false, ownerToken).first(); assertEquals(first.getId(), second.getDependsOn().get(0).getId()); thrown.expect(CatalogException.class); thrown.expectMessage("not found"); catalogManager.getJobManager().submit(studyFqn2, "command-subcommand2", null, Collections.emptyMap(), null, "", - Collections.singletonList(first.getId()), null, token); + Collections.singletonList(first.getId()), null, null, null, false, ownerToken); } @Test public void testGetAllJobs() throws CatalogException { - Query query = new Query(StudyDBAdaptor.QueryParams.OWNER.key(), "user"); - String studyId = catalogManager.getStudyManager().search(query, null, token).first().getId(); + Query query = new Query(); + String studyId = catalogManager.getStudyManager().searchInOrganization(organizationId, query, null, ownerToken).first().getId(); - catalogManager.getJobManager().create(studyId, new Job().setId("myErrorJob"), null, token); + catalogManager.getJobManager().create(studyId, new Job().setId("myErrorJob"), null, ownerToken); QueryOptions options = new QueryOptions(QueryOptions.COUNT, true); - DataResult allJobs = catalogManager.getJobManager().search(studyId, null, options, token); + DataResult allJobs = catalogManager.getJobManager().search(studyId, null, options, ownerToken); assertEquals(1, allJobs.getNumMatches()); assertEquals(1, allJobs.getNumResults()); @@ -993,13 +819,11 @@ public void testGetAllJobs() throws CatalogException { @Test public void testJobsTop() throws CatalogException { - Query query = new Query(StudyDBAdaptor.QueryParams.OWNER.key(), "user"); - List studies = catalogManager.getStudyManager().search(query, null, token).getResults(); - System.out.println(studies.size()); + List studies = Arrays.asList(studyFqn, studyFqn2); for (int i = 99; i > 0; i--) { - String studyId = studies.get(i % studies.size()).getId(); - String id = catalogManager.getJobManager().create(studyId, new Job().setId("myJob-" + i), INCLUDE_RESULT, token).first().getId(); + String studyId = studies.get(i % studies.size()); + String id = catalogManager.getJobManager().create(studyId, new Job().setId("myJob-" + i), INCLUDE_RESULT, ownerToken).first().getId(); String status; switch (i % 3) { case 0: @@ -1015,11 +839,11 @@ public void testJobsTop() throws CatalogException { throw new IllegalArgumentException(); } catalogManager.getJobManager().update(studyId, id, new ObjectMap("internal", - new JobInternal(new Enums.ExecutionStatus(status))), new QueryOptions(), token); + new JobInternal(new Enums.ExecutionStatus(status))), new QueryOptions(), ownerToken); } int limit = 20; - DataResult top = catalogManager.getJobManager().top(new Query(), limit, token); + DataResult top = catalogManager.getJobManager().top(organizationId, new Query(), limit, ownerToken); assertEquals(1, top.getNumMatches()); assertEquals(limit, top.first().getJobs().size()); @@ -1030,7 +854,7 @@ public void testJobsTop() throws CatalogException { @Test public void submitJobOwner() throws CatalogException { OpenCGAResult job = catalogManager.getJobManager().submit(studyFqn, "variant-index", Enums.Priority.MEDIUM, new ObjectMap(), - token); + ownerToken); assertEquals(1, job.getNumResults()); assertEquals(Enums.ExecutionStatus.PENDING, job.first().getInternal().getStatus().getId()); @@ -1038,13 +862,13 @@ public void submitJobOwner() throws CatalogException { @Test public void submitJobWithDependencies() throws CatalogException { - Job job1 = catalogManager.getJobManager().submit(studyFqn, "variant-index", Enums.Priority.MEDIUM, new ObjectMap("param", "file1"), token).first(); - Job job2 = catalogManager.getJobManager().submit(studyFqn, "variant-index", Enums.Priority.MEDIUM, new ObjectMap("param", "file2"), token).first(); + Job job1 = catalogManager.getJobManager().submit(studyFqn, "variant-index", Enums.Priority.MEDIUM, new ObjectMap("param", "file1"), ownerToken).first(); + Job job2 = catalogManager.getJobManager().submit(studyFqn, "variant-index", Enums.Priority.MEDIUM, new ObjectMap("param", "file2"), ownerToken).first(); Job job3 = catalogManager.getJobManager().submit(studyFqn, "variant-index", Enums.Priority.MEDIUM, new ObjectMap("param", "file3"), null, null, - Arrays.asList(job1.getId(), job2.getId()), null, token).first(); + Arrays.asList(job1.getId(), job2.getId()), null, null, null, false, ownerToken).first(); Job job4 = catalogManager.getJobManager().submit(studyFqn, "variant-index", Enums.Priority.MEDIUM, new ObjectMap("param", "file4"), null, null, - Arrays.asList(job1.getUuid(), job2.getUuid()), null, token).first(); + Arrays.asList(job1.getUuid(), job2.getUuid()), null, null, null, false, ownerToken).first(); assertEquals(2, job3.getDependsOn().size()); assertEquals(job1.getUuid(), job3.getDependsOn().get(0).getUuid()); @@ -1059,10 +883,10 @@ public void submitJobWithDependencies() throws CatalogException { public void submitJobFromAdminsGroup() throws CatalogException { // Add user to admins group catalogManager.getStudyManager().updateGroup(studyFqn, "@admins", ParamUtils.BasicUpdateAction.ADD, - new GroupUpdateParams(Collections.singletonList("user3")), token); + new GroupUpdateParams(Collections.singletonList(normalUserId3)), ownerToken); OpenCGAResult job = catalogManager.getJobManager().submit(studyFqn, "variant-index", Enums.Priority.MEDIUM, new ObjectMap(), - token); + ownerToken); assertEquals(1, job.getNumResults()); assertEquals(Enums.ExecutionStatus.PENDING, job.first().getInternal().getStatus().getId()); @@ -1072,21 +896,21 @@ public void submitJobFromAdminsGroup() throws CatalogException { public void submitJobWithoutPermissions() throws CatalogException { // Check there are no ABORTED jobs Query query = new Query(JobDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), Enums.ExecutionStatus.ABORTED); - assertEquals(0, catalogManager.getJobManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(0, catalogManager.getJobManager().count(studyFqn, query, ownerToken).getNumMatches()); // Grant view permissions, but no EXECUTION permission - catalogManager.getStudyManager().updateAcl(Collections.singletonList(studyFqn), "user3", - new StudyAclParams("", ""), ParamUtils.AclAction.SET, token); + catalogManager.getStudyManager().updateAcl(studyFqn, normalUserId3, + new StudyAclParams("", ""), ParamUtils.AclAction.SET, ownerToken); try { - catalogManager.getJobManager().submit(studyFqn, "variant-index", Enums.Priority.MEDIUM, new ObjectMap(), sessionIdUser3); + catalogManager.getJobManager().submit(studyFqn, "variant-index", Enums.Priority.MEDIUM, new ObjectMap(), normalToken3); fail("Submission should have failed with a message saying the user does not have EXECUTION permissions"); } catch (CatalogException e) { assertTrue(e.getMessage().contains("Permission denied")); } // The previous execution should have created an ABORTED job - OpenCGAResult search = catalogManager.getJobManager().search(studyFqn, query, null, token); + OpenCGAResult search = catalogManager.getJobManager().search(studyFqn, query, null, ownerToken); assertEquals(1, search.getNumResults()); assertEquals("variant-index", search.first().getTool().getId()); } @@ -1095,56 +919,55 @@ public void submitJobWithoutPermissions() throws CatalogException { public void submitJobWithPermissions() throws CatalogException { // Check there are no ABORTED jobs Query query = new Query(JobDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), Enums.ExecutionStatus.ABORTED); - assertEquals(0, catalogManager.getJobManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(0, catalogManager.getJobManager().count(studyFqn, query, ownerToken).getNumMatches()); // Grant view permissions, but no EXECUTION permission - catalogManager.getStudyManager().updateAcl(Collections.singletonList(studyFqn), "user3", - new StudyAclParams(StudyPermissions.Permissions.EXECUTE_JOBS.name(), AuthorizationManager.ROLE_VIEW_ONLY), ParamUtils.AclAction.SET, token); + catalogManager.getStudyManager().updateAcl(studyFqn, normalUserId3, + new StudyAclParams(StudyPermissions.Permissions.EXECUTE_JOBS.name(), AuthorizationManager.ROLE_VIEW_ONLY), ParamUtils.AclAction.SET, ownerToken); OpenCGAResult search = catalogManager.getJobManager().submit(studyFqn, "variant-index", Enums.Priority.MEDIUM, new ObjectMap(), - sessionIdUser3); + orgAdminToken2); assertEquals(1, search.getNumResults()); assertEquals(Enums.ExecutionStatus.PENDING, search.first().getInternal().getStatus().getId()); } @Test public void deleteJobTest() throws CatalogException { - // Grant view permissions, but no EXECUTION permission - catalogManager.getStudyManager().updateAcl(Collections.singletonList(studyFqn), "user3", - new StudyAclParams(StudyPermissions.Permissions.EXECUTE_JOBS.name(), AuthorizationManager.ROLE_VIEW_ONLY), ParamUtils.AclAction.SET, token); + catalogManager.getStudyManager().updateAcl(studyFqn, normalUserId1, + new StudyAclParams(StudyPermissions.Permissions.EXECUTE_JOBS.name(), AuthorizationManager.ROLE_VIEW_ONLY), ParamUtils.AclAction.SET, ownerToken); OpenCGAResult search = catalogManager.getJobManager().submit(studyFqn, "variant-index", Enums.Priority.MEDIUM, new ObjectMap(), - sessionIdUser3); + normalToken1); assertEquals(1, search.getNumResults()); assertEquals(Enums.ExecutionStatus.PENDING, search.first().getInternal().getStatus().getId()); thrown.expect(CatalogException.class); thrown.expectMessage("stop the job"); - catalogManager.getJobManager().delete(studyFqn, Collections.singletonList(search.first().getId()), QueryOptions.empty(), token); + catalogManager.getJobManager().delete(studyFqn, Collections.singletonList(search.first().getId()), QueryOptions.empty(), ownerToken); } @Test public void visitJob() throws CatalogException { - Job job = catalogManager.getJobManager().submit(studyFqn, "variant-index", Enums.Priority.MEDIUM, new ObjectMap(), token) + Job job = catalogManager.getJobManager().submit(studyFqn, "variant-index", Enums.Priority.MEDIUM, new ObjectMap(), ownerToken) .first(); Query query = new Query(JobDBAdaptor.QueryParams.VISITED.key(), false); - assertEquals(1, catalogManager.getJobManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(1, catalogManager.getJobManager().count(studyFqn, query, ownerToken).getNumMatches()); // Now we visit the job - catalogManager.getJobManager().visit(studyFqn, job.getId(), token); - assertEquals(0, catalogManager.getJobManager().count(studyFqn, query, token).getNumMatches()); + catalogManager.getJobManager().visit(studyFqn, job.getId(), ownerToken); + assertEquals(0, catalogManager.getJobManager().count(studyFqn, query, ownerToken).getNumMatches()); query.put(JobDBAdaptor.QueryParams.VISITED.key(), true); - assertEquals(1, catalogManager.getJobManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(1, catalogManager.getJobManager().count(studyFqn, query, ownerToken).getNumMatches()); // Now we update setting job to visited false again JobUpdateParams updateParams = new JobUpdateParams().setVisited(false); - catalogManager.getJobManager().update(studyFqn, job.getId(), updateParams, QueryOptions.empty(), token); - assertEquals(0, catalogManager.getJobManager().count(studyFqn, query, token).getNumMatches()); + catalogManager.getJobManager().update(studyFqn, job.getId(), updateParams, QueryOptions.empty(), ownerToken); + assertEquals(0, catalogManager.getJobManager().count(studyFqn, query, ownerToken).getNumMatches()); query = new Query(JobDBAdaptor.QueryParams.VISITED.key(), false); - assertEquals(1, catalogManager.getJobManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(1, catalogManager.getJobManager().count(studyFqn, query, ownerToken).getNumMatches()); } /** @@ -1153,7 +976,7 @@ public void visitJob() throws CatalogException { @Test public void testCreateVariableSet() throws CatalogException { - Study study = catalogManager.getStudyManager().get("user@1000G:phase1", null, token).first(); + Study study = catalogManager.getStudyManager().get(studyFqn, null, ownerToken).first(); long variableSetNum = study.getVariableSets().size(); List variables = new ArrayList<>(); @@ -1172,17 +995,17 @@ public void testCreateVariableSet() throws CatalogException { null, Collections.emptyMap()) )); DataResult queryResult = catalogManager.getStudyManager().createVariableSet(studyFqn, "vs1", "vs1", true, - false, "", null, variables, Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), token); + false, "", null, variables, Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), ownerToken); assertEquals(1, queryResult.getResults().size()); - study = catalogManager.getStudyManager().get(study.getId(), null, token).first(); + study = catalogManager.getStudyManager().get(study.getId(), null, ownerToken).first(); assertEquals(variableSetNum + 1, study.getVariableSets().size()); } @Test public void testCreateRepeatedVariableSet() throws CatalogException { - Study study = catalogManager.getStudyManager().get("user@1000G:phase1", null, token).first(); + Study study = catalogManager.getStudyManager().get(studyFqn, null, ownerToken).first(); List variables = Arrays.asList( new Variable("NAME", "", Variable.VariableType.STRING, "", true, false, Collections.emptyList(), null, 0, "", "", @@ -1200,13 +1023,11 @@ public void testCreateRepeatedVariableSet() throws CatalogException { ); thrown.expect(CatalogException.class); catalogManager.getStudyManager().createVariableSet(study.getFqn(), "vs1", "vs1", true, false, "", null, variables, - Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), token); + Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), ownerToken); } @Test public void testDeleteVariableSet() throws CatalogException { - Study study = catalogManager.getStudyManager().resolveId("1000G:phase1", "user"); - List variables = Arrays.asList( new Variable("NAME", "", Variable.VariableType.STRING, "", true, false, Collections.emptyList(), null, 0, "", "", null, @@ -1214,21 +1035,19 @@ public void testDeleteVariableSet() throws CatalogException { new Variable("AGE", "", Variable.VariableType.DOUBLE, null, true, false, Collections.singletonList("0:99"), null, 1, "", "", null, Collections.emptyMap()) ); - VariableSet vs1 = catalogManager.getStudyManager().createVariableSet(study.getFqn(), "vs1", "vs1", true, false, "", null, variables, - Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), token).first(); + VariableSet vs1 = catalogManager.getStudyManager().createVariableSet(studyFqn, "vs1", "vs1", true, false, "", null, variables, + Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), ownerToken).first(); - DataResult result = catalogManager.getStudyManager().deleteVariableSet(studyFqn, vs1.getId(), false, token); + DataResult result = catalogManager.getStudyManager().deleteVariableSet(studyFqn, vs1.getId(), false, ownerToken); assertEquals(0, result.getNumResults()); thrown.expect(CatalogException.class); //VariableSet does not exist thrown.expectMessage("not found"); - catalogManager.getStudyManager().getVariableSet(studyFqn, vs1.getId(), null, token); + catalogManager.getStudyManager().getVariableSet(studyFqn, vs1.getId(), null, ownerToken); } @Test public void testGetAllVariableSet() throws CatalogException { - Study study = catalogManager.getStudyManager().resolveId("1000G:phase1", "user"); - List variables = Arrays.asList( new Variable("NAME", "", Variable.VariableType.STRING, "", true, false, Collections.emptyList(), null, 0, "", "", null, @@ -1236,49 +1055,35 @@ public void testGetAllVariableSet() throws CatalogException { new Variable("AGE", "", Variable.VariableType.DOUBLE, null, true, false, Collections.singletonList("0:99"), null, 1, "", "", null, Collections.emptyMap()) ); - VariableSet vs1 = catalogManager.getStudyManager().createVariableSet(study.getFqn(), "vs1", "vs1", true, false, "Cancer", null, + VariableSet vs1 = catalogManager.getStudyManager().createVariableSet(studyFqn, "vs1", "vs1", true, false, "Cancer", null, variables, - Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), token).first(); - VariableSet vs2 = catalogManager.getStudyManager().createVariableSet(study.getFqn(), "vs2", "vs2", true, false, "Virgo", null, + Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), ownerToken).first(); + VariableSet vs2 = catalogManager.getStudyManager().createVariableSet(studyFqn, "vs2", "vs2", true, false, "Virgo", null, variables, - Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), token).first(); - VariableSet vs3 = catalogManager.getStudyManager().createVariableSet(study.getFqn(), "vs3", "vs3", true, false, "Piscis", null, + Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), ownerToken).first(); + VariableSet vs3 = catalogManager.getStudyManager().createVariableSet(studyFqn, "vs3", "vs3", true, false, "Piscis", null, variables, - Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), token).first(); - VariableSet vs4 = catalogManager.getStudyManager().createVariableSet(study.getFqn(), "vs4", "vs4", true, false, "Aries", null, + Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), ownerToken).first(); + VariableSet vs4 = catalogManager.getStudyManager().createVariableSet(studyFqn, "vs4", "vs4", true, false, "Aries", null, variables, - Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), token).first(); + Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), ownerToken).first(); long numResults; - numResults = catalogManager.getStudyManager().searchVariableSets(study.getFqn(), - new Query(StudyDBAdaptor.VariableSetParams.ID.key(), "vs1"), QueryOptions.empty(), token).getNumResults(); + numResults = catalogManager.getStudyManager().getVariableSet(studyFqn, "vs1", QueryOptions.empty(), ownerToken).getNumResults(); assertEquals(1, numResults); - numResults = catalogManager.getStudyManager().searchVariableSets(study.getFqn(), - new Query(StudyDBAdaptor.VariableSetParams.ID.key(), "vs1,vs2"), QueryOptions.empty(), token) - .getNumResults(); - assertEquals(2, numResults); - - numResults = catalogManager.getStudyManager().searchVariableSets(study.getFqn(), - new Query(StudyDBAdaptor.VariableSetParams.ID.key(), "VS1"), QueryOptions.empty(), token).getNumResults(); - assertEquals(0, numResults); - - numResults = catalogManager.getStudyManager().searchVariableSets(study.getFqn(), - new Query(StudyDBAdaptor.VariableSetParams.UID.key(), vs1.getId()), QueryOptions.empty(), token).getNumResults(); + numResults = catalogManager.getStudyManager().getVariableSet(studyFqn, "vs2", QueryOptions.empty(), ownerToken).getNumResults(); assertEquals(1, numResults); -// numResults = catalogManager.getStudyManager().searchVariableSets(studyFqn, -// new Query(StudyDBAdaptor.VariableSetParams.ID.key(), vs1.getId() + "," + vs3.getId()), QueryOptions.empty(), -// sessionIdUser).getNumResults(); - numResults = catalogManager.getStudyManager().searchVariableSets(study.getFqn(), - new Query(StudyDBAdaptor.VariableSetParams.UID.key(), vs3.getId()), QueryOptions.empty(), token).getNumResults(); - assertEquals(1, numResults); + thrown.expect(CatalogException.class); + thrown.expectMessage("not found"); + catalogManager.getStudyManager().getVariableSet(studyFqn, "VS1", QueryOptions.empty(), ownerToken); } @Test public void testDeleteVariableSetInUse() throws CatalogException { String sampleId1 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_1"), - INCLUDE_RESULT, token).first().getId(); + INCLUDE_RESULT, ownerToken).first().getId(); List variables = Arrays.asList( new Variable("NAME", "", "", Variable.VariableType.STRING, "", true, false, Collections.emptyList(), null, 0, "", "", null, Collections.emptyMap()), @@ -1287,18 +1092,18 @@ public void testDeleteVariableSetInUse() throws CatalogException { null, Collections.emptyMap()) ); VariableSet vs1 = catalogManager.getStudyManager().createVariableSet(studyFqn, "vs1", "vs1", true, false, "", null, variables, - Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), token).first(); + Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), ownerToken).first(); Map annotations = new HashMap<>(); annotations.put("NAME", "LINUS"); catalogManager.getSampleManager().update(studyFqn, sampleId1, new SampleUpdateParams() .setAnnotationSets(Collections.singletonList(new AnnotationSet("annotationId", vs1.getId(), annotations))), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); try { - catalogManager.getStudyManager().deleteVariableSet(studyFqn, vs1.getId(), false, token); + catalogManager.getStudyManager().deleteVariableSet(studyFqn, vs1.getId(), false, ownerToken); } finally { - VariableSet variableSet = catalogManager.getStudyManager().getVariableSet(studyFqn, vs1.getId(), null, token).first(); + VariableSet variableSet = catalogManager.getStudyManager().getVariableSet(studyFqn, vs1.getId(), null, ownerToken).first(); assertEquals(vs1.getUid(), variableSet.getUid()); thrown.expect(CatalogDBException.class); //Expect the exception from the try @@ -1315,12 +1120,12 @@ public void testDeleteVariableSetInUse() throws CatalogException { */ @Test public void testCreateCohort() throws CatalogException { - Sample sampleId1 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_1"), INCLUDE_RESULT, token).first(); - Sample sampleId2 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_2"), INCLUDE_RESULT, token).first(); - Sample sampleId3 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_3"), INCLUDE_RESULT, token).first(); + Sample sampleId1 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_1"), INCLUDE_RESULT, ownerToken).first(); + Sample sampleId2 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_2"), INCLUDE_RESULT, ownerToken).first(); + Sample sampleId3 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_3"), INCLUDE_RESULT, ownerToken).first(); Cohort myCohort = catalogManager.getCohortManager().create(studyFqn, new Cohort().setId("MyCohort") .setSamples(Arrays.asList(sampleId1, sampleId2, sampleId3)) - .setStatus(new Status("custom", "custom", "description", TimeUtils.getTime())), INCLUDE_RESULT, token).first(); + .setStatus(new Status("custom", "custom", "description", TimeUtils.getTime())), INCLUDE_RESULT, ownerToken).first(); assertEquals("MyCohort", myCohort.getId()); assertEquals(3, myCohort.getSamples().size()); @@ -1334,13 +1139,13 @@ public void testCreateCohort() throws CatalogException { @Test public void createSampleCohortTest() throws Exception { - Sample sampleId1 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_1"), INCLUDE_RESULT, token).first(); - Sample sampleId2 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_2"), INCLUDE_RESULT, token).first(); - Sample sampleId3 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_3"), INCLUDE_RESULT, token).first(); + Sample sampleId1 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_1"), INCLUDE_RESULT, ownerToken).first(); + Sample sampleId2 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_2"), INCLUDE_RESULT, ownerToken).first(); + Sample sampleId3 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_3"), INCLUDE_RESULT, ownerToken).first(); catalogManager.getCohortManager().create(studyFqn, new Cohort().setId("MyCohort1") - .setSamples(Arrays.asList(sampleId1, sampleId2)), null, token).first(); + .setSamples(Arrays.asList(sampleId1, sampleId2)), null, ownerToken).first(); catalogManager.getCohortManager().create(studyFqn, new Cohort().setId("MyCohort2") - .setSamples(Arrays.asList(sampleId2, sampleId3)), null, token).first(); + .setSamples(Arrays.asList(sampleId2, sampleId3)), null, ownerToken).first(); List ids = new ArrayList<>(); ids.add("SAMPLE_1"); @@ -1349,7 +1154,7 @@ public void createSampleCohortTest() throws Exception { QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.COHORT_IDS.key()); - OpenCGAResult sampleDataResult = catalogManager.getSampleManager().get(studyFqn, ids, options, token); + OpenCGAResult sampleDataResult = catalogManager.getSampleManager().get(studyFqn, ids, options, ownerToken); assertEquals(3, sampleDataResult.getNumResults()); for (Sample sample : sampleDataResult.getResults()) { switch (sample.getId()) { @@ -1373,17 +1178,17 @@ public void createSampleCohortTest() throws Exception { @Test public void updateSampleCohortTest() throws Exception { - Sample sampleId1 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_1"), INCLUDE_RESULT, token).first(); - Sample sampleId2 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_2"), INCLUDE_RESULT, token).first(); - Sample sampleId3 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_3"), INCLUDE_RESULT, token).first(); + Sample sampleId1 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_1"), INCLUDE_RESULT, ownerToken).first(); + Sample sampleId2 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_2"), INCLUDE_RESULT, ownerToken).first(); + Sample sampleId3 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_3"), INCLUDE_RESULT, ownerToken).first(); catalogManager.getCohortManager().create(studyFqn, new Cohort().setId("MyCohort1") - .setSamples(Arrays.asList(sampleId1)), null, token).first(); + .setSamples(Arrays.asList(sampleId1)), null, ownerToken).first(); catalogManager.getCohortManager().create(studyFqn, new Cohort().setId("MyCohort2") - .setSamples(Arrays.asList(sampleId2, sampleId3)), null, token).first(); + .setSamples(Arrays.asList(sampleId2, sampleId3)), null, ownerToken).first(); catalogManager.getCohortManager().update(studyFqn, "MyCohort1", new CohortUpdateParams().setSamples(Collections.singletonList(new SampleReferenceParam().setId(sampleId3.getId()))), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); List ids = new ArrayList<>(); ids.add("SAMPLE_1"); @@ -1393,9 +1198,9 @@ public void updateSampleCohortTest() throws Exception { QueryOptions optionsSample = new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.COHORT_IDS.key()); QueryOptions optionsCohort = new QueryOptions(QueryOptions.INCLUDE, CohortDBAdaptor.QueryParams.SAMPLES.key()); - OpenCGAResult sampleDataResult = catalogManager.getSampleManager().get(studyFqn, ids, optionsSample, token); + OpenCGAResult sampleDataResult = catalogManager.getSampleManager().get(studyFqn, ids, optionsSample, ownerToken); - Cohort cohortResult = catalogManager.getCohortManager().get(studyFqn, "MyCohort1", optionsCohort, token).first(); + Cohort cohortResult = catalogManager.getCohortManager().get(studyFqn, "MyCohort1", optionsCohort, ownerToken).first(); assertEquals(2, cohortResult.getSamples().size()); assertTrue(cohortResult.getSamples().stream().map(Sample::getId).collect(Collectors.toSet()) @@ -1424,13 +1229,13 @@ public void updateSampleCohortTest() throws Exception { @Test public void updateSampleCohortWithThreadsTest() throws Exception { - Sample sampleId1 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_1"), INCLUDE_RESULT, token).first(); - Sample sampleId2 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_2"), INCLUDE_RESULT, token).first(); - Sample sampleId3 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_3"), INCLUDE_RESULT, token).first(); + Sample sampleId1 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_1"), INCLUDE_RESULT, ownerToken).first(); + Sample sampleId2 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_2"), INCLUDE_RESULT, ownerToken).first(); + Sample sampleId3 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_3"), INCLUDE_RESULT, ownerToken).first(); catalogManager.getCohortManager().create(studyFqn, new Cohort().setId("MyCohort1") - .setSamples(Arrays.asList(sampleId1)), null, token).first(); + .setSamples(Arrays.asList(sampleId1)), null, ownerToken).first(); catalogManager.getCohortManager().create(studyFqn, new Cohort().setId("MyCohort2") - .setSamples(Arrays.asList(sampleId2, sampleId3)), null, token).first(); + .setSamples(Arrays.asList(sampleId2, sampleId3)), null, ownerToken).first(); ExecutorService executorService = Executors.newFixedThreadPool(10, new ThreadFactoryBuilder() @@ -1448,7 +1253,7 @@ public void updateSampleCohortWithThreadsTest() throws Exception { String sampleId = "SAMPLE_AUTO_" + i; executorService.submit(() -> { try { - catalogManager.getSampleManager().create(studyFqn, new Sample().setId(sampleId), QueryOptions.empty(), token); + catalogManager.getSampleManager().create(studyFqn, new Sample().setId(sampleId), QueryOptions.empty(), ownerToken); } catch (CatalogException e) { throw new RuntimeException(e); } @@ -1475,7 +1280,7 @@ public void updateSampleCohortWithThreadsTest() throws Exception { QueryOptions queryOptions = new QueryOptions(); queryOptions.put(Constants.ACTIONS, actionMap); for (List innerSampleIds : sampleIds) { - Cohort myCohort1 = catalogManager.getCohortManager().get(studyFqn, "MyCohort1", null, token).first(); + Cohort myCohort1 = catalogManager.getCohortManager().get(studyFqn, "MyCohort1", null, ownerToken).first(); List sampleReferenceParamList = new ArrayList<>(myCohort1.getNumSamples() + innerSampleIds.size()); sampleReferenceParamList.addAll(myCohort1.getSamples().stream().map(s -> new SampleReferenceParam().setId(s.getId())).collect(Collectors.toList())); sampleReferenceParamList.addAll(innerSampleIds.stream().map(s -> new SampleReferenceParam().setId(s)).collect(Collectors.toList())); @@ -1484,7 +1289,7 @@ public void updateSampleCohortWithThreadsTest() throws Exception { try { catalogManager.getCohortManager().update(studyFqn, "MyCohort1", new CohortUpdateParams().setSamples(sampleReferenceParamList), - queryOptions, token); + queryOptions, ownerToken); System.out.println("Execution: " + executionId); } catch (CatalogException e) { throw new RuntimeException(e); @@ -1497,8 +1302,8 @@ public void updateSampleCohortWithThreadsTest() throws Exception { // Ensure persistence Query sampleQuery = new Query(SampleDBAdaptor.QueryParams.COHORT_IDS.key(), "MyCohort1"); - OpenCGAResult search = catalogManager.getSampleManager().search(studyFqn, sampleQuery, SampleManager.INCLUDE_SAMPLE_IDS, token); - Cohort myCohort1 = catalogManager.getCohortManager().get(studyFqn, "MyCohort1", null, token).first(); + OpenCGAResult search = catalogManager.getSampleManager().search(studyFqn, sampleQuery, SampleManager.INCLUDE_SAMPLE_IDS, ownerToken); + Cohort myCohort1 = catalogManager.getCohortManager().get(studyFqn, "MyCohort1", null, ownerToken).first(); assertEquals(search.getNumResults(), myCohort1.getNumSamples()); Set sampleIdSet = search.getResults().stream().map(Sample::getId).collect(Collectors.toSet()); assertTrue(myCohort1.getSamples().stream().map(Sample::getId).collect(Collectors.toSet()).containsAll(sampleIdSet)); @@ -1506,17 +1311,17 @@ public void updateSampleCohortWithThreadsTest() throws Exception { @Test public void deleteSampleCohortTest() throws Exception { - Sample sampleId1 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_1"), INCLUDE_RESULT, token).first(); - Sample sampleId2 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_2"), INCLUDE_RESULT, token).first(); - Sample sampleId3 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_3"), INCLUDE_RESULT, token).first(); + Sample sampleId1 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_1"), INCLUDE_RESULT, ownerToken).first(); + Sample sampleId2 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_2"), INCLUDE_RESULT, ownerToken).first(); + Sample sampleId3 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_3"), INCLUDE_RESULT, ownerToken).first(); catalogManager.getCohortManager().create(studyFqn, new Cohort().setId("MyCohort1") - .setSamples(Arrays.asList(sampleId1)), null, token).first(); + .setSamples(Arrays.asList(sampleId1)), null, ownerToken).first(); catalogManager.getCohortManager().create(studyFqn, new Cohort().setId("MyCohort2") - .setSamples(Arrays.asList(sampleId2, sampleId3)), null, token).first(); + .setSamples(Arrays.asList(sampleId2, sampleId3)), null, ownerToken).first(); catalogManager.getCohortManager().update(studyFqn, "MyCohort1", new CohortUpdateParams().setSamples(Arrays.asList(new SampleReferenceParam().setId(sampleId3.getId()))), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); Map actionMap = new HashMap<>(); actionMap.put(CohortDBAdaptor.QueryParams.SAMPLES.key(), ParamUtils.BasicUpdateAction.REMOVE); @@ -1524,7 +1329,7 @@ public void deleteSampleCohortTest() throws Exception { queryOptions.put(Constants.ACTIONS, actionMap); catalogManager.getCohortManager().update(studyFqn, "MyCohort1", new CohortUpdateParams().setSamples(Arrays.asList(new SampleReferenceParam().setId(sampleId1.getId()))), - queryOptions, token); + queryOptions, ownerToken); List ids = new ArrayList<>(); ids.add("SAMPLE_1"); @@ -1534,9 +1339,9 @@ public void deleteSampleCohortTest() throws Exception { QueryOptions optionsSample = new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.COHORT_IDS.key()); QueryOptions optionsCohort = new QueryOptions(QueryOptions.INCLUDE, CohortDBAdaptor.QueryParams.SAMPLES.key()); - OpenCGAResult sampleDataResult = catalogManager.getSampleManager().get(studyFqn, ids, optionsSample, token); + OpenCGAResult sampleDataResult = catalogManager.getSampleManager().get(studyFqn, ids, optionsSample, ownerToken); - Cohort cohortResult = catalogManager.getCohortManager().get(studyFqn, "MyCohort1", optionsCohort, token).first(); + Cohort cohortResult = catalogManager.getCohortManager().get(studyFqn, "MyCohort1", optionsCohort, ownerToken).first(); assertEquals(1, cohortResult.getSamples().size()); assertTrue(cohortResult.getSamples().stream().map(Sample::getId).collect(Collectors.toSet()) @@ -1564,11 +1369,11 @@ public void deleteSampleCohortTest() throws Exception { @Test public void setSampleCohortTest() throws Exception { - Sample sampleId1 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_1"), INCLUDE_RESULT, token).first(); - Sample sampleId2 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_2"), INCLUDE_RESULT, token).first(); - Sample sampleId3 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_3"), INCLUDE_RESULT, token).first(); + Sample sampleId1 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_1"), INCLUDE_RESULT, ownerToken).first(); + Sample sampleId2 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_2"), INCLUDE_RESULT, ownerToken).first(); + Sample sampleId3 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_3"), INCLUDE_RESULT, ownerToken).first(); catalogManager.getCohortManager().create(studyFqn, new Cohort().setId("MyCohort1") - .setSamples(Arrays.asList(sampleId1)), null, token).first(); + .setSamples(Arrays.asList(sampleId1)), null, ownerToken).first(); Map actionMap = new HashMap<>(); actionMap.put(CohortDBAdaptor.QueryParams.SAMPLES.key(), ParamUtils.BasicUpdateAction.SET); @@ -1578,7 +1383,7 @@ public void setSampleCohortTest() throws Exception { new CohortUpdateParams().setSamples(Arrays.asList( new SampleReferenceParam().setId(sampleId2.getId()), new SampleReferenceParam().setId(sampleId3.getId()))), - queryOptions, token); + queryOptions, ownerToken); List ids = new ArrayList<>(); ids.add("SAMPLE_1"); @@ -1588,9 +1393,9 @@ public void setSampleCohortTest() throws Exception { QueryOptions optionsSample = new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.COHORT_IDS.key()); QueryOptions optionsCohort = new QueryOptions(QueryOptions.INCLUDE, CohortDBAdaptor.QueryParams.SAMPLES.key()); - OpenCGAResult sampleDataResult = catalogManager.getSampleManager().get(studyFqn, ids, optionsSample, token); + OpenCGAResult sampleDataResult = catalogManager.getSampleManager().get(studyFqn, ids, optionsSample, ownerToken); - Cohort cohortResult = catalogManager.getCohortManager().get(studyFqn, "MyCohort1", optionsCohort, token).first(); + Cohort cohortResult = catalogManager.getCohortManager().get(studyFqn, "MyCohort1", optionsCohort, ownerToken).first(); assertEquals(2, cohortResult.getSamples().size()); assertTrue(cohortResult.getSamples().stream().map(Sample::getId).collect(Collectors.toSet()) @@ -1614,7 +1419,7 @@ public void setSampleCohortTest() throws Exception { } // @Test // public void createIndividualWithSamples() throws CatalogException { -// DataResult sampleDataResult = catalogManager.getSampleManager().create("user@1000G:phase1", "sample1", "", "", null, +// DataResult sampleDataResult = catalogManager.getSampleManager().create(studyFqn, "sample1", "", "", null, // null, QueryOptions.empty(), sessionIdUser); // // Sample oldSample = new Sample().setId(sampleDataResult.first().getId()); @@ -1622,9 +1427,9 @@ public void setSampleCohortTest() throws Exception { // ServerUtils.IndividualParameters individualParameters = new ServerUtils.IndividualParameters() // .setName("individual").setSamples(Arrays.asList(oldSample, newSample)); // -// long studyUid = catalogManager.getStudyManager().getId("user", "1000G:phase1"); +// long studyUid = catalogManager.getStudyManager().getId(orgOwnerUserId, "1000G:phase1"); // // We create the individual together with the samples -// DataResult individualDataResult = catalogManager.getIndividualManager().create("user@1000G:phase1", +// DataResult individualDataResult = catalogManager.getIndividualManager().create(studyFqn, // individualParameters, QueryOptions.empty(), sessionIdUser); // // assertEquals(1, individualDataResult.getNumResults()); @@ -1646,52 +1451,50 @@ public void setSampleCohortTest() throws Exception { @Test public void testGetAllCohorts() throws CatalogException { - String studyId = "1000G:phase1"; - - Sample sampleId1 = catalogManager.getSampleManager().create(studyId, new Sample().setId("SAMPLE_1"), INCLUDE_RESULT, token).first(); - Sample sampleId2 = catalogManager.getSampleManager().create(studyId, new Sample().setId("SAMPLE_2"), INCLUDE_RESULT, token).first(); - Sample sampleId3 = catalogManager.getSampleManager().create(studyId, new Sample().setId("SAMPLE_3"), INCLUDE_RESULT, token).first(); - Sample sampleId4 = catalogManager.getSampleManager().create(studyId, new Sample().setId("SAMPLE_4"), INCLUDE_RESULT, token).first(); - Sample sampleId5 = catalogManager.getSampleManager().create(studyId, new Sample().setId("SAMPLE_5"), INCLUDE_RESULT, token).first(); - Cohort myCohort1 = catalogManager.getCohortManager().create(studyId, + Sample sampleId1 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_1"), INCLUDE_RESULT, ownerToken).first(); + Sample sampleId2 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_2"), INCLUDE_RESULT, ownerToken).first(); + Sample sampleId3 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_3"), INCLUDE_RESULT, ownerToken).first(); + Sample sampleId4 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_4"), INCLUDE_RESULT, ownerToken).first(); + Sample sampleId5 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_5"), INCLUDE_RESULT, ownerToken).first(); + Cohort myCohort1 = catalogManager.getCohortManager().create(studyFqn, new Cohort().setId("MyCohort1").setType(Enums.CohortType.FAMILY).setSamples(Arrays.asList(sampleId1, sampleId2, sampleId3)), - INCLUDE_RESULT, token).first(); - Cohort myCohort2 = catalogManager.getCohortManager().create(studyId, + INCLUDE_RESULT, ownerToken).first(); + Cohort myCohort2 = catalogManager.getCohortManager().create(studyFqn, new Cohort().setId("MyCohort2").setType(Enums.CohortType.FAMILY) - .setSamples(Arrays.asList(sampleId1, sampleId2, sampleId3, sampleId4)), INCLUDE_RESULT, token).first(); - Cohort myCohort3 = catalogManager.getCohortManager().create(studyId, new Cohort().setId("MyCohort3") - .setType(Enums.CohortType.CASE_CONTROL).setSamples(Arrays.asList(sampleId3, sampleId4)), INCLUDE_RESULT, token).first(); - catalogManager.getCohortManager().create(studyId, new Cohort().setId("MyCohort4").setType(Enums.CohortType.TRIO) - .setSamples(Arrays.asList(sampleId5, sampleId3)), null, token).first(); + .setSamples(Arrays.asList(sampleId1, sampleId2, sampleId3, sampleId4)), INCLUDE_RESULT, ownerToken).first(); + Cohort myCohort3 = catalogManager.getCohortManager().create(studyFqn, new Cohort().setId("MyCohort3") + .setType(Enums.CohortType.CASE_CONTROL).setSamples(Arrays.asList(sampleId3, sampleId4)), INCLUDE_RESULT, ownerToken).first(); + catalogManager.getCohortManager().create(studyFqn, new Cohort().setId("MyCohort4").setType(Enums.CohortType.TRIO) + .setSamples(Arrays.asList(sampleId5, sampleId3)), null, ownerToken).first(); long numResults; - numResults = catalogManager.getCohortManager().search(studyId, new Query(CohortDBAdaptor.QueryParams.SAMPLES.key(), - sampleId1.getId()), new QueryOptions(), token).getNumResults(); + numResults = catalogManager.getCohortManager().search(studyFqn, new Query(CohortDBAdaptor.QueryParams.SAMPLES.key(), + sampleId1.getId()), new QueryOptions(), ownerToken).getNumResults(); assertEquals(2, numResults); - numResults = catalogManager.getCohortManager().search(studyId, new Query(CohortDBAdaptor.QueryParams.SAMPLES.key(), + numResults = catalogManager.getCohortManager().search(studyFqn, new Query(CohortDBAdaptor.QueryParams.SAMPLES.key(), sampleId1.getId() - + "," + sampleId5.getId()), new QueryOptions(), token).getNumResults(); + + "," + sampleId5.getId()), new QueryOptions(), ownerToken).getNumResults(); assertEquals(3, numResults); - numResults = catalogManager.getCohortManager().search(studyId, new Query(CohortDBAdaptor.QueryParams.ID.key(), "MyCohort2"), new - QueryOptions(), token).getNumResults(); + numResults = catalogManager.getCohortManager().search(studyFqn, new Query(CohortDBAdaptor.QueryParams.ID.key(), "MyCohort2"), new + QueryOptions(), ownerToken).getNumResults(); assertEquals(1, numResults); - numResults = catalogManager.getCohortManager().search(studyId, new Query(CohortDBAdaptor.QueryParams.ID.key(), "~MyCohort."), new - QueryOptions(), token).getNumResults(); + numResults = catalogManager.getCohortManager().search(studyFqn, new Query(CohortDBAdaptor.QueryParams.ID.key(), "~MyCohort."), new + QueryOptions(), ownerToken).getNumResults(); assertEquals(4, numResults); - numResults = catalogManager.getCohortManager().search(studyId, new Query(CohortDBAdaptor.QueryParams.TYPE.key(), - Enums.CohortType.FAMILY), new QueryOptions(), token).getNumResults(); + numResults = catalogManager.getCohortManager().search(studyFqn, new Query(CohortDBAdaptor.QueryParams.TYPE.key(), + Enums.CohortType.FAMILY), new QueryOptions(), ownerToken).getNumResults(); assertEquals(2, numResults); - numResults = catalogManager.getCohortManager().search(studyId, new Query(CohortDBAdaptor.QueryParams.TYPE.key(), "CASE_CONTROL"), - new QueryOptions(), token).getNumResults(); + numResults = catalogManager.getCohortManager().search(studyFqn, new Query(CohortDBAdaptor.QueryParams.TYPE.key(), "CASE_CONTROL"), + new QueryOptions(), ownerToken).getNumResults(); assertEquals(1, numResults); - numResults = catalogManager.getCohortManager().search(studyId, new Query(CohortDBAdaptor.QueryParams.UID.key(), myCohort1.getUid() + - "," + myCohort2.getUid() + "," + myCohort3.getUid()), new QueryOptions(), token).getNumResults(); + numResults = catalogManager.getCohortManager().search(studyFqn, new Query(CohortDBAdaptor.QueryParams.UID.key(), myCohort1.getUid() + + "," + myCohort2.getUid() + "," + myCohort3.getUid()), new QueryOptions(), ownerToken).getNumResults(); assertEquals(3, numResults); } @@ -1701,30 +1504,30 @@ public void testCreateCohortFail() throws CatalogException { List sampleList = Arrays.asList(new Sample().setId("a"), new Sample().setId("b"), new Sample().setId("c")); catalogManager.getCohortManager().create(studyFqn, new Cohort().setId("MyCohort").setType(Enums.CohortType.FAMILY).setSamples(sampleList), - null, token); + null, ownerToken); } @Test public void testCreateCohortAlreadyExisting() throws CatalogException { - Sample sampleId1 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_1"), INCLUDE_RESULT, token).first(); + Sample sampleId1 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_1"), INCLUDE_RESULT, ownerToken).first(); catalogManager.getCohortManager().create(studyFqn, new Cohort().setId("MyCohort").setType(Enums.CohortType.FAMILY) - .setSamples(Collections.singletonList(sampleId1)), null, token).first(); + .setSamples(Collections.singletonList(sampleId1)), null, ownerToken).first(); thrown.expect(CatalogDBException.class); catalogManager.getCohortManager().create(studyFqn, new Cohort().setId("MyCohort").setType(Enums.CohortType.FAMILY) - .setSamples(Collections.singletonList(sampleId1)), null, token).first(); + .setSamples(Collections.singletonList(sampleId1)), null, ownerToken).first(); } @Test public void testUpdateCohort() throws CatalogException { - Sample sampleId1 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_1"), INCLUDE_RESULT, token).first(); - Sample sampleId2 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_2"), INCLUDE_RESULT, token).first(); - Sample sampleId3 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_3"), INCLUDE_RESULT, token).first(); - Sample sampleId4 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_4"), INCLUDE_RESULT, token).first(); - Sample sampleId5 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_5"), INCLUDE_RESULT, token).first(); + Sample sampleId1 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_1"), INCLUDE_RESULT, ownerToken).first(); + Sample sampleId2 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_2"), INCLUDE_RESULT, ownerToken).first(); + Sample sampleId3 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_3"), INCLUDE_RESULT, ownerToken).first(); + Sample sampleId4 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_4"), INCLUDE_RESULT, ownerToken).first(); + Sample sampleId5 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_5"), INCLUDE_RESULT, ownerToken).first(); Cohort myCohort = catalogManager.getCohortManager().create(studyFqn, new Cohort().setId("MyCohort").setType(Enums.CohortType.FAMILY) - .setSamples(Arrays.asList(sampleId1, sampleId2, sampleId3, sampleId1)), INCLUDE_RESULT, token).first(); + .setSamples(Arrays.asList(sampleId1, sampleId2, sampleId3, sampleId1)), INCLUDE_RESULT, ownerToken).first(); assertEquals("MyCohort", myCohort.getId()); assertEquals(3, myCohort.getSamples().size()); @@ -1745,10 +1548,10 @@ public void testUpdateCohort() throws CatalogException { new SampleReferenceParam().setId(sampleId3.getId()), new SampleReferenceParam().setId(sampleId4.getId()), new SampleReferenceParam().setId(sampleId5.getId()))), - options, token); + options, ownerToken); assertEquals(1, result.getNumUpdated()); - Cohort myModifiedCohort = catalogManager.getCohortManager().get(studyFqn, "myModifiedCohort", QueryOptions.empty(), token).first(); + Cohort myModifiedCohort = catalogManager.getCohortManager().get(studyFqn, "myModifiedCohort", QueryOptions.empty(), ownerToken).first(); assertEquals("myModifiedCohort", myModifiedCohort.getId()); assertEquals(4, myModifiedCohort.getSamples().size()); assertEquals(4, myModifiedCohort.getNumSamples()); @@ -1758,7 +1561,7 @@ public void testUpdateCohort() throws CatalogException { assertTrue(myModifiedCohort.getSamples().stream().map(Sample::getUid).collect(Collectors.toList()).contains(sampleId5.getUid())); QueryOptions options1 = new QueryOptions(QueryOptions.INCLUDE, CohortDBAdaptor.QueryParams.NUM_SAMPLES.key()); - myModifiedCohort = catalogManager.getCohortManager().get(studyFqn, "myModifiedCohort", options1, token).first(); + myModifiedCohort = catalogManager.getCohortManager().get(studyFqn, "myModifiedCohort", options1, ownerToken).first(); assertEquals(4, myModifiedCohort.getNumSamples()); assertNull(myModifiedCohort.getSamples()); @@ -1767,10 +1570,10 @@ public void testUpdateCohort() throws CatalogException { result = catalogManager.getCohortManager().update(studyFqn, myModifiedCohort.getId(), new CohortUpdateParams() .setSamples(Collections.emptyList()), - options, token); + options, ownerToken); assertEquals(1, result.getNumUpdated()); - myModifiedCohort = catalogManager.getCohortManager().get(studyFqn, "myModifiedCohort", QueryOptions.empty(), token).first(); + myModifiedCohort = catalogManager.getCohortManager().get(studyFqn, "myModifiedCohort", QueryOptions.empty(), ownerToken).first(); assertEquals(0, myModifiedCohort.getSamples().size()); assertEquals(0, myModifiedCohort.getNumSamples()); @@ -1783,9 +1586,9 @@ public void testUpdateCohort() throws CatalogException { new SampleReferenceParam().setId(sampleId3.getId()), new SampleReferenceParam().setId(sampleId1.getId()), new SampleReferenceParam().setId(sampleId3.getId()))), - options, token); + options, ownerToken); assertEquals(1, result.getNumUpdated()); - myModifiedCohort = catalogManager.getCohortManager().get(studyFqn, "myModifiedCohort", QueryOptions.empty(), token).first(); + myModifiedCohort = catalogManager.getCohortManager().get(studyFqn, "myModifiedCohort", QueryOptions.empty(), ownerToken).first(); assertEquals(2, myModifiedCohort.getSamples().size()); assertEquals(2, myModifiedCohort.getNumSamples()); @@ -1796,9 +1599,9 @@ public void testUpdateCohort() throws CatalogException { .setSamples(Arrays.asList( new SampleReferenceParam().setId(sampleId3.getId()), new SampleReferenceParam().setId(sampleId3.getId()))), - options, token); + options, ownerToken); assertEquals(1, result.getNumUpdated()); - myModifiedCohort = catalogManager.getCohortManager().get(studyFqn, "myModifiedCohort", QueryOptions.empty(), token).first(); + myModifiedCohort = catalogManager.getCohortManager().get(studyFqn, "myModifiedCohort", QueryOptions.empty(), ownerToken).first(); assertEquals(1, myModifiedCohort.getSamples().size()); assertEquals(1, myModifiedCohort.getNumSamples()); assertTrue(myModifiedCohort.getSamples().stream().map(Sample::getUid).collect(Collectors.toList()).contains(sampleId1.getUid())); @@ -1810,14 +1613,12 @@ public void testUpdateCohort() throws CatalogException { @Test public void testDeleteCohort() throws CatalogException { - String studyId = "user@1000G:phase1"; - - Sample sampleId1 = catalogManager.getSampleManager().create(studyId, new Sample().setId("SAMPLE_1"), INCLUDE_RESULT, token).first(); - Sample sampleId2 = catalogManager.getSampleManager().create(studyId, new Sample().setId("SAMPLE_2"), INCLUDE_RESULT, token).first(); - Sample sampleId3 = catalogManager.getSampleManager().create(studyId, new Sample().setId("SAMPLE_3"), INCLUDE_RESULT, token).first(); + Sample sampleId1 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_1"), INCLUDE_RESULT, ownerToken).first(); + Sample sampleId2 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_2"), INCLUDE_RESULT, ownerToken).first(); + Sample sampleId3 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_3"), INCLUDE_RESULT, ownerToken).first(); - Cohort myCohort = catalogManager.getCohortManager().create(studyId, new Cohort().setId("MyCohort").setType(Enums.CohortType.FAMILY) - .setSamples(Arrays.asList(sampleId1, sampleId2, sampleId3)), INCLUDE_RESULT, token).first(); + Cohort myCohort = catalogManager.getCohortManager().create(studyFqn, new Cohort().setId("MyCohort").setType(Enums.CohortType.FAMILY) + .setSamples(Arrays.asList(sampleId1, sampleId2, sampleId3)), INCLUDE_RESULT, ownerToken).first(); assertEquals("MyCohort", myCohort.getId()); assertEquals(3, myCohort.getSamples().size()); @@ -1825,39 +1626,37 @@ public void testDeleteCohort() throws CatalogException { assertTrue(myCohort.getSamples().stream().map(Sample::getUid).collect(Collectors.toList()).contains(sampleId2.getUid())); assertTrue(myCohort.getSamples().stream().map(Sample::getUid).collect(Collectors.toList()).contains(sampleId3.getUid())); - DataResult deleteResult = catalogManager.getCohortManager().delete(studyId, - new Query(CohortDBAdaptor.QueryParams.UID.key(), myCohort.getUid()), null, token); + DataResult deleteResult = catalogManager.getCohortManager().delete(studyFqn, + new Query(CohortDBAdaptor.QueryParams.UID.key(), myCohort.getUid()), null, ownerToken); assertEquals(1, deleteResult.getNumDeleted()); Query query = new Query() .append(CohortDBAdaptor.QueryParams.UID.key(), myCohort.getUid()) .append(CohortDBAdaptor.QueryParams.DELETED.key(), true); - Cohort cohort = catalogManager.getCohortManager().search(studyId, query, null, token).first(); + Cohort cohort = catalogManager.getCohortManager().search(studyFqn, query, null, ownerToken).first(); assertEquals(InternalStatus.DELETED, cohort.getInternal().getStatus().getId()); } @Test - public void getSamplesFromCohort() throws CatalogException, IOException { - String studyId = "user@1000G:phase1"; - - Sample sampleId1 = catalogManager.getSampleManager().create(studyId, new Sample().setId("SAMPLE_1"), INCLUDE_RESULT, - token).first(); - Sample sampleId2 = catalogManager.getSampleManager().create(studyId, new Sample().setId("SAMPLE_2"), INCLUDE_RESULT, - token).first(); - Sample sampleId3 = catalogManager.getSampleManager().create(studyId, new Sample().setId("SAMPLE_3"), INCLUDE_RESULT, - token).first(); + public void getSamplesFromCohort() throws CatalogException { + Sample sampleId1 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_1"), INCLUDE_RESULT, + ownerToken).first(); + Sample sampleId2 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_2"), INCLUDE_RESULT, + ownerToken).first(); + Sample sampleId3 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_3"), INCLUDE_RESULT, + ownerToken).first(); - Cohort myCohort = catalogManager.getCohortManager().create(studyId, new Cohort().setId("MyCohort").setType(Enums.CohortType.FAMILY) - .setSamples(Arrays.asList(sampleId1, sampleId2, sampleId3)), INCLUDE_RESULT, token).first(); + Cohort myCohort = catalogManager.getCohortManager().create(studyFqn, new Cohort().setId("MyCohort").setType(Enums.CohortType.FAMILY) + .setSamples(Arrays.asList(sampleId1, sampleId2, sampleId3)), INCLUDE_RESULT, ownerToken).first(); - DataResult myCohort1 = catalogManager.getCohortManager().getSamples(studyId, "MyCohort", token); + DataResult myCohort1 = catalogManager.getCohortManager().getSamples(studyFqn, "MyCohort", ownerToken); assertEquals(3, myCohort1.getNumResults()); thrown.expect(CatalogParameterException.class); - catalogManager.getCohortManager().getSamples(studyId, "MyCohort,AnotherCohort", token); + catalogManager.getCohortManager().getSamples(studyFqn, "MyCohort,AnotherCohort", ownerToken); thrown.expect(CatalogParameterException.class); - catalogManager.getCohortManager().getSamples(studyId, "MyCohort,MyCohort", token); + catalogManager.getCohortManager().getSamples(studyFqn, "MyCohort,MyCohort", ownerToken); } @Test @@ -1865,48 +1664,48 @@ public void testDeleteVariableSetWithAnnotations() throws CatalogException { VariableSet variableSet = createVariableSetAndAnnotationSets(); thrown.expect(CatalogException.class); thrown.expectMessage("in use"); - catalogManager.getStudyManager().deleteVariableSet(studyFqn, variableSet.getId(), false, token); + catalogManager.getStudyManager().deleteVariableSet(studyFqn, variableSet.getId(), false, ownerToken); } private VariableSet createVariableSetAndAnnotationSets() throws CatalogException { - VariableSet variableSet = catalogManager.getStudyManager().getVariableSet(studyFqn, "vs", null, token).first(); + VariableSet variableSet = catalogManager.getStudyManager().getVariableSet(studyFqn, "vs", null, ownerToken).first(); String individualId1 = catalogManager.getIndividualManager().create(studyFqn, new Individual().setId("INDIVIDUAL_1") .setKaryotypicSex(IndividualProperty.KaryotypicSex.UNKNOWN).setLifeStatus(IndividualProperty.LifeStatus.UNKNOWN), - INCLUDE_RESULT, token).first().getId(); + INCLUDE_RESULT, ownerToken).first().getId(); String individualId2 = catalogManager.getIndividualManager().create(studyFqn, new Individual().setId("INDIVIDUAL_2") .setKaryotypicSex(IndividualProperty.KaryotypicSex.UNKNOWN).setLifeStatus(IndividualProperty.LifeStatus.UNKNOWN), - INCLUDE_RESULT, token).first().getId(); + INCLUDE_RESULT, ownerToken).first().getId(); String individualId3 = catalogManager.getIndividualManager().create(studyFqn, new Individual().setId("INDIVIDUAL_3") .setKaryotypicSex(IndividualProperty.KaryotypicSex.UNKNOWN).setLifeStatus(IndividualProperty.LifeStatus.UNKNOWN), - INCLUDE_RESULT, token).first().getId(); + INCLUDE_RESULT, ownerToken).first().getId(); catalogManager.getIndividualManager().update(studyFqn, individualId1, new IndividualUpdateParams() .setAnnotationSets(Collections.singletonList(new AnnotationSet("annot1", variableSet.getId(), new ObjectMap("NAME", "INDIVIDUAL_1").append("AGE", 5).append("PHEN", "CASE").append("ALIVE", true)))), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); catalogManager.getIndividualManager().update(studyFqn, individualId2, new IndividualUpdateParams() .setAnnotationSets(Collections.singletonList(new AnnotationSet("annot1", variableSet.getId(), new ObjectMap("NAME", "INDIVIDUAL_2").append("AGE", 15).append("PHEN", "CONTROL").append("ALIVE", true)))), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); catalogManager.getIndividualManager().update(studyFqn, individualId3, new IndividualUpdateParams() .setAnnotationSets(Collections.singletonList(new AnnotationSet("annot1", variableSet.getId(), new ObjectMap("NAME", "INDIVIDUAL_3").append("AGE", 25).append("PHEN", "CASE").append("ALIVE", true)))), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); QueryOptions options = new QueryOptions() .append(QueryOptions.LIMIT, 0) .append(QueryOptions.COUNT, true); OpenCGAResult result = catalogManager.getIndividualManager().search(studyFqn, new Query(IndividualDBAdaptor.QueryParams.ANNOTATION.key(), Constants.VARIABLE_SET + "=" + variableSet.getId()), options, - token); + ownerToken); assertEquals(3, result.getNumMatches()); result = catalogManager.getSampleManager().search(studyFqn, new Query(SampleDBAdaptor.QueryParams.ANNOTATION.key(), Constants.VARIABLE_SET + "=" + variableSet.getId()), options, - token); + ownerToken); assertEquals(8, result.getNumMatches()); return variableSet; } @@ -1914,7 +1713,7 @@ private VariableSet createVariableSetAndAnnotationSets() throws CatalogException @Test public void testForceDeleteVariableSetWithAnnotations() throws CatalogException { VariableSet variableSet = createVariableSetAndAnnotationSets(); - catalogManager.getStudyManager().deleteVariableSet(studyFqn, variableSet.getId(), true, token); + catalogManager.getStudyManager().deleteVariableSet(studyFqn, variableSet.getId(), true, ownerToken); QueryOptions options = new QueryOptions() .append(QueryOptions.LIMIT, 0) @@ -1922,7 +1721,7 @@ public void testForceDeleteVariableSetWithAnnotations() throws CatalogException try { catalogManager.getIndividualManager().search(studyFqn, new Query(IndividualDBAdaptor.QueryParams.ANNOTATION.key(), Constants.VARIABLE_SET + "=" + variableSet.getId()), options, - token); + ownerToken); fail("It should fail saying Variable set does not exist"); } catch (CatalogException e) { assertTrue(e.getMessage().contains("not found")); @@ -1931,41 +1730,287 @@ public void testForceDeleteVariableSetWithAnnotations() throws CatalogException try { catalogManager.getSampleManager().search(studyFqn, new Query(SampleDBAdaptor.QueryParams.ANNOTATION.key(), Constants.VARIABLE_SET + "=" + variableSet.getId()), options, - token); + ownerToken); fail("It should fail saying Variable set does not exist"); } catch (CatalogException e) { assertTrue(e.getMessage().contains("not found")); } try { - catalogManager.getStudyManager().getVariableSet(studyFqn, variableSet.getId(), QueryOptions.empty(), token); + catalogManager.getStudyManager().getVariableSet(studyFqn, variableSet.getId(), QueryOptions.empty(), ownerToken); fail("It should fail saying Variable set does not exist"); } catch (CatalogException e) { assertTrue(e.getMessage().contains("not found")); } } - @Test public void generateCohortFromSampleQuery() throws CatalogException, IOException { - String studyId = "user@1000G:phase1"; - - catalogManager.getSampleManager().create(studyId, new Sample().setId("SAMPLE_1"), INCLUDE_RESULT, token); - catalogManager.getSampleManager().create(studyId, new Sample().setId("SAMPLE_2"), INCLUDE_RESULT, token); - catalogManager.getSampleManager().create(studyId, new Sample().setId("SAMPLE_3"), INCLUDE_RESULT, token); + catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_1"), INCLUDE_RESULT, ownerToken); + catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_2"), INCLUDE_RESULT, ownerToken); + catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_3"), INCLUDE_RESULT, ownerToken); Query query = new Query(); - Cohort myCohort = catalogManager.getCohortManager().generate(studyId, query, new Cohort().setId("MyCohort"), INCLUDE_RESULT, token).first(); + Cohort myCohort = catalogManager.getCohortManager().generate(studyFqn, query, new Cohort().setId("MyCohort"), INCLUDE_RESULT, ownerToken).first(); assertEquals(12, myCohort.getSamples().size()); query = new Query(SampleDBAdaptor.QueryParams.ID.key(), "~^SAM"); - myCohort = catalogManager.getCohortManager().generate(studyId, query, new Cohort() + myCohort = catalogManager.getCohortManager().generate(studyFqn, query, new Cohort() .setId("MyCohort2") - .setStatus(new Status("custom", "custom", "description", TimeUtils.getTime())), INCLUDE_RESULT, token).first(); + .setStatus(new Status("custom", "custom", "description", TimeUtils.getTime())), INCLUDE_RESULT, ownerToken).first(); assertEquals(3, myCohort.getSamples().size()); assertNotNull(myCohort.getStatus()); assertEquals("custom", myCohort.getStatus().getName()); assertEquals("description", myCohort.getStatus().getDescription()); } + // Effective permissions testing + @Test + public void getEffectivePermissionsNoAdmins() throws CatalogException { + thrown.expect(CatalogAuthorizationException.class); + thrown.expectMessage("study administrators"); + catalogManager.getAdminManager().getEffectivePermissions(studyFqn, Arrays.asList(s_1Id, s_2Id), Enums.Resource.SAMPLE.name(), + normalToken1); + } + + @Test + public void getEffectivePermissionsMissingEntries() throws CatalogException { + thrown.expect(CatalogParameterException.class); + thrown.expectMessage("entry id list"); + catalogManager.getAdminManager().getEffectivePermissions(studyFqn, Collections.emptyList(), "sampl", ownerToken); + } + + @Test + public void getEffectivePermissionsWrongCategory() throws CatalogException { + thrown.expect(CatalogParameterException.class); + thrown.expectMessage("category"); + catalogManager.getAdminManager().getEffectivePermissions(studyFqn, Collections.singletonList(s_1Id), "sampl", ownerToken); + } + + @Test + public void getEffectivePermissionsWrongPermission() throws CatalogException { + thrown.expect(CatalogParameterException.class); + thrown.expectMessage("permission"); + catalogManager.getAdminManager().getEffectivePermissions(studyFqn, Collections.singletonList(s_1Id), Collections.singletonList("VIE"), + Enums.Resource.SAMPLE.name(), ownerToken); + } + + @Test + public void getEffectivePermissions() throws CatalogException { + OpenCGAResult aclList = catalogManager.getAdminManager().getEffectivePermissions(studyFqn, Collections.singletonList(s_1Id), + Collections.singletonList(SamplePermissions.VIEW.name()), Enums.Resource.SAMPLE.name(), ownerToken); + assertEquals(1, aclList.getNumResults()); + assertEquals(s_1Id, aclList.first().getId()); + assertEquals(1, aclList.first().getPermissions().size()); + assertEquals(SamplePermissions.VIEW.name(), aclList.first().getPermissions().get(0).getId()); + assertEquals(7, aclList.first().getPermissions().get(0).getUserIds().size()); + assertTrue(Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, normalUserId2, normalUserId3) + .containsAll(aclList.first().getPermissions().get(0).getUserIds())); + + aclList = catalogManager.getAdminManager().getEffectivePermissions(studyFqn, Collections.singletonList(s_1Id), + Enums.Resource.SAMPLE.name(), ownerToken); + assertEquals(1, aclList.getNumResults()); + assertEquals(s_1Id, aclList.first().getId()); + assertEquals(8, aclList.first().getPermissions().size()); + for (Acl.Permission permission : aclList.first().getPermissions()) { + if ("NONE".equals(permission.getId())) { + assertEquals(1, permission.getUserIds().size()); + assertEquals(ParamConstants.ANONYMOUS_USER_ID, permission.getUserIds().get(0)); + } else { + if (permission.getId().startsWith("VIEW") || permission.getId().startsWith("WRITE")) { + assertEquals(7, permission.getUserIds().size()); + assertTrue(Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, normalUserId2, normalUserId3) + .containsAll(permission.getUserIds())); + } else { + // DELETE + assertTrue(permission.getId().startsWith("DELETE")); + assertEquals(6, permission.getUserIds().size()); + assertTrue(Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId2, normalUserId3) + .containsAll(permission.getUserIds())); + } + } + } + + generatePermissionScenario(); + aclList = catalogManager.getAdminManager().getEffectivePermissions(studyFqn, Arrays.asList(s_7Id, s_8Id, s_9Id), + Enums.Resource.SAMPLE.name(), ownerToken); + assertEquals(3, aclList.getNumResults()); + assertPermissions(s_7Id, Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, normalUserId2, normalUserId3, "user4", "user5", "user6", "user7"), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, normalUserId2, "user6", "user7"), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, "user5", "user6"), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, "user6"), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, "user5", "user6"), + Collections.singletonList(ParamConstants.ANONYMOUS_USER_ID), aclList.getResults().get(0)); + assertPermissions(s_8Id, Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, normalUserId2, "user4", "user5", "user6", "user7"), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, normalUserId2, "user6", "user7"), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, "user5", "user6"), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, "user6"), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, "user5", "user6"), + Arrays.asList(normalUserId3, ParamConstants.ANONYMOUS_USER_ID), aclList.getResults().get(1)); + assertPermissions(s_9Id, Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, normalUserId2, normalUserId3, "user5", "user6"), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, normalUserId3, "user6"), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, normalUserId2, normalUserId3, "user5", "user6"), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, normalUserId3, "user6"), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, normalUserId2, normalUserId3, "user5", "user6"), + Arrays.asList(ParamConstants.ANONYMOUS_USER_ID, "user4", "user7"), aclList.getResults().get(2)); + + // Now, we grant view_only access to anonymous user + catalogManager.getStudyManager().updateAcl(studyFqn, ParamConstants.ANONYMOUS_USER_ID, new StudyAclParams(null, "view_only"), + ParamUtils.AclAction.SET, ownerToken); + + aclList = catalogManager.getAdminManager().getEffectivePermissions(studyFqn, Arrays.asList(s_7Id, s_8Id, s_9Id), Enums.Resource.SAMPLE.name(), ownerToken); + assertEquals(3, aclList.getNumResults()); + assertPermissions(s_7Id, Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, normalUserId2, normalUserId3, "user4", "user5", "user6", "user7", ParamConstants.ANONYMOUS_USER_ID), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, normalUserId2, "user6", "user7"), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, "user5", "user6", ParamConstants.ANONYMOUS_USER_ID), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, "user6"), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, "user5", "user6", ParamConstants.ANONYMOUS_USER_ID), + Collections.emptyList(), aclList.getResults().get(0)); + assertPermissions(s_8Id, Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, normalUserId2, "user4", "user5", "user6", "user7", ParamConstants.ANONYMOUS_USER_ID), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, normalUserId2, "user6", "user7"), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, "user5", "user6", ParamConstants.ANONYMOUS_USER_ID), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, "user6"), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, "user5", "user6", ParamConstants.ANONYMOUS_USER_ID), + Collections.singletonList(normalUserId3), aclList.getResults().get(1)); + assertPermissions(s_9Id, Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, normalUserId2, normalUserId3, "user5", "user6", ParamConstants.ANONYMOUS_USER_ID), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, normalUserId3, "user6"), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, normalUserId2, normalUserId3, "user5", "user6", ParamConstants.ANONYMOUS_USER_ID), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, normalUserId3, "user6"), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, normalUserId2, normalUserId3, "user5", "user6", ParamConstants.ANONYMOUS_USER_ID), + Arrays.asList("user4", "user7"), aclList.getResults().get(2)); + + catalogManager.getOrganizationManager().updateConfiguration(organizationId, new OrganizationConfiguration().setOptimizations(new Optimizations(true)), null, ownerToken); + aclList = catalogManager.getAdminManager().getEffectivePermissions(studyFqn, Arrays.asList(s_7Id, s_8Id, s_9Id), Enums.Resource.SAMPLE.name(), ownerToken); + assertEquals(3, aclList.getNumResults()); + assertPermissions(s_7Id, Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, normalUserId2, normalUserId3, "user4", "user5", "user6", "user7", ParamConstants.ANONYMOUS_USER_ID), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, normalUserId2, "user6", "user7"), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, "user5", "user6", ParamConstants.ANONYMOUS_USER_ID), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, "user6"), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, "user5", "user6", ParamConstants.ANONYMOUS_USER_ID), + Collections.emptyList(), aclList.getResults().get(0)); + assertPermissions(s_8Id, Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, normalUserId2, "user4", "user5", "user6", "user7", ParamConstants.ANONYMOUS_USER_ID), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, normalUserId2, "user6", "user7"), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, "user5", "user6", ParamConstants.ANONYMOUS_USER_ID), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, "user6"), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, "user5", "user6", ParamConstants.ANONYMOUS_USER_ID), + Collections.singletonList(normalUserId3), aclList.getResults().get(1)); + assertPermissions(s_9Id, Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, normalUserId2, normalUserId3, "user4", "user5", "user6", "user7", ParamConstants.ANONYMOUS_USER_ID), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, normalUserId3, "user6"), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, normalUserId2, normalUserId3, "user4", "user5", "user6", "user7", ParamConstants.ANONYMOUS_USER_ID), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, normalUserId3, "user6"), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1), + Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, normalUserId2, normalUserId3, "user4", "user5", "user6", "user7", ParamConstants.ANONYMOUS_USER_ID), + Collections.emptyList(), aclList.getResults().get(2)); + + } + + private void assertPermissions(String id, List view, List write, List delete, List viewAnnots, + List writeAnnots, List deleteAnnots, List viewVariants, List none, + Acl acl) { + assertEquals(id, acl.getId()); + for (Acl.Permission permission : acl.getPermissions()) { + SamplePermissions samplePermission = SamplePermissions.valueOf(permission.getId()); + switch (samplePermission) { + case NONE: + assertEquals("Sample " + id + ": " + SamplePermissions.NONE + " permission", none.size(), permission.getUserIds().size()); + assertTrue("Sample " + id + ": " + SamplePermissions.NONE + " permission", permission.getUserIds().containsAll(none)); + break; + case VIEW: + assertEquals("Sample " + id + ": " + SamplePermissions.VIEW + " permission", view.size(), permission.getUserIds().size()); + assertTrue("Sample " + id + ": " + SamplePermissions.VIEW + " permission", permission.getUserIds().containsAll(view)); + break; + case WRITE: + assertEquals("Sample " + id + ": " + SamplePermissions.WRITE + " permission", write.size(), permission.getUserIds().size()); + assertTrue("Sample " + id + ": " + SamplePermissions.WRITE + " permission", permission.getUserIds().containsAll(write)); + break; + case DELETE: + assertEquals("Sample " + id + ": " + SamplePermissions.DELETE + " permission", delete.size(), permission.getUserIds().size()); + assertTrue("Sample " + id + ": " + SamplePermissions.DELETE + " permission", permission.getUserIds().containsAll(delete)); + break; + case VIEW_ANNOTATIONS: + assertEquals("Sample " + id + ": " + SamplePermissions.VIEW_ANNOTATIONS + " permission", viewAnnots.size(), permission.getUserIds().size()); + assertTrue("Sample " + id + ": " + SamplePermissions.VIEW_ANNOTATIONS + " permission", permission.getUserIds().containsAll(viewAnnots)); + break; + case WRITE_ANNOTATIONS: + assertEquals("Sample " + id + ": " + SamplePermissions.WRITE_ANNOTATIONS + " permission", writeAnnots.size(), permission.getUserIds().size()); + assertTrue("Sample " + id + ": " + SamplePermissions.WRITE_ANNOTATIONS + " permission", permission.getUserIds().containsAll(writeAnnots)); + break; + case DELETE_ANNOTATIONS: + assertEquals("Sample " + id + ": " + SamplePermissions.DELETE_ANNOTATIONS + " permission", deleteAnnots.size(), permission.getUserIds().size()); + assertTrue("Sample " + id + ": " + SamplePermissions.DELETE_ANNOTATIONS + " permission", permission.getUserIds().containsAll(deleteAnnots)); + break; + case VIEW_VARIANTS: + assertEquals("Sample " + id + ": " + SamplePermissions.VIEW_VARIANTS + " permission", viewVariants.size(), permission.getUserIds().size()); + assertTrue("Sample " + id + ": " + SamplePermissions.VIEW_VARIANTS + " permission", permission.getUserIds().containsAll(viewVariants)); + break; + } + } + } + + private void generatePermissionScenario() throws CatalogException { + /* + Study groups - permissions - users + @members - - user1 + @view - VIEW_ONLY (annots included) - user2, user7 + @write - VIEW_WRITE (annots included) - user3 + Study permissions to users + user4 - NONE + user5 - VIEW (annots included) + user6 - VIEW_WRITE (annots included) + user7 - NONE + + Sample s_7 permissions + ====================== + Group permissions + @view - WRITE + @write - NONE + User permissions + user3, user4 - VIEW + + Sample s_8 permissions + ====================== + Group permissions + @view - WRITE + @write - NONE + User permissions + user4 - VIEW + + Sample s_9 permissions + ====================== + No permissions assigned + */ + catalogManager.getUserManager().create("user4", "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, organizationId, null, opencgaToken); + catalogManager.getUserManager().create("user5", "User2 Name", "mail2@ebi.ac.uk", TestParamConstants.PASSWORD, organizationId, null, opencgaToken); + catalogManager.getUserManager().create("user6", "User3 Name", "user.2@e.mail", TestParamConstants.PASSWORD, organizationId, null, opencgaToken); + catalogManager.getUserManager().create("user7", "User3 Name", "user.2@e.mail", TestParamConstants.PASSWORD, organizationId, null, opencgaToken); + + catalogManager.getStudyManager().createGroup(studyFqn, "@view", Arrays.asList(normalUserId2, "user7"), ownerToken); + catalogManager.getStudyManager().createGroup(studyFqn, "@write", Collections.singletonList(normalUserId3), ownerToken); + + catalogManager.getStudyManager().updateAcl(studyFqn, "@view,user5", new StudyAclParams(null, "view_only"), ParamUtils.AclAction.SET, ownerToken); + catalogManager.getStudyManager().updateAcl(studyFqn, "@write,user6", new StudyAclParams(null, "analyst"), ParamUtils.AclAction.SET, ownerToken); + catalogManager.getStudyManager().updateAcl(studyFqn, "user4,user7", new StudyAclParams(null, null), ParamUtils.AclAction.SET, ownerToken); + + catalogManager.getSampleManager().updateAcl(studyFqn, Arrays.asList(s_7Id, s_8Id), "@view", new SampleAclParams(null, null, null, null, "WRITE"), ParamUtils.AclAction.SET, ownerToken); + catalogManager.getSampleManager().updateAcl(studyFqn, Arrays.asList(s_7Id, s_8Id), "@write", new SampleAclParams(null, null, null, null, null), ParamUtils.AclAction.SET, ownerToken); + catalogManager.getSampleManager().updateAcl(studyFqn, Arrays.asList(s_7Id, s_8Id), "user4", new SampleAclParams(null, null, null, null, "VIEW"), ParamUtils.AclAction.SET, ownerToken); + catalogManager.getSampleManager().updateAcl(studyFqn, Collections.singletonList(s_7Id), normalUserId3, new SampleAclParams(null, null, null, null, "VIEW"), ParamUtils.AclAction.SET, ownerToken); + } } \ No newline at end of file diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java index c5b5fbc49ce..6d42dbc6fbd 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java @@ -18,17 +18,13 @@ import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.StringUtils; -import org.junit.After; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; -import org.junit.rules.ExpectedException; import org.opencb.biodata.models.clinical.*; import org.opencb.biodata.models.clinical.interpretation.ClinicalVariant; import org.opencb.biodata.models.clinical.interpretation.ClinicalVariantEvidence; import org.opencb.biodata.models.clinical.interpretation.InterpretationMethod; -import org.opencb.biodata.models.common.Status; import org.opencb.biodata.models.core.SexOntologyTermAnnotation; import org.opencb.biodata.models.variant.avro.VariantAvro; import org.opencb.biodata.models.variant.avro.VariantType; @@ -36,11 +32,12 @@ import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.commons.test.GenericTest; import org.opencb.opencga.TestParamConstants; import org.opencb.opencga.catalog.db.api.ClinicalAnalysisDBAdaptor; import org.opencb.opencga.catalog.db.api.InterpretationDBAdaptor; +import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.models.ClinicalAnalysisLoadResult; import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.api.ParamConstants; @@ -64,9 +61,7 @@ import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.sample.SamplePermissions; import org.opencb.opencga.core.models.sample.SampleUpdateParams; -import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.models.study.Variable; -import org.opencb.opencga.core.models.study.VariableSet; +import org.opencb.opencga.core.models.study.*; import org.opencb.opencga.core.models.study.configuration.ClinicalConsent; import org.opencb.opencga.core.models.study.configuration.*; import org.opencb.opencga.core.models.user.Account; @@ -75,51 +70,23 @@ import org.opencb.opencga.core.testclassification.duration.MediumTests; import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.*; import java.util.stream.Collectors; import static org.junit.Assert.*; @Category(MediumTests.class) -public class ClinicalAnalysisManagerTest extends GenericTest { +public class ClinicalAnalysisManagerTest extends AbstractManagerTest { - public final static String STUDY = "user@1000G:phase1"; - @Rule - public ExpectedException thrown = ExpectedException.none(); - - @Rule - public CatalogManagerExternalResource catalogManagerResource = new CatalogManagerExternalResource(); - - protected CatalogManager catalogManager; - private String opencgaToken; - protected String sessionIdUser; private FamilyManager familyManager; - private static final QueryOptions INCLUDE_RESULT = new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true); - @Before - public void setUp() throws IOException, CatalogException { - catalogManager = catalogManagerResource.getCatalogManager(); + public void setUp() throws Exception { + super.setUp(); familyManager = catalogManager.getFamilyManager(); - setUpCatalogManager(catalogManager); - } - - public void setUpCatalogManager(CatalogManager catalogManager) throws IOException, CatalogException { - opencgaToken = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); - - catalogManager.getUserManager().create("user", "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, "", null, Account.AccountType.FULL, opencgaToken); - sessionIdUser = catalogManager.getUserManager().login("user", TestParamConstants.PASSWORD).getToken(); - - catalogManager.getUserManager().create("user2", "User Name2", "mail2@ebi.ac.uk", TestParamConstants.PASSWORD, "", null, Account.AccountType.GUEST, - opencgaToken); - - String projectId = catalogManager.getProjectManager().create("1000G", "Project about some genomes", "", "Homo sapiens", - null, "GRCh38", INCLUDE_RESULT, sessionIdUser).first().getId(); - catalogManager.getStudyManager().create(projectId, "phase1", null, "Phase 1", "Done", null, null, null, null, null, sessionIdUser); - } - - @After - public void tearDown() throws Exception { } private Family getDummyFamily() { @@ -132,7 +99,7 @@ private Family getDummyFamily() { Individual mother = new Individual().setId("mother") .setSex(SexOntologyTermAnnotation.initFemale()) .setDisorders(Arrays.asList(new Disorder("dis2", "dis2", "OT", null, "", - null))); + null))); // We create a new father and mother with the same information to mimic the behaviour of the webservices. Otherwise, we would be // ingesting references to exactly the same object and this test would not work exactly the same way. @@ -184,14 +151,13 @@ private Family getDummyFamily() { private DataResult createDummyFamily() throws CatalogException { Family family = getDummyFamily(); - - return familyManager.create(STUDY, family, INCLUDE_RESULT, sessionIdUser); + return familyManager.create(studyFqn, family, INCLUDE_RESULT, ownerToken); } private DataResult createDummyEnvironment(boolean createFamily, boolean createDefaultInterpretation) throws CatalogException { ClinicalAnalysis clinicalAnalysis = new ClinicalAnalysis() - .setStatus(new Status().setId(ClinicalAnalysisStatus.READY_FOR_INTERPRETATION)) + .setStatus(new ClinicalStatus().setId("READY_FOR_INTERPRETATION")) .setId("analysis" + RandomStringUtils.randomAlphanumeric(3)) .setDescription("My description").setType(ClinicalAnalysis.Type.FAMILY) .setProband(new Individual().setId("child1").setSamples(Arrays.asList(new Sample().setId("sample2")))); @@ -202,25 +168,25 @@ private DataResult createDummyEnvironment(boolean createFamily clinicalAnalysis.setFamily(new Family().setId("family") .setMembers(Arrays.asList(new Individual().setId("child1").setSamples(Arrays.asList(new Sample().setId("sample2")))))); - return catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, !createDefaultInterpretation, - INCLUDE_RESULT, sessionIdUser); + return catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, !createDefaultInterpretation, + INCLUDE_RESULT, ownerToken); } private List registerDummyFiles() throws CatalogException { List files = new LinkedList<>(); String vcfFile = getClass().getResource("/biofiles/variant-test-file.vcf.gz").getFile(); - files.add(catalogManager.getFileManager().link(STUDY, new FileLinkParams(vcfFile, "", "", "", null, null, null, null, - null), false, sessionIdUser).first()); + files.add(catalogManager.getFileManager().link(studyFqn, new FileLinkParams(vcfFile, "", "", "", null, null, null, null, + null), false, ownerToken).first()); vcfFile = getClass().getResource("/biofiles/family.vcf").getFile(); - files.add(catalogManager.getFileManager().link(STUDY, new FileLinkParams(vcfFile, "", "", "", null, null, null, null, - null), false, sessionIdUser).first()); + files.add(catalogManager.getFileManager().link(studyFqn, new FileLinkParams(vcfFile, "", "", "", null, null, null, null, + null), false, ownerToken).first()); String bamFile = getClass().getResource("/biofiles/HG00096.chrom20.small.bam").getFile(); - files.add(catalogManager.getFileManager().link(STUDY, new FileLinkParams(bamFile, "", "", "", null, null, null, null, - null), false, sessionIdUser).first()); + files.add(catalogManager.getFileManager().link(studyFqn, new FileLinkParams(bamFile, "", "", "", null, null, null, null, + null), false, ownerToken).first()); bamFile = getClass().getResource("/biofiles/NA19600.chrom20.small.bam").getFile(); - files.add(catalogManager.getFileManager().link(STUDY, new FileLinkParams(bamFile, "", "", "", null, null, null, null, - null), false, sessionIdUser).first()); + files.add(catalogManager.getFileManager().link(studyFqn, new FileLinkParams(bamFile, "", "", "", null, null, null, null, + null), false, ownerToken).first()); return files; } @@ -235,27 +201,23 @@ public void createAndTestStatusIdIsNotNull() throws CatalogException { .setFamily(new Family().setId("family") .setMembers(Collections.singletonList(new Individual().setId("child1").setSamples(Arrays.asList(new Sample().setId("sample2")))))); - OpenCGAResult result = catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, false, - INCLUDE_RESULT, sessionIdUser); - assertEquals("", result.first().getStatus().getId()); + OpenCGAResult result = catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, false, + INCLUDE_RESULT, ownerToken); + assertEquals(ClinicalStatusValue.ClinicalStatusType.NOT_STARTED, result.first().getStatus().getType()); clinicalAnalysis = new ClinicalAnalysis() .setId("analysis" + RandomStringUtils.randomAlphanumeric(3)) - .setStatus(new Status(null, null, null, null)) + .setStatus(new ClinicalStatus()) .setDescription("My description").setType(ClinicalAnalysis.Type.FAMILY) .setProband(new Individual().setId("child1").setSamples(Arrays.asList(new Sample().setId("sample2")))) .setFamily(new Family().setId("family") .setMembers(Collections.singletonList(new Individual().setId("child1").setSamples(Arrays.asList(new Sample().setId("sample2")))))); - result = catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, false, INCLUDE_RESULT, sessionIdUser); - assertEquals("", result.first().getStatus().getId()); + result = catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, false, INCLUDE_RESULT, ownerToken); + assertEquals(ClinicalStatusValue.ClinicalStatusType.NOT_STARTED, result.first().getStatus().getType()); - result = catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysis.getId(), - new ClinicalAnalysisUpdateParams().setStatus(new StatusParam(null)), INCLUDE_RESULT, sessionIdUser); - assertEquals("", result.first().getStatus().getId()); - - result = catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysis.getId(), - new ClinicalAnalysisUpdateParams().setStatus(new StatusParam("")), INCLUDE_RESULT, sessionIdUser); - assertEquals("", result.first().getStatus().getId()); + String clinicalId = clinicalAnalysis.getId(); + assertThrows(CatalogException.class, () -> catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalId, + new ClinicalAnalysisUpdateParams().setStatus(new StatusParam(null)), INCLUDE_RESULT, ownerToken)); } @Test @@ -266,13 +228,13 @@ public void createMultipleCasesSameFamily() throws CatalogException { createDummyEnvironment(false, true).first(); createDummyEnvironment(false, true).first(); - catalogManager.getClinicalAnalysisManager().update(STUDY, case1.getId(), new ClinicalAnalysisUpdateParams().setLocked(true), QueryOptions.empty(), sessionIdUser); + catalogManager.getClinicalAnalysisManager().update(studyFqn, case1.getId(), new ClinicalAnalysisUpdateParams().setLocked(true), QueryOptions.empty(), ownerToken); // Update proband's sample - catalogManager.getSampleManager().update(STUDY, case1.getProband().getSamples().get(0).getId(), - new SampleUpdateParams().setDescription("new description"), QueryOptions.empty(), sessionIdUser); + catalogManager.getSampleManager().update(studyFqn, case1.getProband().getSamples().get(0).getId(), + new SampleUpdateParams().setDescription("new description"), QueryOptions.empty(), ownerToken); - OpenCGAResult search = catalogManager.getClinicalAnalysisManager().search(STUDY, new Query(), new QueryOptions(), - sessionIdUser); + OpenCGAResult search = catalogManager.getClinicalAnalysisManager().search(studyFqn, new Query(), new QueryOptions(), + ownerToken); assertEquals(5, search.getNumResults()); for (ClinicalAnalysis casee : search.getResults()) { if (casee.getId().equals(case1.getId())) { @@ -291,13 +253,13 @@ public void createMultipleCasesSameFamily() throws CatalogException { public void updateClinicalAnalystsTest() throws CatalogException { ClinicalAnalysis case1 = createDummyEnvironment(true, true).first(); - catalogManager.getUserManager().create(new User().setId("u1").setName("u1").setAccount(new Account()), TestParamConstants.PASSWORD, opencgaToken); - catalogManager.getUserManager().create(new User().setId("u2").setName("u2").setAccount(new Account()), TestParamConstants.PASSWORD, opencgaToken); + catalogManager.getUserManager().create(new User().setId("u1").setName("u1").setOrganization(organizationId), TestParamConstants.PASSWORD, opencgaToken); + catalogManager.getUserManager().create(new User().setId("u2").setName("u2").setOrganization(organizationId), TestParamConstants.PASSWORD, opencgaToken); // Add analysts - OpenCGAResult result = catalogManager.getClinicalAnalysisManager().update(STUDY, case1.getId(), + OpenCGAResult result = catalogManager.getClinicalAnalysisManager().update(studyFqn, case1.getId(), new ClinicalAnalysisUpdateParams().setAnalysts( - Arrays.asList(new ClinicalAnalystParam("u1"), new ClinicalAnalystParam("u2"))), INCLUDE_RESULT, sessionIdUser); + Arrays.asList(new ClinicalAnalystParam("u1"), new ClinicalAnalystParam("u2"))), INCLUDE_RESULT, ownerToken); assertEquals(3, result.first().getAnalysts().size()); assertTrue(result.first().getAnalysts().stream().map(ClinicalAnalyst::getId).collect(Collectors.toSet()).containsAll(Arrays.asList("u1", "u2"))); @@ -315,9 +277,9 @@ public void updateClinicalAnalystsTest() throws CatalogException { QueryOptions options = new QueryOptions() .append(Constants.ACTIONS, actionMap) .append(ParamConstants.INCLUDE_RESULT_PARAM, true); - result = catalogManager.getClinicalAnalysisManager().update(STUDY, case1.getId(), + result = catalogManager.getClinicalAnalysisManager().update(studyFqn, case1.getId(), new ClinicalAnalysisUpdateParams().setAnalysts( - Arrays.asList(new ClinicalAnalystParam("u1"), new ClinicalAnalystParam("u2"))), options, sessionIdUser); + Arrays.asList(new ClinicalAnalystParam("u1"), new ClinicalAnalystParam("u2"))), options, ownerToken); assertEquals(1, result.first().getAnalysts().size()); assertTrue(result.first().getAnalysts().stream().map(ClinicalAnalyst::getId).noneMatch(x -> Arrays.asList("u1", "u2").contains(x))); @@ -326,17 +288,17 @@ public void updateClinicalAnalystsTest() throws CatalogException { options = new QueryOptions() .append(Constants.ACTIONS, actionMap) .append(ParamConstants.INCLUDE_RESULT_PARAM, true); - result = catalogManager.getClinicalAnalysisManager().update(STUDY, case1.getId(), + result = catalogManager.getClinicalAnalysisManager().update(studyFqn, case1.getId(), new ClinicalAnalysisUpdateParams().setAnalysts( - Arrays.asList(new ClinicalAnalystParam("u1"), new ClinicalAnalystParam("u2"))), options, sessionIdUser); + Arrays.asList(new ClinicalAnalystParam("u1"), new ClinicalAnalystParam("u2"))), options, ownerToken); assertEquals(2, result.first().getAnalysts().size()); assertTrue(result.first().getAnalysts().stream().map(ClinicalAnalyst::getId).allMatch(x -> Arrays.asList("u1", "u2").contains(x))); thrown.expect(CatalogException.class); thrown.expectMessage("not found"); - catalogManager.getClinicalAnalysisManager().update(STUDY, case1.getId(), + catalogManager.getClinicalAnalysisManager().update(studyFqn, case1.getId(), new ClinicalAnalysisUpdateParams().setAnalysts( - Arrays.asList(new ClinicalAnalystParam("unknown"), new ClinicalAnalystParam("u2"))), options, sessionIdUser); + Arrays.asList(new ClinicalAnalystParam("unknown"), new ClinicalAnalystParam("u2"))), options, ownerToken); } @Test @@ -344,14 +306,14 @@ public void updateClinicalAnalysisRequest() throws CatalogException { ClinicalAnalysis case1 = createDummyEnvironment(true, true).first(); assertTrue(StringUtils.isEmpty(case1.getRequest().getId())); - catalogManager.getUserManager().create(new User().setId("u1").setName("u1").setEmail("mail@mail.com").setAccount(new Account()), + catalogManager.getUserManager().create(new User().setId("u1").setName("u1").setOrganization(organizationId).setEmail("mail@mail.com"), TestParamConstants.PASSWORD, opencgaToken); ClinicalRequest request = new ClinicalRequest("requestId", "bla", null, new ClinicalResponsible().setId("u1"), new HashMap<>()); // Change request - OpenCGAResult result = catalogManager.getClinicalAnalysisManager().update(STUDY, case1.getId(), - new ClinicalAnalysisUpdateParams().setRequest(request), INCLUDE_RESULT, sessionIdUser); + OpenCGAResult result = catalogManager.getClinicalAnalysisManager().update(studyFqn, case1.getId(), + new ClinicalAnalysisUpdateParams().setRequest(request), INCLUDE_RESULT, ownerToken); assertNotNull(result.first().getRequest()); assertTrue(StringUtils.isNotEmpty(result.first().getRequest().getDate())); assertEquals("requestId", result.first().getRequest().getId()); @@ -361,8 +323,8 @@ public void updateClinicalAnalysisRequest() throws CatalogException { // Remove request responsible request.setResponsible(null); - result = catalogManager.getClinicalAnalysisManager().update(STUDY, case1.getId(), - new ClinicalAnalysisUpdateParams().setRequest(request), INCLUDE_RESULT, sessionIdUser); + result = catalogManager.getClinicalAnalysisManager().update(studyFqn, case1.getId(), + new ClinicalAnalysisUpdateParams().setRequest(request), INCLUDE_RESULT, ownerToken); assertNotNull(result.first().getRequest()); assertTrue(StringUtils.isNotEmpty(result.first().getRequest().getDate())); assertEquals("requestId", result.first().getRequest().getId()); @@ -372,23 +334,24 @@ public void updateClinicalAnalysisRequest() throws CatalogException { request.setResponsible(new ClinicalResponsible().setId("unknown")); thrown.expect(CatalogException.class); thrown.expectMessage("not found"); - catalogManager.getClinicalAnalysisManager().update(STUDY, case1.getId(), - new ClinicalAnalysisUpdateParams().setRequest(request), INCLUDE_RESULT, sessionIdUser); + catalogManager.getClinicalAnalysisManager().update(studyFqn, case1.getId(), + new ClinicalAnalysisUpdateParams().setRequest(request), INCLUDE_RESULT, ownerToken); } @Test public void updateClinicalAnalysisResponsible() throws CatalogException { ClinicalAnalysis case1 = createDummyEnvironment(true, true).first(); - assertEquals("user", case1.getResponsible().getId()); + assertEquals(orgOwnerUserId, case1.getResponsible().getId()); - catalogManager.getUserManager().create(new User().setId("u1").setName("u1").setEmail("mail@mail.com").setAccount(new Account()), + catalogManager.getUserManager().create(new User().setId("u1").setName("u1").setEmail("mail@mail.com") + .setOrganization(organizationId), TestParamConstants.PASSWORD, opencgaToken); ClinicalResponsible responsible = new ClinicalResponsible().setId("u1"); // Change responsible - OpenCGAResult result = catalogManager.getClinicalAnalysisManager().update(STUDY, case1.getId(), - new ClinicalAnalysisUpdateParams().setResponsible(responsible), INCLUDE_RESULT, sessionIdUser); + OpenCGAResult result = catalogManager.getClinicalAnalysisManager().update(studyFqn, case1.getId(), + new ClinicalAnalysisUpdateParams().setResponsible(responsible), INCLUDE_RESULT, ownerToken); assertNotNull(result.first().getResponsible()); assertEquals("u1", result.first().getResponsible().getId()); assertEquals("u1", result.first().getResponsible().getName()); @@ -398,8 +361,8 @@ public void updateClinicalAnalysisResponsible() throws CatalogException { responsible.setId("unknown"); thrown.expect(CatalogException.class); thrown.expectMessage("not found"); - catalogManager.getClinicalAnalysisManager().update(STUDY, case1.getId(), - new ClinicalAnalysisUpdateParams().setResponsible(responsible), INCLUDE_RESULT, sessionIdUser); + catalogManager.getClinicalAnalysisManager().update(studyFqn, case1.getId(), + new ClinicalAnalysisUpdateParams().setResponsible(responsible), INCLUDE_RESULT, ownerToken); } @Test @@ -412,39 +375,39 @@ public void updateClinicalAnalysisReport() throws CatalogException { .setComments(Arrays.asList(new ClinicalComment("author", "msg", null, null), new ClinicalComment("author2", "msg", null, null))); // Change report - OpenCGAResult result = catalogManager.getClinicalAnalysisManager().update(STUDY, case1.getId(), - new ClinicalAnalysisUpdateParams().setReport(report), INCLUDE_RESULT, sessionIdUser); + OpenCGAResult result = catalogManager.getClinicalAnalysisManager().update(studyFqn, case1.getId(), + new ClinicalAnalysisUpdateParams().setReport(report), INCLUDE_RESULT, ownerToken); assertNotNull(result.first().getReport()); assertEquals(report.getTitle(), result.first().getReport().getTitle()); assertEquals(2, result.first().getReport().getComments().size()); for (ClinicalComment comment : result.first().getReport().getComments()) { - assertEquals("user", comment.getAuthor()); + assertEquals(orgOwnerUserId, comment.getAuthor()); assertTrue(StringUtils.isNotEmpty(comment.getDate())); } // Add files - catalogManager.getFileManager().create(STUDY, + catalogManager.getFileManager().create(studyFqn, new FileCreateParams() .setContent(RandomStringUtils.randomAlphanumeric(1000)) .setPath("/data/file1.txt") .setType(File.Type.FILE), - true, sessionIdUser); - catalogManager.getFileManager().create(STUDY, + true, ownerToken); + catalogManager.getFileManager().create(studyFqn, new FileCreateParams() .setContent(RandomStringUtils.randomAlphanumeric(1000)) .setPath("/data/file2.txt") .setType(File.Type.FILE), - true, sessionIdUser); + true, ownerToken); List fileList = Arrays.asList(new File().setId("data:file1.txt"), new File().setId("data:file2.txt")); report.setSupportingEvidences(fileList); - result = catalogManager.getClinicalAnalysisManager().update(STUDY, case1.getId(), - new ClinicalAnalysisUpdateParams().setReport(report), INCLUDE_RESULT, sessionIdUser); + result = catalogManager.getClinicalAnalysisManager().update(studyFqn, case1.getId(), + new ClinicalAnalysisUpdateParams().setReport(report), INCLUDE_RESULT, ownerToken); assertNotNull(result.first().getReport()); assertEquals(report.getTitle(), result.first().getReport().getTitle()); assertEquals(2, result.first().getReport().getComments().size()); for (ClinicalComment comment : result.first().getReport().getComments()) { - assertEquals("user", comment.getAuthor()); + assertEquals(orgOwnerUserId, comment.getAuthor()); assertTrue(StringUtils.isNotEmpty(comment.getDate())); } assertEquals(2, result.first().getReport().getSupportingEvidences().size()); @@ -453,13 +416,13 @@ public void updateClinicalAnalysisReport() throws CatalogException { assertNull(result.first().getReport().getFiles()); report.setFiles(fileList); - result = catalogManager.getClinicalAnalysisManager().update(STUDY, case1.getId(), - new ClinicalAnalysisUpdateParams().setReport(report), INCLUDE_RESULT, sessionIdUser); + result = catalogManager.getClinicalAnalysisManager().update(studyFqn, case1.getId(), + new ClinicalAnalysisUpdateParams().setReport(report), INCLUDE_RESULT, ownerToken); assertNotNull(result.first().getReport()); assertEquals(report.getTitle(), result.first().getReport().getTitle()); assertEquals(2, result.first().getReport().getComments().size()); for (ClinicalComment comment : result.first().getReport().getComments()) { - assertEquals("user", comment.getAuthor()); + assertEquals(orgOwnerUserId, comment.getAuthor()); assertTrue(StringUtils.isNotEmpty(comment.getDate())); } assertEquals(2, result.first().getReport().getSupportingEvidences().size()); @@ -473,8 +436,8 @@ public void updateClinicalAnalysisReport() throws CatalogException { report.setFiles(Collections.singletonList(new File().setId("nonexisting.txt"))); thrown.expect(CatalogException.class); thrown.expectMessage("not found"); - catalogManager.getClinicalAnalysisManager().update(STUDY, case1.getId(), - new ClinicalAnalysisUpdateParams().setReport(report), INCLUDE_RESULT, sessionIdUser); + catalogManager.getClinicalAnalysisManager().update(studyFqn, case1.getId(), + new ClinicalAnalysisUpdateParams().setReport(report), INCLUDE_RESULT, ownerToken); } @Test @@ -483,24 +446,24 @@ public void updateClinicalAnalysisReportWithActions() throws CatalogException { assertNull(case1.getReport()); // Add files - catalogManager.getFileManager().create(STUDY, + catalogManager.getFileManager().create(studyFqn, new FileCreateParams() .setContent(RandomStringUtils.randomAlphanumeric(1000)) .setPath("/data/file1.txt") .setType(File.Type.FILE), - true, sessionIdUser); - catalogManager.getFileManager().create(STUDY, + true, ownerToken); + catalogManager.getFileManager().create(studyFqn, new FileCreateParams() .setContent(RandomStringUtils.randomAlphanumeric(1000)) .setPath("/data/file2.txt") .setType(File.Type.FILE), - true, sessionIdUser); - catalogManager.getFileManager().create(STUDY, + true, ownerToken); + catalogManager.getFileManager().create(studyFqn, new FileCreateParams() .setContent(RandomStringUtils.randomAlphanumeric(1000)) .setPath("/data/file3.txt") .setType(File.Type.FILE), - true, sessionIdUser); + true, ownerToken); ClinicalReport report = new ClinicalReport("title", "overview", new ClinicalDiscussion("me", TimeUtils.getTime(), "text"), "logo", "me", "signature", TimeUtils.getTime(), Arrays.asList( @@ -509,8 +472,8 @@ public void updateClinicalAnalysisReportWithActions() throws CatalogException { ), Collections.singletonList(new File().setId("data:file1.txt")), Collections.singletonList(new File().setId("data:file2.txt"))); - OpenCGAResult result = catalogManager.getClinicalAnalysisManager().update(STUDY, case1.getId(), - new ClinicalAnalysisUpdateParams().setReport(report), INCLUDE_RESULT, sessionIdUser); + OpenCGAResult result = catalogManager.getClinicalAnalysisManager().update(studyFqn, case1.getId(), + new ClinicalAnalysisUpdateParams().setReport(report), INCLUDE_RESULT, ownerToken); assertNotNull(result.first().getReport()); assertEquals(report.getTitle(), result.first().getReport().getTitle()); assertEquals(report.getOverview(), result.first().getReport().getOverview()); @@ -539,8 +502,8 @@ public void updateClinicalAnalysisReportWithActions() throws CatalogException { new File().setId("data:file3.txt") )) .setSupportingEvidences(Collections.singletonList(new File().setId("data:file1.txt"))); - ClinicalReport reportResult = catalogManager.getClinicalAnalysisManager().updateReport(STUDY, case1.getId(), reportToUpdate, - options, sessionIdUser).first(); + ClinicalReport reportResult = catalogManager.getClinicalAnalysisManager().updateReport(studyFqn, case1.getId(), reportToUpdate, + options, ownerToken).first(); // Check comments assertEquals(3, reportResult.getComments().size()); assertEquals("comment1", reportResult.getComments().get(0).getMessage()); @@ -573,8 +536,8 @@ public void updateClinicalAnalysisReportWithActions() throws CatalogException { new File().setId("data:file3.txt") )); ClinicalComment pendingComment = reportResult.getComments().get(2); - reportResult = catalogManager.getClinicalAnalysisManager().updateReport(STUDY, case1.getId(), reportToUpdate, - options, sessionIdUser).first(); + reportResult = catalogManager.getClinicalAnalysisManager().updateReport(studyFqn, case1.getId(), reportToUpdate, + options, ownerToken).first(); // Check comments assertEquals(1, reportResult.getComments().size()); assertEquals(pendingComment.getMessage(), reportResult.getComments().get(0).getMessage()); @@ -605,8 +568,8 @@ public void updateClinicalAnalysisReportWithActions() throws CatalogException { .setSupportingEvidences(Collections.singletonList( new File().setId("data:file2.txt") )); - reportResult = catalogManager.getClinicalAnalysisManager().updateReport(STUDY, case1.getId(), reportToUpdate, - options, sessionIdUser).first(); + reportResult = catalogManager.getClinicalAnalysisManager().updateReport(studyFqn, case1.getId(), reportToUpdate, + options, ownerToken).first(); // Check comments assertEquals(1, reportResult.getComments().size()); assertEquals("comment3", reportResult.getComments().get(0).getMessage()); @@ -623,11 +586,37 @@ public void updateClinicalAnalysisReportWithActions() throws CatalogException { } @Test + public void queryByVersionTest() throws CatalogException { + Individual individual = new Individual().setId("child1").setSamples(Arrays.asList(new Sample().setId("sample2"))); + catalogManager.getIndividualManager().create(studyFqn, individual, null, ownerToken); + + ClinicalComment comment = new ClinicalComment(orgOwnerUserId, "my comment", new ArrayList<>(), ""); + ClinicalAnalysis clinicalAnalysis = new ClinicalAnalysis() + .setId("analysis" + RandomStringUtils.randomAlphanumeric(3)) + .setDescription("My description").setType(ClinicalAnalysis.Type.SINGLE) + .setQualityControl(new ClinicalAnalysisQualityControl(ClinicalAnalysisQualityControl.QualityControlSummary.LOW, + Collections.singletonList(comment), Collections.emptyList())) + .setProband(individual); + + catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, true, INCLUDE_RESULT, ownerToken).first(); + catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams().setDescription("blabla"), null, ownerToken); + + ClinicalAnalysis case1 = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), null, ownerToken).first(); + assertEquals(2, case1.getVersion()); + assertEquals("blabla", case1.getDescription()); + + Query query = new Query(ParamConstants.CLINICAL_VERSION_PARAM, 1); + case1 = catalogManager.getClinicalAnalysisManager().get(studyFqn, Collections.singletonList(clinicalAnalysis.getId()), query, null, false, ownerToken).first(); + assertEquals(1, case1.getVersion()); + assertEquals(clinicalAnalysis.getDescription(), case1.getDescription()); + } + + @Test public void createAndUpdateClinicalAnalysisWithQualityControl() throws CatalogException, InterruptedException { Individual individual = new Individual().setId("child1").setSamples(Arrays.asList(new Sample().setId("sample2"))); - catalogManager.getIndividualManager().create(STUDY, individual, null, sessionIdUser); + catalogManager.getIndividualManager().create(studyFqn, individual, null, ownerToken); - ClinicalComment comment = new ClinicalComment("user", "my comment", new ArrayList<>(), ""); + ClinicalComment comment = new ClinicalComment(orgOwnerUserId, "my comment", new ArrayList<>(), ""); ClinicalAnalysis clinicalAnalysis = new ClinicalAnalysis() .setId("analysis" + RandomStringUtils.randomAlphanumeric(3)) .setDescription("My description").setType(ClinicalAnalysis.Type.SINGLE) @@ -635,12 +624,12 @@ public void createAndUpdateClinicalAnalysisWithQualityControl() throws CatalogEx Collections.singletonList(comment), Collections.emptyList())) .setProband(individual); - ClinicalAnalysis ca = catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, true, INCLUDE_RESULT, - sessionIdUser).first(); + ClinicalAnalysis ca = catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, true, INCLUDE_RESULT, + ownerToken).first(); assertEquals(ClinicalAnalysisQualityControl.QualityControlSummary.LOW, ca.getQualityControl().getSummary()); assertEquals("my comment", ca.getQualityControl().getComments().get(0).getMessage()); - assertEquals("user", ca.getQualityControl().getComments().get(0).getAuthor()); + assertEquals(orgOwnerUserId, ca.getQualityControl().getComments().get(0).getAuthor()); assertNotNull(ca.getQualityControl().getComments().get(0).getDate()); String date = ca.getQualityControl().getComments().get(0).getDate(); @@ -652,12 +641,12 @@ public void createAndUpdateClinicalAnalysisWithQualityControl() throws CatalogEx Collections.singletonList("other"), Collections.emptyList()); ClinicalAnalysisUpdateParams updateParams = new ClinicalAnalysisUpdateParams().setQualityControl(qualityControlUpdateParam); - catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysis.getId(), updateParams, null, sessionIdUser); - ca = catalogManager.getClinicalAnalysisManager().get(STUDY, clinicalAnalysis.getId(), null, sessionIdUser).first(); + catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), updateParams, null, ownerToken); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), null, ownerToken).first(); assertEquals(ClinicalAnalysisQualityControl.QualityControlSummary.HIGH, ca.getQualityControl().getSummary()); assertEquals("other", ca.getQualityControl().getComments().get(0).getMessage()); - assertEquals("user", ca.getQualityControl().getComments().get(0).getAuthor()); + assertEquals(orgOwnerUserId, ca.getQualityControl().getComments().get(0).getAuthor()); assertNotNull(ca.getQualityControl().getComments().get(0).getDate()); assertNotEquals(date, ca.getQualityControl().getComments().get(0).getDate()); } @@ -667,29 +656,30 @@ public void automaticallyLockCaseTest() throws CatalogException { Individual individual = new Individual() .setId("proband") .setSamples(Collections.singletonList(new Sample().setId("sample"))); - catalogManager.getIndividualManager().create(STUDY, individual, QueryOptions.empty(), sessionIdUser); + catalogManager.getIndividualManager().create(studyFqn, individual, QueryOptions.empty(), ownerToken); ClinicalAnalysis clinicalAnalysis = new ClinicalAnalysis() .setId("Clinical") .setType(ClinicalAnalysis.Type.SINGLE) .setProband(individual); - OpenCGAResult clinical = catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, - INCLUDE_RESULT, sessionIdUser); - assertTrue(StringUtils.isEmpty(clinical.first().getStatus().getId())); + OpenCGAResult clinical = catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, + INCLUDE_RESULT, ownerToken); + assertEquals(ClinicalStatusValue.ClinicalStatusType.NOT_STARTED, clinical.first().getStatus().getType()); assertFalse(clinical.first().isLocked()); - clinical = catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysis.getId(), - new ClinicalAnalysisUpdateParams().setStatus(new StatusParam("CLOSED")), INCLUDE_RESULT, sessionIdUser); + clinical = catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), + new ClinicalAnalysisUpdateParams().setStatus(new StatusParam("CLOSED")), INCLUDE_RESULT, ownerToken); assertEquals("CLOSED", clinical.first().getStatus().getId()); assertTrue(clinical.first().isLocked()); clinicalAnalysis = new ClinicalAnalysis() .setId("Clinical2") .setType(ClinicalAnalysis.Type.SINGLE) - .setStatus(new ClinicalAnalysisStatus().setId("CLOSED")) + .setStatus(new ClinicalStatus().setId("CLOSED")) .setProband(individual); - clinical = catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, INCLUDE_RESULT, sessionIdUser); + clinical = catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, INCLUDE_RESULT, ownerToken); assertEquals("CLOSED", clinical.first().getStatus().getId()); + assertEquals(ClinicalStatusValue.ClinicalStatusType.CLOSED, clinical.first().getStatus().getType()); assertTrue(clinical.first().isLocked()); } @@ -698,14 +688,14 @@ public void createSingleClinicalAnalysisTestWithoutDisorder() throws CatalogExce Individual individual = new Individual() .setId("proband") .setSamples(Collections.singletonList(new Sample().setId("sample"))); - catalogManager.getIndividualManager().create(STUDY, individual, QueryOptions.empty(), sessionIdUser); + catalogManager.getIndividualManager().create(studyFqn, individual, QueryOptions.empty(), ownerToken); ClinicalAnalysis clinicalAnalysis = new ClinicalAnalysis() .setId("Clinical") .setType(ClinicalAnalysis.Type.SINGLE) .setProband(individual); - OpenCGAResult clinical = catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, - INCLUDE_RESULT, sessionIdUser); + OpenCGAResult clinical = catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, + INCLUDE_RESULT, ownerToken); assertEquals(1, clinical.getNumResults()); assertTrue(StringUtils.isNotEmpty(clinical.first().getDueDate())); } @@ -719,20 +709,20 @@ public void queryClinicalAnalysisByDate() throws CatalogException { .append(ClinicalAnalysisDBAdaptor.QueryParams.DUE_DATE.key(), ">=" + TimeUtils.getTime(TimeUtils.add24HtoDate(TimeUtils.getDate())) ); - OpenCGAResult search = catalogManager.getClinicalAnalysisManager().search(STUDY, query, QueryOptions.empty(), sessionIdUser); + OpenCGAResult search = catalogManager.getClinicalAnalysisManager().search(studyFqn, query, QueryOptions.empty(), ownerToken); assertEquals(1, search.getNumResults()); assertEquals(result.first().getId(), search.first().getId()); String dueDate = TimeUtils.getTime(); ClinicalAnalysisUpdateParams updateParams = new ClinicalAnalysisUpdateParams() .setDueDate(dueDate); - catalogManager.getClinicalAnalysisManager().update(STUDY, result.first().getId(), updateParams, QueryOptions.empty(), sessionIdUser); - search = catalogManager.getClinicalAnalysisManager().search(STUDY, query, QueryOptions.empty(), sessionIdUser); + catalogManager.getClinicalAnalysisManager().update(studyFqn, result.first().getId(), updateParams, QueryOptions.empty(), ownerToken); + search = catalogManager.getClinicalAnalysisManager().search(studyFqn, query, QueryOptions.empty(), ownerToken); assertEquals(0, search.getNumResults()); query.put(ClinicalAnalysisDBAdaptor.QueryParams.DUE_DATE.key(), "<" + TimeUtils.getTime(TimeUtils.add24HtoDate(TimeUtils.getDate()))); - search = catalogManager.getClinicalAnalysisManager().search(STUDY, query, QueryOptions.empty(), sessionIdUser); + search = catalogManager.getClinicalAnalysisManager().search(studyFqn, query, QueryOptions.empty(), ownerToken); assertEquals(1, search.getNumResults()); assertEquals(result.first().getId(), search.first().getId()); assertEquals(dueDate, search.first().getDueDate()); @@ -743,7 +733,7 @@ public void createClinicalWithComments() throws CatalogException { Individual individual = new Individual() .setId("proband") .setSamples(Collections.singletonList(new Sample().setId("sample"))); - catalogManager.getIndividualManager().create(STUDY, individual, QueryOptions.empty(), sessionIdUser); + catalogManager.getIndividualManager().create(studyFqn, individual, QueryOptions.empty(), ownerToken); ClinicalAnalysis clinicalAnalysis = new ClinicalAnalysis() .setId("Clinical") @@ -752,14 +742,14 @@ public void createClinicalWithComments() throws CatalogException { new ClinicalComment("", "My first comment", Arrays.asList("tag1", "tag2"), ""), new ClinicalComment("", "My second comment", Arrays.asList("1tag", "2tag"), ""))) .setProband(individual); - OpenCGAResult clinical = catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, - INCLUDE_RESULT, sessionIdUser); + OpenCGAResult clinical = catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, + INCLUDE_RESULT, ownerToken); assertEquals(1, clinical.getNumResults()); assertEquals(2, clinical.first().getComments().size()); - assertEquals("user", clinical.first().getComments().get(0).getAuthor()); + assertEquals(orgOwnerUserId, clinical.first().getComments().get(0).getAuthor()); assertEquals("My first comment", clinical.first().getComments().get(0).getMessage()); assertEquals(2, clinical.first().getComments().get(0).getTags().size()); - assertEquals("user", clinical.first().getComments().get(1).getAuthor()); + assertEquals(orgOwnerUserId, clinical.first().getComments().get(1).getAuthor()); assertEquals("My second comment", clinical.first().getComments().get(1).getMessage()); assertEquals(2, clinical.first().getComments().get(1).getTags().size()); assertTrue(StringUtils.isNotEmpty(clinical.first().getComments().get(0).getDate())); @@ -777,7 +767,7 @@ public void createClinicalWithMissingSamplesInFamily() throws CatalogException { member.setSamples(Collections.emptyList()); } } - familyManager.create(STUDY, family, QueryOptions.empty(), sessionIdUser); + familyManager.create(studyFqn, family, QueryOptions.empty(), ownerToken); // And only add sample to proband for (Individual member : family.getMembers()) { @@ -791,8 +781,8 @@ public void createClinicalWithMissingSamplesInFamily() throws CatalogException { .setDueDate("20180510100000") .setProband(new Individual().setId("child1")); clinicalAnalysis.setFamily(family); - DataResult clinicalAnalysisDataResult = catalogManager.getClinicalAnalysisManager().create(STUDY, - clinicalAnalysis, INCLUDE_RESULT, sessionIdUser); + DataResult clinicalAnalysisDataResult = catalogManager.getClinicalAnalysisManager().create(studyFqn, + clinicalAnalysis, INCLUDE_RESULT, ownerToken); assertEquals("child1", clinicalAnalysisDataResult.first().getFamily().getMembers().get(0).getId()); assertEquals("father", clinicalAnalysisDataResult.first().getFamily().getMembers().get(1).getId()); @@ -811,7 +801,7 @@ public void updateClinicalComments() throws CatalogException { Individual individual = new Individual() .setId("proband") .setSamples(Collections.singletonList(new Sample().setId("sample"))); - catalogManager.getIndividualManager().create(STUDY, individual, QueryOptions.empty(), sessionIdUser); + catalogManager.getIndividualManager().create(studyFqn, individual, QueryOptions.empty(), ownerToken); ClinicalAnalysis clinicalAnalysis = new ClinicalAnalysis() .setId("Clinical") @@ -819,7 +809,7 @@ public void updateClinicalComments() throws CatalogException { .setComments(Collections.singletonList(new ClinicalComment("", "My first comment", Arrays.asList("tag1", "tag2"), ""))) .setProband(individual); - catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, QueryOptions.empty(), sessionIdUser); + catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, QueryOptions.empty(), ownerToken); List commentParamList = new ArrayList<>(); commentParamList.add(new ClinicalCommentParam("My second comment", Arrays.asList("myTag"))); @@ -828,20 +818,20 @@ public void updateClinicalComments() throws CatalogException { ObjectMap actionMap = new ObjectMap(ClinicalAnalysisDBAdaptor.QueryParams.COMMENTS.key(), ParamUtils.AddRemoveAction.ADD); QueryOptions options = new QueryOptions(Constants.ACTIONS, actionMap); - catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() - .setComments(commentParamList), options, sessionIdUser); + catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() + .setComments(commentParamList), options, ownerToken); - OpenCGAResult clinical = catalogManager.getClinicalAnalysisManager().get(STUDY, clinicalAnalysis.getId(), - QueryOptions.empty(), sessionIdUser); + OpenCGAResult clinical = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), + QueryOptions.empty(), ownerToken); assertEquals(1, clinical.getNumResults()); assertEquals(3, clinical.first().getComments().size()); - assertEquals("user", clinical.first().getComments().get(1).getAuthor()); + assertEquals(orgOwnerUserId, clinical.first().getComments().get(1).getAuthor()); assertEquals("My second comment", clinical.first().getComments().get(1).getMessage()); assertEquals(1, clinical.first().getComments().get(1).getTags().size()); assertEquals("myTag", clinical.first().getComments().get(1).getTags().get(0)); assertTrue(StringUtils.isNotEmpty(clinical.first().getComments().get(1).getDate())); - assertEquals("user", clinical.first().getComments().get(2).getAuthor()); + assertEquals(orgOwnerUserId, clinical.first().getComments().get(2).getAuthor()); assertEquals("My third comment", clinical.first().getComments().get(2).getMessage()); assertEquals(1, clinical.first().getComments().get(2).getTags().size()); assertEquals("myTag2", clinical.first().getComments().get(2).getTags().get(0)); @@ -857,19 +847,19 @@ public void updateClinicalComments() throws CatalogException { actionMap = new ObjectMap(ClinicalAnalysisDBAdaptor.QueryParams.COMMENTS.key(), ParamUtils.AddRemoveReplaceAction.REPLACE); options = new QueryOptions(Constants.ACTIONS, actionMap); - catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() - .setComments(commentParamList), options, sessionIdUser); - clinical = catalogManager.getClinicalAnalysisManager().get(STUDY, clinicalAnalysis.getId(), QueryOptions.empty(), sessionIdUser); + catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() + .setComments(commentParamList), options, ownerToken); + clinical = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), ownerToken); assertEquals(1, clinical.getNumResults()); assertEquals(3, clinical.first().getComments().size()); - assertEquals("user", clinical.first().getComments().get(1).getAuthor()); + assertEquals(orgOwnerUserId, clinical.first().getComments().get(1).getAuthor()); assertEquals("My updated second comment", clinical.first().getComments().get(1).getMessage()); assertEquals(2, clinical.first().getComments().get(1).getTags().size()); assertEquals("myTag", clinical.first().getComments().get(1).getTags().get(0)); assertEquals("myOtherTag", clinical.first().getComments().get(1).getTags().get(1)); assertTrue(StringUtils.isNotEmpty(clinical.first().getComments().get(1).getDate())); - assertEquals("user", clinical.first().getComments().get(2).getAuthor()); + assertEquals(orgOwnerUserId, clinical.first().getComments().get(2).getAuthor()); assertEquals("My also updated third comment", clinical.first().getComments().get(2).getMessage()); assertEquals(2, clinical.first().getComments().get(2).getTags().size()); assertEquals("myTag2", clinical.first().getComments().get(2).getTags().get(0)); @@ -884,13 +874,13 @@ public void updateClinicalComments() throws CatalogException { actionMap = new ObjectMap(ClinicalAnalysisDBAdaptor.QueryParams.COMMENTS.key(), ParamUtils.AddRemoveAction.REMOVE); options = new QueryOptions(Constants.ACTIONS, actionMap); - catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() - .setComments(commentParamList), options, sessionIdUser); + catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() + .setComments(commentParamList), options, ownerToken); - clinical = catalogManager.getClinicalAnalysisManager().get(STUDY, clinicalAnalysis.getId(), QueryOptions.empty(), sessionIdUser); + clinical = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), ownerToken); assertEquals(1, clinical.getNumResults()); assertEquals(1, clinical.first().getComments().size()); - assertEquals("user", clinical.first().getComments().get(0).getAuthor()); + assertEquals(orgOwnerUserId, clinical.first().getComments().get(0).getAuthor()); assertEquals("My updated second comment", clinical.first().getComments().get(0).getMessage()); assertEquals(2, clinical.first().getComments().get(0).getTags().size()); assertEquals("myTag", clinical.first().getComments().get(0).getTags().get(0)); @@ -903,10 +893,10 @@ public void updateClinicalComments() throws CatalogException { actionMap = new ObjectMap(ClinicalAnalysisDBAdaptor.QueryParams.COMMENTS.key(), ParamUtils.AddRemoveAction.REMOVE); options = new QueryOptions(Constants.ACTIONS, actionMap); - catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() - .setComments(commentParamList), options, sessionIdUser); + catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() + .setComments(commentParamList), options, ownerToken); - clinical = catalogManager.getClinicalAnalysisManager().get(STUDY, clinicalAnalysis.getId(), QueryOptions.empty(), sessionIdUser); + clinical = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), ownerToken); assertEquals(1, clinical.getNumResults()); assertEquals(0, clinical.first().getComments().size()); @@ -916,8 +906,8 @@ public void updateClinicalComments() throws CatalogException { options = new QueryOptions(Constants.ACTIONS, actionMap); try { - catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() - .setComments(commentParamList), options, sessionIdUser); + catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() + .setComments(commentParamList), options, ownerToken); fail("It should fail because the comment has no date"); } catch (CatalogException e) { assertTrue(e.getMessage().contains("date")); @@ -930,8 +920,8 @@ public void updateClinicalComments() throws CatalogException { thrown.expect(CatalogException.class); thrown.expectMessage("date"); - catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() - .setComments(commentParamList), options, sessionIdUser); + catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() + .setComments(commentParamList), options, ownerToken); } @Test @@ -939,7 +929,7 @@ public void updateClinicalAnalysis() throws CatalogException { Individual individual = new Individual() .setId("proband") .setSamples(Collections.singletonList(new Sample().setId("sample"))); - catalogManager.getIndividualManager().create(STUDY, individual, QueryOptions.empty(), sessionIdUser); + catalogManager.getIndividualManager().create(studyFqn, individual, QueryOptions.empty(), ownerToken); ClinicalAnalysis clinicalAnalysis = new ClinicalAnalysis() .setId("Clinical") @@ -947,7 +937,7 @@ public void updateClinicalAnalysis() throws CatalogException { .setComments(Collections.singletonList(new ClinicalComment("", "My first comment", Arrays.asList("tag1", "tag2"), ""))) .setProband(individual); - ClinicalAnalysis clinical = catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, INCLUDE_RESULT, sessionIdUser).first(); + ClinicalAnalysis clinical = catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, INCLUDE_RESULT, ownerToken).first(); assertTrue(clinical.getAttributes().isEmpty()); Map attributes = new HashMap<>(); @@ -956,19 +946,70 @@ public void updateClinicalAnalysis() throws CatalogException { ClinicalAnalysisUpdateParams updateParams = new ClinicalAnalysisUpdateParams() .setAttributes(attributes); - clinical = catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysis.getId(), updateParams, INCLUDE_RESULT, sessionIdUser).first(); + clinical = catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), updateParams, INCLUDE_RESULT, ownerToken).first(); assertFalse(clinical.getAttributes().isEmpty()); assertEquals(2, clinical.getAttributes().size()); assertEquals(attributes.get("a"), clinical.getAttributes().get("a")); assertEquals(attributes.get("b"), clinical.getAttributes().get("b")); } + @Test + public void versioningTest() throws CatalogException { + Individual individual = new Individual() + .setId("proband") + .setSamples(Collections.singletonList(new Sample().setId("sample"))); + catalogManager.getIndividualManager().create(studyFqn, individual, QueryOptions.empty(), ownerToken); + + List findingList = new ArrayList<>(); + VariantAvro variantAvro = new VariantAvro("id1", null, "chr2", 1, 2, "", "", "+", null, 1, null, null, null); + ClinicalVariantEvidence evidence = new ClinicalVariantEvidence().setInterpretationMethodName("method"); + ClinicalVariant cv1 = new ClinicalVariant(variantAvro, Collections.singletonList(evidence), null, null, new ClinicalDiscussion(), + ClinicalVariant.Status.NOT_REVIEWED, Collections.emptyList(), null); + findingList.add(cv1); + variantAvro = new VariantAvro("id2", null, "chr2", 1, 2, "", "", "+", null, 1, null, null, null); + ClinicalVariant cv2 = new ClinicalVariant(variantAvro, Collections.singletonList(evidence), null, null, new ClinicalDiscussion(), + ClinicalVariant.Status.NOT_REVIEWED, Collections.emptyList(), null); + findingList.add(cv2); + + ClinicalAnalysis clinicalAnalysis = new ClinicalAnalysis() + .setId("Clinical") + .setType(ClinicalAnalysis.Type.SINGLE) + .setProband(individual) + .setInterpretation(new Interpretation() + .setPrimaryFindings(findingList) + ); + ClinicalAnalysis clinical = catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, INCLUDE_RESULT, ownerToken).first(); + assertEquals(1, clinical.getVersion()); + assertEquals(1, clinical.getInterpretation().getVersion()); + + // Update clinical analysis + clinical = catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() + .setDescription("my new description"), INCLUDE_RESULT, ownerToken).first(); + assertEquals("my new description", clinical.getDescription()); + assertEquals(2, clinical.getVersion()); + assertEquals(1, clinical.getInterpretation().getVersion()); + + // Update interpretation + Interpretation interpretation = catalogManager.getInterpretationManager().update(studyFqn, clinicalAnalysis.getId(), + clinical.getInterpretation().getId(), new InterpretationUpdateParams().setDescription("my new interpretation description"), + null, INCLUDE_RESULT, ownerToken).first(); + assertEquals("my new interpretation description", interpretation.getDescription()); + assertEquals(2, interpretation.getVersion()); + + // Get clinical Analysis + clinical = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), ownerToken).first(); + assertEquals("my new description", clinical.getDescription()); + assertEquals(3, clinical.getVersion()); + assertEquals("my new interpretation description", clinical.getInterpretation().getDescription()); + assertEquals(2, clinical.getInterpretation().getVersion()); + } + @Test public void createRepeatedInterpretationPrimaryFindings() throws CatalogException { Individual individual = new Individual() .setId("proband") .setSamples(Collections.singletonList(new Sample().setId("sample"))); - catalogManager.getIndividualManager().create(STUDY, individual, QueryOptions.empty(), sessionIdUser); + catalogManager.getIndividualManager().create(studyFqn, individual, QueryOptions.empty(), ownerToken); List findingList = new ArrayList<>(); VariantAvro variantAvro = new VariantAvro("id1", null, "chr2", 1, 2, "", "", "+", null, 1, null, null, null); @@ -991,7 +1032,7 @@ public void createRepeatedInterpretationPrimaryFindings() throws CatalogExceptio ); thrown.expect(CatalogException.class); thrown.expectMessage("repeated"); - catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, QueryOptions.empty(), sessionIdUser); + catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, QueryOptions.empty(), ownerToken); } @Test @@ -999,7 +1040,7 @@ public void createRepeatedInterpretationSecondaryFindings() throws CatalogExcept Individual individual = new Individual() .setId("proband") .setSamples(Collections.singletonList(new Sample().setId("sample"))); - catalogManager.getIndividualManager().create(STUDY, individual, QueryOptions.empty(), sessionIdUser); + catalogManager.getIndividualManager().create(studyFqn, individual, QueryOptions.empty(), ownerToken); List findingList = new ArrayList<>(); VariantAvro variantAvro = new VariantAvro("id1", null, "chr2", 1, 2, "", "", "+", null, 1, null, null, null); @@ -1022,7 +1063,7 @@ public void createRepeatedInterpretationSecondaryFindings() throws CatalogExcept ); thrown.expect(CatalogException.class); thrown.expectMessage("repeated"); - catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, QueryOptions.empty(), sessionIdUser); + catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, QueryOptions.empty(), ownerToken); } @Test @@ -1030,7 +1071,7 @@ public void updatePrimaryFindings() throws CatalogException { Individual individual = new Individual() .setId("proband") .setSamples(Collections.singletonList(new Sample().setId("sample"))); - catalogManager.getIndividualManager().create(STUDY, individual, QueryOptions.empty(), sessionIdUser); + catalogManager.getIndividualManager().create(studyFqn, individual, QueryOptions.empty(), ownerToken); List findingList = new ArrayList<>(); VariantAvro variantAvro = new VariantAvro("id1", null, "chr2", 1, 2, "", "", "+", null, 1, null, null, null); @@ -1050,10 +1091,10 @@ public void updatePrimaryFindings() throws CatalogException { .setInterpretation(new Interpretation() .setPrimaryFindings(findingList) ); - catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, QueryOptions.empty(), sessionIdUser); + catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, QueryOptions.empty(), ownerToken); - Interpretation interpretation = catalogManager.getInterpretationManager().get(STUDY, clinicalAnalysis.getId() + ".1", QueryOptions.empty(), - sessionIdUser).first(); + Interpretation interpretation = catalogManager.getInterpretationManager().get(studyFqn, clinicalAnalysis.getId() + ".1", QueryOptions.empty(), + ownerToken).first(); assertEquals(2, interpretation.getPrimaryFindings().size()); assertEquals(2, interpretation.getStats().getPrimaryFindings().getNumVariants()); assertEquals(2, (int) interpretation.getStats().getPrimaryFindings().getStatusCount().get(ClinicalVariant.Status.NOT_REVIEWED)); @@ -1071,10 +1112,10 @@ public void updatePrimaryFindings() throws CatalogException { ObjectMap actionMap = new ObjectMap(InterpretationDBAdaptor.QueryParams.PRIMARY_FINDINGS.key(), ParamUtils.UpdateAction.ADD); QueryOptions options = new QueryOptions(Constants.ACTIONS, actionMap); - catalogManager.getInterpretationManager().update(STUDY, clinicalAnalysis.getId(), clinicalAnalysis.getId() + ".1", updateParams, null, options, - sessionIdUser); + catalogManager.getInterpretationManager().update(studyFqn, clinicalAnalysis.getId(), clinicalAnalysis.getId() + ".1", updateParams, null, options, + ownerToken); interpretation = - catalogManager.getInterpretationManager().get(STUDY, clinicalAnalysis.getId() + ".1", QueryOptions.empty(), sessionIdUser).first(); + catalogManager.getInterpretationManager().get(studyFqn, clinicalAnalysis.getId() + ".1", QueryOptions.empty(), ownerToken).first(); assertEquals(3, interpretation.getPrimaryFindings().size()); assertEquals("method2", interpretation.getPrimaryFindings().get(2).getEvidences().get(0).getInterpretationMethodName()); assertEquals("id3", interpretation.getPrimaryFindings().get(2).getId()); @@ -1084,8 +1125,8 @@ public void updatePrimaryFindings() throws CatalogException { // Add existing finding cv3.setDiscussion(new ClinicalDiscussion("author", "20220728", "My discussion")); try { - catalogManager.getInterpretationManager().update(STUDY, clinicalAnalysis.getId(), clinicalAnalysis.getId() + ".1", updateParams, null, - options, sessionIdUser); + catalogManager.getInterpretationManager().update(studyFqn, clinicalAnalysis.getId(), clinicalAnalysis.getId() + ".1", updateParams, null, + options, ownerToken); fail("It should not allow adding an already existing finding"); } catch (CatalogException e) { assertTrue(e.getMessage().contains("repeated")); @@ -1097,10 +1138,10 @@ public void updatePrimaryFindings() throws CatalogException { actionMap = new ObjectMap(InterpretationDBAdaptor.QueryParams.PRIMARY_FINDINGS.key(), ParamUtils.UpdateAction.REMOVE); options = new QueryOptions(Constants.ACTIONS, actionMap); - catalogManager.getInterpretationManager().update(STUDY, clinicalAnalysis.getId(), clinicalAnalysis.getId() + ".1", updateParams, null, options, - sessionIdUser); + catalogManager.getInterpretationManager().update(studyFqn, clinicalAnalysis.getId(), clinicalAnalysis.getId() + ".1", updateParams, null, options, + ownerToken); interpretation = - catalogManager.getInterpretationManager().get(STUDY, clinicalAnalysis.getId() + ".1", QueryOptions.empty(), sessionIdUser).first(); + catalogManager.getInterpretationManager().get(studyFqn, clinicalAnalysis.getId() + ".1", QueryOptions.empty(), ownerToken).first(); assertEquals(1, interpretation.getPrimaryFindings().size()); assertEquals("method", interpretation.getPrimaryFindings().get(0).getEvidences().get(0).getInterpretationMethodName()); assertEquals("id2", interpretation.getPrimaryFindings().get(0).getId()); @@ -1113,10 +1154,10 @@ public void updatePrimaryFindings() throws CatalogException { actionMap = new ObjectMap(InterpretationDBAdaptor.QueryParams.PRIMARY_FINDINGS.key(), ParamUtils.UpdateAction.SET); options = new QueryOptions(Constants.ACTIONS, actionMap); - catalogManager.getInterpretationManager().update(STUDY, clinicalAnalysis.getId(), clinicalAnalysis.getId() + ".1", updateParams, null, options, - sessionIdUser); + catalogManager.getInterpretationManager().update(studyFqn, clinicalAnalysis.getId(), clinicalAnalysis.getId() + ".1", updateParams, null, options, + ownerToken); interpretation = - catalogManager.getInterpretationManager().get(STUDY, clinicalAnalysis.getId() + ".1", QueryOptions.empty(), sessionIdUser).first(); + catalogManager.getInterpretationManager().get(studyFqn, clinicalAnalysis.getId() + ".1", QueryOptions.empty(), ownerToken).first(); assertEquals(3, interpretation.getPrimaryFindings().size()); assertEquals("method", interpretation.getPrimaryFindings().get(0).getEvidences().get(0).getInterpretationMethodName()); assertEquals("id1", interpretation.getPrimaryFindings().get(0).getId()); @@ -1140,10 +1181,10 @@ public void updatePrimaryFindings() throws CatalogException { actionMap = new ObjectMap(InterpretationDBAdaptor.QueryParams.PRIMARY_FINDINGS.key(), ParamUtils.UpdateAction.REPLACE); options = new QueryOptions(Constants.ACTIONS, actionMap); - catalogManager.getInterpretationManager().update(STUDY, clinicalAnalysis.getId(), clinicalAnalysis.getId() + ".1", updateParams, null, options, - sessionIdUser); + catalogManager.getInterpretationManager().update(studyFqn, clinicalAnalysis.getId(), clinicalAnalysis.getId() + ".1", updateParams, null, options, + ownerToken); interpretation = - catalogManager.getInterpretationManager().get(STUDY, clinicalAnalysis.getId() + ".1", QueryOptions.empty(), sessionIdUser).first(); + catalogManager.getInterpretationManager().get(studyFqn, clinicalAnalysis.getId() + ".1", QueryOptions.empty(), ownerToken).first(); assertEquals(3, interpretation.getPrimaryFindings().size()); assertEquals("method", interpretation.getPrimaryFindings().get(0).getEvidences().get(0).getInterpretationMethodName()); assertEquals("id1", interpretation.getPrimaryFindings().get(0).getId()); @@ -1167,8 +1208,8 @@ public void updatePrimaryFindings() throws CatalogException { options = new QueryOptions(Constants.ACTIONS, actionMap); try { - catalogManager.getInterpretationManager().update(STUDY, clinicalAnalysis.getId(), clinicalAnalysis.getId() + ".1", updateParams, null, - options, sessionIdUser); + catalogManager.getInterpretationManager().update(studyFqn, clinicalAnalysis.getId(), clinicalAnalysis.getId() + ".1", updateParams, null, + options, ownerToken); fail("It should fail because finding id is missing"); } catch (CatalogException e) { assertTrue(e.getMessage().contains("id")); @@ -1179,8 +1220,8 @@ public void updatePrimaryFindings() throws CatalogException { options = new QueryOptions(Constants.ACTIONS, actionMap); try { - catalogManager.getInterpretationManager().update(STUDY, clinicalAnalysis.getId(), clinicalAnalysis.getId() + ".1", updateParams, null, - options, sessionIdUser); + catalogManager.getInterpretationManager().update(studyFqn, clinicalAnalysis.getId(), clinicalAnalysis.getId() + ".1", updateParams, null, + options, ownerToken); fail("It should fail because finding id is missing"); } catch (CatalogException e) { assertTrue(e.getMessage().contains("id")); @@ -1192,7 +1233,7 @@ public void updateSecondaryFindings() throws CatalogException { Individual individual = new Individual() .setId("proband") .setSamples(Collections.singletonList(new Sample().setId("sample"))); - catalogManager.getIndividualManager().create(STUDY, individual, QueryOptions.empty(), sessionIdUser); + catalogManager.getIndividualManager().create(studyFqn, individual, QueryOptions.empty(), ownerToken); List findingList = new ArrayList<>(); VariantAvro variantAvro = new VariantAvro("id1", null, "chr2", 1, 2, "", "", "+", null, 1, null, null, null); @@ -1212,10 +1253,10 @@ public void updateSecondaryFindings() throws CatalogException { .setInterpretation(new Interpretation() .setSecondaryFindings(findingList) ); - catalogManager.getClinicalAnalysisManager().create(STUDY, ca, QueryOptions.empty(), sessionIdUser); + catalogManager.getClinicalAnalysisManager().create(studyFqn, ca, QueryOptions.empty(), ownerToken); - Interpretation interpretation = catalogManager.getInterpretationManager().get(STUDY, ca.getId() + ".1", QueryOptions.empty(), - sessionIdUser).first(); + Interpretation interpretation = catalogManager.getInterpretationManager().get(studyFqn, ca.getId() + ".1", QueryOptions.empty(), + ownerToken).first(); assertEquals(2, interpretation.getSecondaryFindings().size()); assertNotNull(interpretation.getStats()); assertEquals(2, interpretation.getStats().getSecondaryFindings().getNumVariants()); @@ -1235,10 +1276,10 @@ public void updateSecondaryFindings() throws CatalogException { ObjectMap actionMap = new ObjectMap(InterpretationDBAdaptor.QueryParams.SECONDARY_FINDINGS.key(), ParamUtils.UpdateAction.ADD); QueryOptions options = new QueryOptions(Constants.ACTIONS, actionMap); - catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getId() + ".1", updateParams, null, options, - sessionIdUser); + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getId() + ".1", updateParams, null, options, + ownerToken); interpretation = - catalogManager.getInterpretationManager().get(STUDY, ca.getId() + ".1", QueryOptions.empty(), sessionIdUser).first(); + catalogManager.getInterpretationManager().get(studyFqn, ca.getId() + ".1", QueryOptions.empty(), ownerToken).first(); assertEquals(3, interpretation.getSecondaryFindings().size()); assertEquals("method2", interpretation.getSecondaryFindings().get(2).getEvidences().get(0).getInterpretationMethodName()); assertEquals("id3", interpretation.getSecondaryFindings().get(2).getId()); @@ -1249,8 +1290,8 @@ public void updateSecondaryFindings() throws CatalogException { // Add existing finding try { - catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getId() + ".1", updateParams, null, - options, sessionIdUser); + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getId() + ".1", updateParams, null, + options, ownerToken); fail("It should not allow adding an already existing finding"); } catch (CatalogException e) { assertTrue(e.getMessage().contains("repeated")); @@ -1262,10 +1303,10 @@ public void updateSecondaryFindings() throws CatalogException { actionMap = new ObjectMap(InterpretationDBAdaptor.QueryParams.SECONDARY_FINDINGS.key(), ParamUtils.UpdateAction.REMOVE); options = new QueryOptions(Constants.ACTIONS, actionMap); - catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getId() + ".1", updateParams, null, options, - sessionIdUser); + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getId() + ".1", updateParams, null, options, + ownerToken); interpretation = - catalogManager.getInterpretationManager().get(STUDY, ca.getId() + ".1", QueryOptions.empty(), sessionIdUser).first(); + catalogManager.getInterpretationManager().get(studyFqn, ca.getId() + ".1", QueryOptions.empty(), ownerToken).first(); assertEquals(1, interpretation.getSecondaryFindings().size()); assertEquals("method", interpretation.getSecondaryFindings().get(0).getEvidences().get(0).getInterpretationMethodName()); assertEquals("id2", interpretation.getSecondaryFindings().get(0).getId()); @@ -1280,10 +1321,10 @@ public void updateSecondaryFindings() throws CatalogException { actionMap = new ObjectMap(InterpretationDBAdaptor.QueryParams.SECONDARY_FINDINGS.key(), ParamUtils.UpdateAction.SET); options = new QueryOptions(Constants.ACTIONS, actionMap); - catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getId() + ".1", updateParams, null, options, - sessionIdUser); + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getId() + ".1", updateParams, null, options, + ownerToken); interpretation = - catalogManager.getInterpretationManager().get(STUDY, ca.getId() + ".1", QueryOptions.empty(), sessionIdUser).first(); + catalogManager.getInterpretationManager().get(studyFqn, ca.getId() + ".1", QueryOptions.empty(), ownerToken).first(); assertEquals(3, interpretation.getSecondaryFindings().size()); assertEquals("method", interpretation.getSecondaryFindings().get(0).getEvidences().get(0).getInterpretationMethodName()); assertEquals("id1", interpretation.getSecondaryFindings().get(0).getId()); @@ -1305,10 +1346,10 @@ public void updateSecondaryFindings() throws CatalogException { actionMap = new ObjectMap(InterpretationDBAdaptor.QueryParams.SECONDARY_FINDINGS.key(), ParamUtils.UpdateAction.REPLACE); options = new QueryOptions(Constants.ACTIONS, actionMap); - catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getId() + ".1", updateParams, null, options, - sessionIdUser); + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getId() + ".1", updateParams, null, options, + ownerToken); interpretation = - catalogManager.getInterpretationManager().get(STUDY, ca.getId() + ".1", QueryOptions.empty(), sessionIdUser).first(); + catalogManager.getInterpretationManager().get(studyFqn, ca.getId() + ".1", QueryOptions.empty(), ownerToken).first(); assertEquals(3, interpretation.getSecondaryFindings().size()); assertEquals("method", interpretation.getSecondaryFindings().get(0).getEvidences().get(0).getInterpretationMethodName()); assertEquals("id1", interpretation.getSecondaryFindings().get(0).getId()); @@ -1334,8 +1375,8 @@ public void updateSecondaryFindings() throws CatalogException { options = new QueryOptions(Constants.ACTIONS, actionMap); try { - catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getId() + ".1", updateParams, null, - options, sessionIdUser); + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getId() + ".1", updateParams, null, + options, ownerToken); fail("It should fail because finding id is missing"); } catch (CatalogException e) { assertTrue(e.getMessage().contains("id")); @@ -1346,20 +1387,55 @@ public void updateSecondaryFindings() throws CatalogException { options = new QueryOptions(Constants.ACTIONS, actionMap); try { - catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getId() + ".1", updateParams, null, - options, sessionIdUser); + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getId() + ".1", updateParams, null, + options, ownerToken); fail("It should fail because finding id is missing"); } catch (CatalogException e) { assertTrue(e.getMessage().contains("id")); } } + @Test + public void updateStatusTest() throws CatalogException { + ClinicalAnalysis ca = createDummyEnvironment(true, false).first(); + + Interpretation interpretation = catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation(), + ParamUtils.SaveInterpretationAs.PRIMARY, INCLUDE_RESULT, ownerToken).first(); + + // Create 2 allowed statuses of type CLOSED + ClinicalAnalysisStudyConfiguration studyConfiguration = ClinicalAnalysisStudyConfiguration.defaultConfiguration(); + List statusValueList = new ArrayList<>(); + for (ClinicalStatusValue status : studyConfiguration.getStatus()) { + if (!status.getType().equals(ClinicalStatusValue.ClinicalStatusType.CLOSED)) { + statusValueList.add(status); + } + } + // Add two statuses of type CLOSED + statusValueList.add(new ClinicalStatusValue("closed1", "my desc", ClinicalStatusValue.ClinicalStatusType.CLOSED)); + statusValueList.add(new ClinicalStatusValue("closed2", "my desc", ClinicalStatusValue.ClinicalStatusType.CLOSED)); + studyConfiguration.setStatus(statusValueList); + catalogManager.getClinicalAnalysisManager().configureStudy(studyFqn, studyConfiguration, studyAdminToken1); + + // Update status to one of the new statuses + catalogManager.getClinicalAnalysisManager().update(studyFqn, ca.getId(), + new ClinicalAnalysisUpdateParams().setStatus(new StatusParam("closed1")), QueryOptions.empty(), studyAdminToken1); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, ca.getId(), QueryOptions.empty(), studyAdminToken1).first(); + assertEquals("closed1", ca.getStatus().getId()); + assertEquals(ClinicalStatusValue.ClinicalStatusType.CLOSED, ca.getStatus().getType()); + + // Update status to the other new CLOSED status + catalogManager.getClinicalAnalysisManager().update(studyFqn, ca.getId(), + new ClinicalAnalysisUpdateParams().setStatus(new StatusParam("closed2")), QueryOptions.empty(), studyAdminToken1); + assertEquals("closed1", ca.getStatus().getId()); + assertEquals(ClinicalStatusValue.ClinicalStatusType.CLOSED, ca.getStatus().getType()); + } + @Test public void updateInterpretationComments() throws CatalogException { Individual individual = new Individual() .setId("proband") .setSamples(Collections.singletonList(new Sample().setId("sample"))); - catalogManager.getIndividualManager().create(STUDY, individual, QueryOptions.empty(), sessionIdUser); + catalogManager.getIndividualManager().create(studyFqn, individual, QueryOptions.empty(), ownerToken); ClinicalAnalysis clinicalAnalysis = new ClinicalAnalysis() .setId("Clinical") @@ -1369,7 +1445,7 @@ public void updateInterpretationComments() throws CatalogException { .setComments(Collections.singletonList(new ClinicalComment("", "My first comment", Arrays.asList("tag1", "tag2"), ""))) ); - catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, QueryOptions.empty(), sessionIdUser); + catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, QueryOptions.empty(), ownerToken); List commentParamList = new ArrayList<>(); commentParamList.add(new ClinicalCommentParam("My second comment", Arrays.asList("myTag"))); @@ -1378,20 +1454,20 @@ public void updateInterpretationComments() throws CatalogException { ObjectMap actionMap = new ObjectMap(InterpretationDBAdaptor.QueryParams.COMMENTS.key(), ParamUtils.AddRemoveAction.ADD); QueryOptions options = new QueryOptions(Constants.ACTIONS, actionMap); - catalogManager.getInterpretationManager().update(STUDY, clinicalAnalysis.getId(), clinicalAnalysis.getId() + ".1", new InterpretationUpdateParams() - .setComments(commentParamList), null, options, sessionIdUser); + catalogManager.getInterpretationManager().update(studyFqn, clinicalAnalysis.getId(), clinicalAnalysis.getId() + ".1", new InterpretationUpdateParams() + .setComments(commentParamList), null, options, ownerToken); - OpenCGAResult interpretation = catalogManager.getInterpretationManager().get(STUDY, clinicalAnalysis.getId() + ".1", - QueryOptions.empty(), sessionIdUser); + OpenCGAResult interpretation = catalogManager.getInterpretationManager().get(studyFqn, clinicalAnalysis.getId() + ".1", + QueryOptions.empty(), ownerToken); assertEquals(1, interpretation.getNumResults()); assertEquals(3, interpretation.first().getComments().size()); - assertEquals("user", interpretation.first().getComments().get(1).getAuthor()); + assertEquals(orgOwnerUserId, interpretation.first().getComments().get(1).getAuthor()); assertEquals("My second comment", interpretation.first().getComments().get(1).getMessage()); assertEquals(1, interpretation.first().getComments().get(1).getTags().size()); assertEquals("myTag", interpretation.first().getComments().get(1).getTags().get(0)); assertTrue(StringUtils.isNotEmpty(interpretation.first().getComments().get(1).getDate())); - assertEquals("user", interpretation.first().getComments().get(2).getAuthor()); + assertEquals(orgOwnerUserId, interpretation.first().getComments().get(2).getAuthor()); assertEquals("My third comment", interpretation.first().getComments().get(2).getMessage()); assertEquals(1, interpretation.first().getComments().get(2).getTags().size()); assertEquals("myTag2", interpretation.first().getComments().get(2).getTags().get(0)); @@ -1407,19 +1483,19 @@ public void updateInterpretationComments() throws CatalogException { actionMap = new ObjectMap(InterpretationDBAdaptor.QueryParams.COMMENTS.key(), ParamUtils.AddRemoveReplaceAction.REPLACE); options = new QueryOptions(Constants.ACTIONS, actionMap); - catalogManager.getInterpretationManager().update(STUDY, clinicalAnalysis.getId(), clinicalAnalysis.getId() + ".1", new InterpretationUpdateParams() - .setComments(commentParamList), null, options, sessionIdUser); - interpretation = catalogManager.getInterpretationManager().get(STUDY, clinicalAnalysis.getId() + ".1", QueryOptions.empty(), sessionIdUser); + catalogManager.getInterpretationManager().update(studyFqn, clinicalAnalysis.getId(), clinicalAnalysis.getId() + ".1", new InterpretationUpdateParams() + .setComments(commentParamList), null, options, ownerToken); + interpretation = catalogManager.getInterpretationManager().get(studyFqn, clinicalAnalysis.getId() + ".1", QueryOptions.empty(), ownerToken); assertEquals(1, interpretation.getNumResults()); assertEquals(3, interpretation.first().getComments().size()); - assertEquals("user", interpretation.first().getComments().get(1).getAuthor()); + assertEquals(orgOwnerUserId, interpretation.first().getComments().get(1).getAuthor()); assertEquals("My updated second comment", interpretation.first().getComments().get(1).getMessage()); assertEquals(2, interpretation.first().getComments().get(1).getTags().size()); assertEquals("myTag", interpretation.first().getComments().get(1).getTags().get(0)); assertEquals("myOtherTag", interpretation.first().getComments().get(1).getTags().get(1)); assertTrue(StringUtils.isNotEmpty(interpretation.first().getComments().get(1).getDate())); - assertEquals("user", interpretation.first().getComments().get(2).getAuthor()); + assertEquals(orgOwnerUserId, interpretation.first().getComments().get(2).getAuthor()); assertEquals("My also updated third comment", interpretation.first().getComments().get(2).getMessage()); assertEquals(2, interpretation.first().getComments().get(2).getTags().size()); assertEquals("myTag2", interpretation.first().getComments().get(2).getTags().get(0)); @@ -1434,13 +1510,13 @@ public void updateInterpretationComments() throws CatalogException { actionMap = new ObjectMap(InterpretationDBAdaptor.QueryParams.COMMENTS.key(), ParamUtils.AddRemoveAction.REMOVE); options = new QueryOptions(Constants.ACTIONS, actionMap); - catalogManager.getInterpretationManager().update(STUDY, clinicalAnalysis.getId(), clinicalAnalysis.getId() + ".1", new InterpretationUpdateParams() - .setComments(commentParamList), null, options, sessionIdUser); + catalogManager.getInterpretationManager().update(studyFqn, clinicalAnalysis.getId(), clinicalAnalysis.getId() + ".1", new InterpretationUpdateParams() + .setComments(commentParamList), null, options, ownerToken); - interpretation = catalogManager.getInterpretationManager().get(STUDY, clinicalAnalysis.getId() + ".1", QueryOptions.empty(), sessionIdUser); + interpretation = catalogManager.getInterpretationManager().get(studyFqn, clinicalAnalysis.getId() + ".1", QueryOptions.empty(), ownerToken); assertEquals(1, interpretation.getNumResults()); assertEquals(1, interpretation.first().getComments().size()); - assertEquals("user", interpretation.first().getComments().get(0).getAuthor()); + assertEquals(orgOwnerUserId, interpretation.first().getComments().get(0).getAuthor()); assertEquals("My updated second comment", interpretation.first().getComments().get(0).getMessage()); assertEquals(2, interpretation.first().getComments().get(0).getTags().size()); assertEquals("myTag", interpretation.first().getComments().get(0).getTags().get(0)); @@ -1452,10 +1528,10 @@ public void updateInterpretationComments() throws CatalogException { actionMap = new ObjectMap(InterpretationDBAdaptor.QueryParams.COMMENTS.key(), ParamUtils.AddRemoveAction.REMOVE); options = new QueryOptions(Constants.ACTIONS, actionMap); - catalogManager.getInterpretationManager().update(STUDY, clinicalAnalysis.getId(), clinicalAnalysis.getId() + ".1", new InterpretationUpdateParams() - .setComments(commentParamList), null, options, sessionIdUser); + catalogManager.getInterpretationManager().update(studyFqn, clinicalAnalysis.getId(), clinicalAnalysis.getId() + ".1", new InterpretationUpdateParams() + .setComments(commentParamList), null, options, ownerToken); - interpretation = catalogManager.getInterpretationManager().get(STUDY, clinicalAnalysis.getId() + ".1", QueryOptions.empty(), sessionIdUser); + interpretation = catalogManager.getInterpretationManager().get(studyFqn, clinicalAnalysis.getId() + ".1", QueryOptions.empty(), ownerToken); assertEquals(1, interpretation.getNumResults()); assertEquals(0, interpretation.first().getComments().size()); @@ -1465,9 +1541,9 @@ public void updateInterpretationComments() throws CatalogException { options = new QueryOptions(Constants.ACTIONS, actionMap); try { - catalogManager.getInterpretationManager().update(STUDY, clinicalAnalysis.getId(), clinicalAnalysis.getId() + ".1", + catalogManager.getInterpretationManager().update(studyFqn, clinicalAnalysis.getId(), clinicalAnalysis.getId() + ".1", new InterpretationUpdateParams() - .setComments(commentParamList), null, options, sessionIdUser); + .setComments(commentParamList), null, options, ownerToken); fail("It should fail because the comment has no date"); } catch (CatalogException e) { assertTrue(e.getMessage().contains("date")); @@ -1480,97 +1556,97 @@ public void updateInterpretationComments() throws CatalogException { thrown.expect(CatalogException.class); thrown.expectMessage("date"); - catalogManager.getInterpretationManager().update(STUDY, clinicalAnalysis.getId(), clinicalAnalysis.getId() + ".1", new InterpretationUpdateParams() - .setComments(commentParamList), null, options, sessionIdUser); + catalogManager.getInterpretationManager().update(studyFqn, clinicalAnalysis.getId(), clinicalAnalysis.getId() + ".1", new InterpretationUpdateParams() + .setComments(commentParamList), null, options, ownerToken); } @Test public void assignPermissions() throws CatalogException { ClinicalAnalysis clinicalAnalysis = createDummyEnvironment(true, false).first(); - catalogManager.getUserManager().create("external", "User Name", "external@mail.com", TestParamConstants.PASSWORD, "", null, - Account.AccountType.GUEST, opencgaToken); + catalogManager.getUserManager().create("external", "User Name", "external@mail.com", TestParamConstants.PASSWORD, organizationId, null, + opencgaToken); OpenCGAResult> aclResult = - catalogManager.getClinicalAnalysisManager().getAcls(STUDY, Collections.singletonList(clinicalAnalysis.getId()), "external", - false, sessionIdUser); + catalogManager.getClinicalAnalysisManager().getAcls(studyFqn, Collections.singletonList(clinicalAnalysis.getId()), "external", + false, ownerToken); assertEquals(1, aclResult.getNumResults()); assertEquals(1, aclResult.first().getAcl().size()); assertEquals("external", aclResult.first().getAcl().get(0).getMember()); assertNull(aclResult.first().getAcl().get(0).getPermissions()); - OpenCGAResult> fAclResult = catalogManager.getFamilyManager().getAcls(STUDY, - Collections.singletonList(clinicalAnalysis.getFamily().getId()), "external", false, sessionIdUser); + OpenCGAResult> fAclResult = catalogManager.getFamilyManager().getAcls(studyFqn, + Collections.singletonList(clinicalAnalysis.getFamily().getId()), "external", false, ownerToken); assertEquals(1, fAclResult.getNumResults()); assertEquals(1, fAclResult.first().getAcl().size()); assertEquals("external", fAclResult.first().getAcl().get(0).getMember()); assertNull(fAclResult.first().getAcl().get(0).getPermissions()); - OpenCGAResult> iAclResult = catalogManager.getIndividualManager().getAcls(STUDY, - Collections.singletonList(clinicalAnalysis.getProband().getId()), "external", false, sessionIdUser); + OpenCGAResult> iAclResult = catalogManager.getIndividualManager().getAcls(studyFqn, + Collections.singletonList(clinicalAnalysis.getProband().getId()), "external", false, ownerToken); assertEquals(1, iAclResult.getNumResults()); assertEquals(1, iAclResult.first().getAcl().size()); assertEquals("external", iAclResult.first().getAcl().get(0).getMember()); assertNull(iAclResult.first().getAcl().get(0).getPermissions()); - OpenCGAResult> sAclResult = catalogManager.getSampleManager().getAcls(STUDY, - Collections.singletonList(clinicalAnalysis.getProband().getSamples().get(0).getId()), "external", false, sessionIdUser); + OpenCGAResult> sAclResult = catalogManager.getSampleManager().getAcls(studyFqn, + Collections.singletonList(clinicalAnalysis.getProband().getSamples().get(0).getId()), "external", false, ownerToken); assertEquals(1, sAclResult.getNumResults()); assertEquals(1, sAclResult.first().getAcl().size()); assertEquals("external", sAclResult.first().getAcl().get(0).getMember()); assertNull(sAclResult.first().getAcl().get(0).getPermissions()); // Assign permissions to clinical analysis without propagating the permissions - catalogManager.getClinicalAnalysisManager().updateAcl(STUDY, Collections.singletonList(clinicalAnalysis.getId()), "external", - new AclParams(ClinicalAnalysisPermissions.DELETE.name()), ParamUtils.AclAction.ADD, false, sessionIdUser); + catalogManager.getClinicalAnalysisManager().updateAcl(studyFqn, Collections.singletonList(clinicalAnalysis.getId()), "external", + new AclParams(ClinicalAnalysisPermissions.DELETE.name()), ParamUtils.AclAction.ADD, false, ownerToken); - aclResult = catalogManager.getClinicalAnalysisManager().getAcls(STUDY, - Collections.singletonList(clinicalAnalysis.getId()), "external", false, sessionIdUser); + aclResult = catalogManager.getClinicalAnalysisManager().getAcls(studyFqn, + Collections.singletonList(clinicalAnalysis.getId()), "external", false, ownerToken); assertEquals(1, aclResult.getNumResults()); assertEquals(3, aclResult.first().getAcl().get(0).getPermissions().size()); - fAclResult = catalogManager.getFamilyManager().getAcls(STUDY, - Collections.singletonList(clinicalAnalysis.getFamily().getId()), "external", false, sessionIdUser); + fAclResult = catalogManager.getFamilyManager().getAcls(studyFqn, + Collections.singletonList(clinicalAnalysis.getFamily().getId()), "external", false, ownerToken); assertEquals(1, fAclResult.getNumResults()); assertEquals(1, fAclResult.first().getAcl().size()); assertEquals("external", fAclResult.first().getAcl().get(0).getMember()); assertNull(fAclResult.first().getAcl().get(0).getPermissions()); - iAclResult = catalogManager.getIndividualManager().getAcls(STUDY, - Collections.singletonList(clinicalAnalysis.getProband().getId()), "external", false, sessionIdUser); + iAclResult = catalogManager.getIndividualManager().getAcls(studyFqn, + Collections.singletonList(clinicalAnalysis.getProband().getId()), "external", false, ownerToken); assertEquals(1, iAclResult.getNumResults()); assertEquals(1, iAclResult.first().getAcl().size()); assertEquals("external", iAclResult.first().getAcl().get(0).getMember()); assertNull(iAclResult.first().getAcl().get(0).getPermissions()); - sAclResult = catalogManager.getSampleManager().getAcls(STUDY, - Collections.singletonList(clinicalAnalysis.getProband().getSamples().get(0).getId()), "external", false, sessionIdUser); + sAclResult = catalogManager.getSampleManager().getAcls(studyFqn, + Collections.singletonList(clinicalAnalysis.getProband().getSamples().get(0).getId()), "external", false, ownerToken); assertEquals(1, sAclResult.getNumResults()); assertEquals(1, sAclResult.first().getAcl().size()); assertEquals("external", sAclResult.first().getAcl().get(0).getMember()); assertNull(sAclResult.first().getAcl().get(0).getPermissions()); // Assign permissions to clinical analysis PROPAGATING the permissions - catalogManager.getClinicalAnalysisManager().updateAcl(STUDY, Collections.singletonList(clinicalAnalysis.getId()), "external", + catalogManager.getClinicalAnalysisManager().updateAcl(studyFqn, Collections.singletonList(clinicalAnalysis.getId()), "external", new AclParams(ClinicalAnalysisPermissions.DELETE.name()), ParamUtils.AclAction.ADD, true, - sessionIdUser); + ownerToken); - aclResult = catalogManager.getClinicalAnalysisManager().getAcls(STUDY, - Collections.singletonList(clinicalAnalysis.getId()), "external", false, sessionIdUser); + aclResult = catalogManager.getClinicalAnalysisManager().getAcls(studyFqn, + Collections.singletonList(clinicalAnalysis.getId()), "external", false, ownerToken); assertEquals(1, aclResult.getNumResults()); assertEquals(3, aclResult.first().getAcl().get(0).getPermissions().size()); - fAclResult = catalogManager.getFamilyManager().getAcls(STUDY, - Collections.singletonList(clinicalAnalysis.getFamily().getId()), "external", false, sessionIdUser); + fAclResult = catalogManager.getFamilyManager().getAcls(studyFqn, + Collections.singletonList(clinicalAnalysis.getFamily().getId()), "external", false, ownerToken); assertEquals(1, fAclResult.getNumResults()); assertEquals(2, fAclResult.first().getAcl().get(0).getPermissions().size()); - iAclResult = catalogManager.getIndividualManager().getAcls(STUDY, - Collections.singletonList(clinicalAnalysis.getProband().getId()), "external", false, sessionIdUser); + iAclResult = catalogManager.getIndividualManager().getAcls(studyFqn, + Collections.singletonList(clinicalAnalysis.getProband().getId()), "external", false, ownerToken); assertEquals(1, iAclResult.getNumResults()); assertEquals(2, iAclResult.first().getAcl().get(0).getPermissions().size()); - sAclResult = catalogManager.getSampleManager().getAcls(STUDY, - Collections.singletonList(clinicalAnalysis.getProband().getSamples().get(0).getId()), "external", false, sessionIdUser); + sAclResult = catalogManager.getSampleManager().getAcls(studyFqn, + Collections.singletonList(clinicalAnalysis.getProband().getSamples().get(0).getId()), "external", false, ownerToken); assertEquals(1, sAclResult.getNumResults()); assertEquals(2, sAclResult.first().getAcl().get(0).getPermissions().size()); } @@ -1596,7 +1672,7 @@ public void createClinicalAnalysisTest() throws CatalogException { assertEquals(1, dummyEnvironment.first().getProband().getSamples().size()); assertEquals("sample2", dummyEnvironment.first().getProband().getSamples().get(0).getId()); - assertEquals(catalogManager.getSampleManager().get(STUDY, "sample2", SampleManager.INCLUDE_SAMPLE_IDS, sessionIdUser) + assertEquals(catalogManager.getSampleManager().get(studyFqn, "sample2", SampleManager.INCLUDE_SAMPLE_IDS, ownerToken) .first().getUid(), dummyEnvironment.first().getProband().getSamples().get(0).getUid()); dummyEnvironment = createDummyEnvironment(false, true); @@ -1614,33 +1690,106 @@ public void updateClinicalAnalysisTest() throws CatalogException { .setDescription("My description") .setPriority(new PriorityParam("URGENT")); - OpenCGAResult update = catalogManager.getClinicalAnalysisManager().update(STUDY, dummyEnvironment.first().getId(), - updateParams, QueryOptions.empty(), sessionIdUser); + OpenCGAResult update = catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), + updateParams, QueryOptions.empty(), ownerToken); assertEquals(1, update.getNumUpdated()); - ClinicalAnalysis ca = catalogManager.getClinicalAnalysisManager().get(STUDY, dummyEnvironment.first().getId(), QueryOptions.empty(), - sessionIdUser).first(); + ClinicalAnalysis ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, dummyEnvironment.first().getId(), QueryOptions.empty(), + ownerToken).first(); assertEquals("My description", ca.getDescription()); assertEquals("URGENT", ca.getPriority().getId()); } + @Test + public void adminPermissionTest() throws CatalogException { + // Add ADMIN permissions to the user2 + catalogManager.getStudyManager().updateAcl(studyFqn, normalUserId2, + new StudyAclParams(StudyPermissions.Permissions.ADMIN_CLINICAL_ANALYSIS.name(), null), ParamUtils.AclAction.SET, ownerToken); + + DataResult dummyEnvironment = createDummyEnvironment(true, false); + + // Update ClinicalAnalysis with user1 + ClinicalAnalysis clinicalAnalysis = catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), + new ClinicalAnalysisUpdateParams().setDescription("My description"), INCLUDE_RESULT, normalToken1).first(); + assertEquals("My description", clinicalAnalysis.getDescription()); + + // Update ClinicalAnalysis with user2 + clinicalAnalysis = catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), + new ClinicalAnalysisUpdateParams().setDescription("My description 2"), INCLUDE_RESULT, normalToken2).first(); + assertEquals("My description 2", clinicalAnalysis.getDescription()); + + // Set status to CLOSED with user1 - FAIL + assertThrows(CatalogAuthorizationException.class, () -> + catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), + new ClinicalAnalysisUpdateParams().setStatus(new StatusParam("CLOSED")), INCLUDE_RESULT, normalToken1) + ); + + // Set status to CLOSED with user2 - WORKS + clinicalAnalysis = catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), + new ClinicalAnalysisUpdateParams().setStatus(new StatusParam("CLOSED")), INCLUDE_RESULT, normalToken2).first(); + assertEquals("CLOSED", clinicalAnalysis.getStatus().getId()); + assertTrue(clinicalAnalysis.isLocked()); + + // Unset status from CLOSED to other with user1 - FAIL + assertThrows(CatalogAuthorizationException.class, () -> + catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), + new ClinicalAnalysisUpdateParams().setStatus(new StatusParam("READY_FOR_INTERPRETATION")), INCLUDE_RESULT, normalToken1) + ); + + // Edit CLOSED ClinicalAnalysis from user with ADMIN permission + assertThrows(CatalogException.class, () -> + catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), + new ClinicalAnalysisUpdateParams().setDescription("new description"), INCLUDE_RESULT, normalToken2) + ); + + // Send CLOSED status and edit description from CLOSED ClinicalAnalysis from user with ADMIN permission + assertThrows(CatalogException.class, () -> + catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), + new ClinicalAnalysisUpdateParams().setStatus(new StatusParam("CLOSED")).setDescription("new description"), + INCLUDE_RESULT, normalToken2) + ); + + // Remove CLOSED status and edit description from CLOSED ClinicalAnalysis from user with ADMIN permission + clinicalAnalysis = catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), + new ClinicalAnalysisUpdateParams().setStatus(new StatusParam("READY_FOR_INTERPRETATION")).setDescription("new description"), + INCLUDE_RESULT, normalToken2).first(); + assertEquals("READY_FOR_INTERPRETATION", clinicalAnalysis.getStatus().getId()); + assertEquals("new description", clinicalAnalysis.getDescription()); + + // Set status to CLOSED with study admin - WORKS + clinicalAnalysis = catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), + new ClinicalAnalysisUpdateParams().setStatus(new StatusParam("CLOSED")), INCLUDE_RESULT, studyAdminToken1).first(); + assertEquals("CLOSED", clinicalAnalysis.getStatus().getId()); + + // Unset status from CLOSED to other with study admin - WORKS + clinicalAnalysis = catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), + new ClinicalAnalysisUpdateParams().setStatus(new StatusParam("READY_FOR_INTERPRETATION")).setLocked(false), INCLUDE_RESULT, + studyAdminToken1).first(); + assertEquals("READY_FOR_INTERPRETATION", clinicalAnalysis.getStatus().getId()); + + // Update ClinicalAnalysis with user1 - WORKS + clinicalAnalysis = catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), + new ClinicalAnalysisUpdateParams().setDescription("My description 3"), INCLUDE_RESULT, normalToken1).first(); + assertEquals("My description 3", clinicalAnalysis.getDescription()); + } + @Test public void updateCustomStatusTest() throws CatalogException { - Study study = catalogManager.getStudyManager().get(STUDY, QueryOptions.empty(), sessionIdUser).first(); + Study study = catalogManager.getStudyManager().get(studyFqn, QueryOptions.empty(), ownerToken).first(); ClinicalAnalysisStudyConfiguration configuration = study.getInternal().getConfiguration().getClinical(); DataResult dummyEnvironment = createDummyEnvironment(true, false); - ClinicalStatusValue status = configuration.getStatus().get(dummyEnvironment.first().getType()).get(0); + ClinicalStatusValue status = configuration.getStatus().get(0); ClinicalAnalysisUpdateParams updateParams = new ClinicalAnalysisUpdateParams() .setStatus(new StatusParam(status.getId())); - OpenCGAResult update = catalogManager.getClinicalAnalysisManager().update(STUDY, dummyEnvironment.first().getId(), - updateParams, QueryOptions.empty(), sessionIdUser); + OpenCGAResult update = catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), + updateParams, QueryOptions.empty(), ownerToken); assertEquals(1, update.getNumUpdated()); - ClinicalAnalysis ca = catalogManager.getClinicalAnalysisManager().get(STUDY, dummyEnvironment.first().getId(), QueryOptions.empty(), - sessionIdUser).first(); + ClinicalAnalysis ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, dummyEnvironment.first().getId(), QueryOptions.empty(), + ownerToken).first(); assertEquals(status.getId(), ca.getStatus().getId()); assertEquals(status.getDescription(), ca.getStatus().getDescription()); assertNotNull(ca.getStatus().getDate()); @@ -1648,7 +1797,7 @@ public void updateCustomStatusTest() throws CatalogException { @Test public void updateCustomPriorityTest() throws CatalogException { - Study study = catalogManager.getStudyManager().get(STUDY, QueryOptions.empty(), sessionIdUser).first(); + Study study = catalogManager.getStudyManager().get(studyFqn, QueryOptions.empty(), ownerToken).first(); ClinicalAnalysisStudyConfiguration configuration = study.getInternal().getConfiguration().getClinical(); DataResult dummyEnvironment = createDummyEnvironment(true, false); @@ -1657,12 +1806,12 @@ public void updateCustomPriorityTest() throws CatalogException { ClinicalAnalysisUpdateParams updateParams = new ClinicalAnalysisUpdateParams() .setPriority(new PriorityParam(priority.getId())); - OpenCGAResult update = catalogManager.getClinicalAnalysisManager().update(STUDY, dummyEnvironment.first().getId(), - updateParams, QueryOptions.empty(), sessionIdUser); + OpenCGAResult update = catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), + updateParams, QueryOptions.empty(), ownerToken); assertEquals(1, update.getNumUpdated()); - ClinicalAnalysis ca = catalogManager.getClinicalAnalysisManager().get(STUDY, dummyEnvironment.first().getId(), QueryOptions.empty(), - sessionIdUser).first(); + ClinicalAnalysis ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, dummyEnvironment.first().getId(), QueryOptions.empty(), + ownerToken).first(); assertEquals(priority.getId(), ca.getPriority().getId()); assertEquals(priority.getDescription(), ca.getPriority().getDescription()); assertEquals(priority.getRank(), ca.getPriority().getRank()); @@ -1671,14 +1820,14 @@ public void updateCustomPriorityTest() throws CatalogException { @Test public void updateCustomFlagTest() throws CatalogException { - Study study = catalogManager.getStudyManager().get(STUDY, QueryOptions.empty(), sessionIdUser).first(); + Study study = catalogManager.getStudyManager().get(studyFqn, QueryOptions.empty(), ownerToken).first(); ClinicalAnalysisStudyConfiguration configuration = study.getInternal().getConfiguration().getClinical(); DataResult dummyEnvironment = createDummyEnvironment(true, false); - FlagValue flag1 = configuration.getFlags().get(dummyEnvironment.first().getType()).get(1); - FlagValue flag2 = configuration.getFlags().get(dummyEnvironment.first().getType()).get(3); - FlagValue flag3 = configuration.getFlags().get(dummyEnvironment.first().getType()).get(4); + FlagValue flag1 = configuration.getFlags().get(1); + FlagValue flag2 = configuration.getFlags().get(3); + FlagValue flag3 = configuration.getFlags().get(4); ObjectMap actionMap = new ObjectMap(ClinicalAnalysisDBAdaptor.QueryParams.FLAGS.key(), ParamUtils.BasicUpdateAction.ADD); QueryOptions options = new QueryOptions(Constants.ACTIONS, actionMap); @@ -1686,18 +1835,18 @@ public void updateCustomFlagTest() throws CatalogException { ClinicalAnalysisUpdateParams updateParams = new ClinicalAnalysisUpdateParams() .setFlags(Arrays.asList(new FlagValueParam(flag1.getId()), new FlagValueParam(flag1.getId()), new FlagValueParam(flag2.getId()))); - OpenCGAResult update = catalogManager.getClinicalAnalysisManager().update(STUDY, dummyEnvironment.first().getId(), - updateParams, options, sessionIdUser); + OpenCGAResult update = catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), + updateParams, options, ownerToken); assertEquals(1, update.getNumUpdated()); updateParams = new ClinicalAnalysisUpdateParams() .setFlags(Arrays.asList(new FlagValueParam(flag2.getId()), new FlagValueParam(flag3.getId()))); - update = catalogManager.getClinicalAnalysisManager().update(STUDY, dummyEnvironment.first().getId(), updateParams, options, - sessionIdUser); + update = catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), updateParams, options, + ownerToken); assertEquals(1, update.getNumUpdated()); - ClinicalAnalysis ca = catalogManager.getClinicalAnalysisManager().get(STUDY, dummyEnvironment.first().getId(), QueryOptions.empty(), - sessionIdUser).first(); + ClinicalAnalysis ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, dummyEnvironment.first().getId(), QueryOptions.empty(), + ownerToken).first(); assertEquals(3, ca.getFlags().size()); for (FlagAnnotation flag : ca.getFlags()) { FlagValue flagToCompare = null; @@ -1715,20 +1864,20 @@ public void updateCustomFlagTest() throws CatalogException { } // Set other flags - flag1 = configuration.getFlags().get(dummyEnvironment.first().getType()).get(0); - flag2 = configuration.getFlags().get(dummyEnvironment.first().getType()).get(2); + flag1 = configuration.getFlags().get(0); + flag2 = configuration.getFlags().get(2); actionMap = new ObjectMap(ClinicalAnalysisDBAdaptor.QueryParams.FLAGS.key(), ParamUtils.BasicUpdateAction.SET); options = new QueryOptions(Constants.ACTIONS, actionMap); updateParams = new ClinicalAnalysisUpdateParams() .setFlags(Arrays.asList(new FlagValueParam(flag1.getId()), new FlagValueParam(flag2.getId()))); - update = catalogManager.getClinicalAnalysisManager().update(STUDY, dummyEnvironment.first().getId(), updateParams, options, - sessionIdUser); + update = catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), updateParams, options, + ownerToken); assertEquals(1, update.getNumUpdated()); - ca = catalogManager.getClinicalAnalysisManager().get(STUDY, dummyEnvironment.first().getId(), QueryOptions.empty(), - sessionIdUser).first(); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, dummyEnvironment.first().getId(), QueryOptions.empty(), + ownerToken).first(); assertEquals(2, ca.getFlags().size()); assertEquals(flag1.getId(), ca.getFlags().get(0).getId()); assertEquals(flag1.getDescription(), ca.getFlags().get(0).getDescription()); @@ -1744,12 +1893,12 @@ public void updateCustomFlagTest() throws CatalogException { updateParams = new ClinicalAnalysisUpdateParams() .setFlags(Collections.singletonList(new FlagValueParam(flag1.getId()))); - update = catalogManager.getClinicalAnalysisManager().update(STUDY, dummyEnvironment.first().getId(), updateParams, options, - sessionIdUser); + update = catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), updateParams, options, + ownerToken); assertEquals(1, update.getNumUpdated()); - ca = catalogManager.getClinicalAnalysisManager().get(STUDY, dummyEnvironment.first().getId(), QueryOptions.empty(), - sessionIdUser).first(); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, dummyEnvironment.first().getId(), QueryOptions.empty(), + ownerToken).first(); assertEquals(1, ca.getFlags().size()); assertEquals(flag2.getId(), ca.getFlags().get(0).getId()); assertEquals(flag2.getDescription(), ca.getFlags().get(0).getDescription()); @@ -1758,7 +1907,7 @@ public void updateCustomFlagTest() throws CatalogException { @Test public void updateCustomConsentTest() throws CatalogException { - Study study = catalogManager.getStudyManager().get(STUDY, QueryOptions.empty(), sessionIdUser).first(); + Study study = catalogManager.getStudyManager().get(studyFqn, QueryOptions.empty(), ownerToken).first(); ClinicalAnalysisStudyConfiguration configuration = study.getInternal().getConfiguration().getClinical(); DataResult dummyEnvironment = createDummyEnvironment(true, false); @@ -1772,12 +1921,12 @@ public void updateCustomConsentTest() throws CatalogException { ClinicalAnalysisUpdateParams updateParams = new ClinicalAnalysisUpdateParams() .setConsent(new ClinicalConsentAnnotationParam(Collections.singletonList( new ClinicalConsentAnnotationParam.ClinicalConsentParam(consents.get(1).getId(), ClinicalConsentParam.Value.YES)))); - OpenCGAResult update = catalogManager.getClinicalAnalysisManager().update(STUDY, dummyEnvironment.first().getId(), - updateParams, QueryOptions.empty(), sessionIdUser); + OpenCGAResult update = catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), + updateParams, QueryOptions.empty(), ownerToken); assertEquals(1, update.getNumUpdated()); - ClinicalAnalysis ca = catalogManager.getClinicalAnalysisManager().get(STUDY, dummyEnvironment.first().getId(), QueryOptions.empty(), - sessionIdUser).first(); + ClinicalAnalysis ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, dummyEnvironment.first().getId(), QueryOptions.empty(), + ownerToken).first(); assertEquals(consents.size(), ca.getConsent().getConsents().size()); assertNotNull(ca.getConsent().getDate()); @@ -1795,20 +1944,20 @@ public void updateCustomConsentTest() throws CatalogException { @Test public void updateInterpretationCustomStatusTest() throws CatalogException { - Study study = catalogManager.getStudyManager().get(STUDY, QueryOptions.empty(), sessionIdUser).first(); + Study study = catalogManager.getStudyManager().get(studyFqn, QueryOptions.empty(), ownerToken).first(); InterpretationStudyConfiguration configuration = study.getInternal().getConfiguration().getClinical().getInterpretation(); DataResult dummyEnvironment = createDummyEnvironment(true, true); - ClinicalStatusValue status = configuration.getStatus().get(dummyEnvironment.first().getType()).get(0); + ClinicalStatusValue status = configuration.getStatus().get(1); InterpretationUpdateParams updateParams = new InterpretationUpdateParams() .setStatus(new StatusParam(status.getId())); - OpenCGAResult update = catalogManager.getInterpretationManager().update(STUDY, dummyEnvironment.first().getId(), - dummyEnvironment.first().getInterpretation().getId(), updateParams, null, QueryOptions.empty(), sessionIdUser); + OpenCGAResult update = catalogManager.getInterpretationManager().update(studyFqn, dummyEnvironment.first().getId(), + dummyEnvironment.first().getInterpretation().getId(), updateParams, null, QueryOptions.empty(), ownerToken); assertEquals(1, update.getNumUpdated()); - Interpretation interpretation = catalogManager.getInterpretationManager().get(STUDY, - dummyEnvironment.first().getInterpretation().getId(), QueryOptions.empty(), sessionIdUser).first(); + Interpretation interpretation = catalogManager.getInterpretationManager().get(studyFqn, + dummyEnvironment.first().getInterpretation().getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(status.getId(), interpretation.getStatus().getId()); assertEquals(status.getDescription(), interpretation.getStatus().getDescription()); assertNotNull(interpretation.getStatus().getDate()); @@ -1818,43 +1967,43 @@ public void updateInterpretationCustomStatusTest() throws CatalogException { public void createInterpretationTest() throws CatalogException { ClinicalAnalysis ca = createDummyEnvironment(true, false).first(); - catalogManager.getInterpretationManager().create(STUDY, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.PRIMARY, - QueryOptions.empty(), sessionIdUser); - ca = catalogManager.getClinicalAnalysisManager().get(STUDY, ca.getId(), QueryOptions.empty(), sessionIdUser).first(); + catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.PRIMARY, + QueryOptions.empty(), ownerToken); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, ca.getId(), QueryOptions.empty(), ownerToken).first(); assertNotNull(ca.getInterpretation()); assertEquals(ca.getId() + ".1", ca.getInterpretation().getId()); // Delete old interpretation and create a new primary one - catalogManager.getInterpretationManager().create(STUDY, ca.getId(), new Interpretation(), - ParamUtils.SaveInterpretationAs.PRIMARY, QueryOptions.empty(), sessionIdUser); - catalogManager.getInterpretationManager().delete(STUDY, ca.getId(), Collections.singletonList(ca.getId() + ".1"), sessionIdUser); + catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation(), + ParamUtils.SaveInterpretationAs.PRIMARY, QueryOptions.empty(), ownerToken); + catalogManager.getInterpretationManager().delete(studyFqn, ca.getId(), Collections.singletonList(ca.getId() + ".1"), ownerToken); - ca = catalogManager.getClinicalAnalysisManager().get(STUDY, ca.getId(), QueryOptions.empty(), sessionIdUser).first(); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, ca.getId(), QueryOptions.empty(), ownerToken).first(); assertNotNull(ca.getInterpretation()); assertEquals(ca.getId() + ".2", ca.getInterpretation().getId()); assertEquals(0, ca.getSecondaryInterpretations().size()); - assertEquals(0, catalogManager.getInterpretationManager().search(STUDY, - new Query(InterpretationDBAdaptor.QueryParams.ID.key(), ca.getId() + ".1"), QueryOptions.empty(), sessionIdUser) + assertEquals(0, catalogManager.getInterpretationManager().search(studyFqn, + new Query(InterpretationDBAdaptor.QueryParams.ID.key(), ca.getId() + ".1"), QueryOptions.empty(), ownerToken) .getNumResults()); // Old interpretation was deleted - assertEquals(1, catalogManager.getInterpretationManager().search(STUDY, new Query() + assertEquals(1, catalogManager.getInterpretationManager().search(studyFqn, new Query() .append(InterpretationDBAdaptor.QueryParams.ID.key(), ca.getId() + ".1") - .append(InterpretationDBAdaptor.QueryParams.DELETED.key(), true), QueryOptions.empty(), sessionIdUser) + .append(InterpretationDBAdaptor.QueryParams.DELETED.key(), true), QueryOptions.empty(), ownerToken) .getNumResults()); // Interpretation2 should be moved to secondary interpretations - catalogManager.getInterpretationManager().create(STUDY, ca.getId(), new Interpretation(), - ParamUtils.SaveInterpretationAs.PRIMARY, QueryOptions.empty(), sessionIdUser); - ca = catalogManager.getClinicalAnalysisManager().get(STUDY, ca.getId(), QueryOptions.empty(), sessionIdUser).first(); + catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation(), + ParamUtils.SaveInterpretationAs.PRIMARY, QueryOptions.empty(), ownerToken); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, ca.getId(), QueryOptions.empty(), ownerToken).first(); assertNotNull(ca.getInterpretation()); assertEquals(ca.getId() + ".3", ca.getInterpretation().getId()); assertEquals(1, ca.getSecondaryInterpretations().size()); assertEquals(ca.getId() + ".2", ca.getSecondaryInterpretations().get(0).getId()); // Interpretation4 should be added to secondary interpretations - catalogManager.getInterpretationManager().create(STUDY, ca.getId(), new Interpretation(), - ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), sessionIdUser); - ca = catalogManager.getClinicalAnalysisManager().get(STUDY, ca.getId(), QueryOptions.empty(), sessionIdUser).first(); + catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation(), + ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), ownerToken); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, ca.getId(), QueryOptions.empty(), ownerToken).first(); assertNotNull(ca.getInterpretation()); assertEquals(ca.getId() + ".3", ca.getInterpretation().getId()); assertEquals(2, ca.getSecondaryInterpretations().size()); @@ -1863,7 +2012,7 @@ public void createInterpretationTest() throws CatalogException { thrown.expect(CatalogException.class); thrown.expectMessage("Missing"); - catalogManager.getInterpretationManager().create(STUDY, ca.getId(), new Interpretation(), null, QueryOptions.empty(), sessionIdUser); + catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation(), null, QueryOptions.empty(), ownerToken); } @Test @@ -1878,11 +2027,11 @@ public void clearPrimaryInterpretation() throws CatalogException { .setSecondaryFindings(Collections.singletonList(new ClinicalVariant(new VariantAvro("id", Collections.emptyList(), "chr1" , 1, 2, "ref", "alt", "+", null, 1, null, null, null)))) .setComments(Collections.singletonList(new ClinicalComment("me", "message", null, TimeUtils.getTime()))); - catalogManager.getInterpretationManager().create(STUDY, ca.getId(), interpretation, ParamUtils.SaveInterpretationAs.PRIMARY, - QueryOptions.empty(), sessionIdUser); + catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), interpretation, ParamUtils.SaveInterpretationAs.PRIMARY, + QueryOptions.empty(), ownerToken); - Interpretation interpretationResult = catalogManager.getInterpretationManager().get(STUDY, ca.getId() + ".1", - QueryOptions.empty(), sessionIdUser).first(); + Interpretation interpretationResult = catalogManager.getInterpretationManager().get(studyFqn, ca.getId() + ".1", + QueryOptions.empty(), ownerToken).first(); assertEquals(ca.getId() + ".1", interpretationResult.getId()); assertEquals(1, interpretationResult.getVersion()); assertEquals("description", interpretationResult.getDescription()); @@ -1892,9 +2041,9 @@ public void clearPrimaryInterpretation() throws CatalogException { assertEquals(1, interpretationResult.getSecondaryFindings().size()); assertEquals(1, interpretationResult.getComments().size()); - catalogManager.getInterpretationManager().clear(STUDY, ca.getId(), Collections.singletonList(ca.getId() + ".1"), sessionIdUser); - interpretationResult = catalogManager.getInterpretationManager().get(STUDY, ca.getId() + ".1", QueryOptions.empty(), - sessionIdUser).first(); + catalogManager.getInterpretationManager().clear(studyFqn, ca.getId(), Collections.singletonList(ca.getId() + ".1"), ownerToken); + interpretationResult = catalogManager.getInterpretationManager().get(studyFqn, ca.getId() + ".1", QueryOptions.empty(), + ownerToken).first(); assertEquals(ca.getId() + ".1", interpretationResult.getId()); assertEquals(2, interpretationResult.getVersion()); assertEquals("", interpretationResult.getDescription()); @@ -1917,11 +2066,11 @@ public void clearSecondaryInterpretation() throws CatalogException { .setSecondaryFindings(Collections.singletonList(new ClinicalVariant(new VariantAvro("id", Collections.emptyList(), "chr1" , 1, 2, "ref", "alt", "+", null, 1, null, null, null)))) .setComments(Collections.singletonList(new ClinicalComment("me", "message", null, TimeUtils.getTime()))); - catalogManager.getInterpretationManager().create(STUDY, ca.getId(), interpretation, ParamUtils.SaveInterpretationAs.SECONDARY, - QueryOptions.empty(), sessionIdUser); + catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), interpretation, ParamUtils.SaveInterpretationAs.SECONDARY, + QueryOptions.empty(), ownerToken); - Interpretation interpretationResult = catalogManager.getInterpretationManager().get(STUDY, ca.getId() + ".1", - QueryOptions.empty(), sessionIdUser).first(); + Interpretation interpretationResult = catalogManager.getInterpretationManager().get(studyFqn, ca.getId() + ".1", + QueryOptions.empty(), ownerToken).first(); assertEquals(ca.getId() + ".1", interpretationResult.getId()); assertEquals(1, interpretationResult.getVersion()); assertEquals("description", interpretationResult.getDescription()); @@ -1931,9 +2080,9 @@ public void clearSecondaryInterpretation() throws CatalogException { assertEquals(1, interpretationResult.getSecondaryFindings().size()); assertEquals(1, interpretationResult.getComments().size()); - catalogManager.getInterpretationManager().clear(STUDY, ca.getId(), Collections.singletonList(ca.getId() + ".1"), sessionIdUser); - interpretationResult = catalogManager.getInterpretationManager().get(STUDY, ca.getId() + ".1", QueryOptions.empty(), - sessionIdUser).first(); + catalogManager.getInterpretationManager().clear(studyFqn, ca.getId(), Collections.singletonList(ca.getId() + ".1"), ownerToken); + interpretationResult = catalogManager.getInterpretationManager().get(studyFqn, ca.getId() + ".1", QueryOptions.empty(), + ownerToken).first(); assertEquals(ca.getId() + ".1", interpretationResult.getId()); assertEquals(2, interpretationResult.getVersion()); assertEquals("", interpretationResult.getDescription()); @@ -1948,8 +2097,8 @@ public void clearSecondaryInterpretation() throws CatalogException { public void updateInterpretationFindingsTest() throws CatalogException { ClinicalAnalysis ca = createDummyEnvironment(true, false).first(); - catalogManager.getInterpretationManager().create(STUDY, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.PRIMARY, - QueryOptions.empty(), sessionIdUser); + catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.PRIMARY, + QueryOptions.empty(), ownerToken); ClinicalVariant clinicalVariant = new ClinicalVariant(); clinicalVariant.setId("variantId"); @@ -1960,14 +2109,14 @@ public void updateInterpretationFindingsTest() throws CatalogException { InterpretationUpdateParams params = new InterpretationUpdateParams() .setPrimaryFindings(Collections.singletonList(clinicalVariant)); - OpenCGAResult result = catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getId() + ".1", - params, null, QueryOptions.empty(), sessionIdUser); + OpenCGAResult result = catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getId() + ".1", + params, null, QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumUpdated()); thrown.expect(CatalogException.class); thrown.expectMessage("repeated"); - catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getId() + ".1", params, null, QueryOptions.empty(), - sessionIdUser); + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getId() + ".1", params, null, QueryOptions.empty(), + ownerToken); } // @Test @@ -1975,8 +2124,8 @@ public void updateInterpretationFindingsTest() throws CatalogException { // ClinicalAnalysis ca = createDummyEnvironment(true, false).first(); // // Interpretation interpretation = new Interpretation().setId("interpretation1"); -// catalogManager.getInterpretationManager().create(STUDY, ca.getId(), interpretation, ParamUtils.SaveInterpretationAs.PRIMARY, -// QueryOptions.empty(), sessionIdUser); +// catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), interpretation, ParamUtils.SaveInterpretationAs.PRIMARY, +// QueryOptions.empty(), token); // // ClinicalVariant clinicalVariant = new ClinicalVariant(); // clinicalVariant.setId("variantId"); @@ -1990,8 +2139,8 @@ public void updateInterpretationFindingsTest() throws CatalogException { // .setMethods(Collections.singletonList(new InterpretationMethod("method1", Collections.emptyMap(), Collections.emptyList(), // Collections.emptyList()))) // .setPrimaryFindings(Collections.singletonList(clinicalVariant)); -// OpenCGAResult result = catalogManager.getInterpretationManager().update(STUDY, ca.getId(), "interpretation1", -// params, null, QueryOptions.empty(), sessionIdUser); +// OpenCGAResult result = catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), "interpretation1", +// params, null, QueryOptions.empty(), token); // assertEquals(1, result.getNumUpdated()); // // List variantList = new ArrayList<>(); @@ -2015,12 +2164,12 @@ public void updateInterpretationFindingsTest() throws CatalogException { // new InterpretationMethod("method2", Collections.emptyMap(), Collections.emptyList(), // Collections.emptyList())) // ); -// OpenCGAResult merge = catalogManager.getInterpretationManager().merge(STUDY, ca.getId(), interpretation.getId(), -// interpretationAux, Collections.emptyList(), sessionIdUser); +// OpenCGAResult merge = catalogManager.getInterpretationManager().merge(studyFqn, ca.getId(), interpretation.getId(), +// interpretationAux, Collections.emptyList(), token); // assertEquals(1, merge.getNumUpdated()); // -// Interpretation first = catalogManager.getInterpretationManager().get(STUDY, interpretation.getId(), QueryOptions.empty(), -// sessionIdUser).first(); +// Interpretation first = catalogManager.getInterpretationManager().get(studyFqn, interpretation.getId(), QueryOptions.empty(), +// token).first(); // assertEquals(2, first.getMethods().size()); // assertEquals(2, first.getPrimaryFindings().size()); // assertEquals(Arrays.asList("method1", "method2"), first.getPrimaryFindings().get(0).getInterpretationMethodNames()); @@ -2047,14 +2196,14 @@ public void updateInterpretationFindingsTest() throws CatalogException { // new InterpretationMethod("method2", Collections.emptyMap(), Collections.emptyList(), // Collections.emptyList())) // ); -// catalogManager.getInterpretationManager().create(STUDY, ca.getId(), interpretationAux, ParamUtils.SaveInterpretationAs.SECONDARY, -// QueryOptions.empty(), sessionIdUser); +// catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), interpretationAux, ParamUtils.SaveInterpretationAs.SECONDARY, +// QueryOptions.empty(), token); // -// merge = catalogManager.getInterpretationManager().merge(STUDY, ca.getId(), interpretation.getId(), interpretationAux.getId(), -// Collections.singletonList("variantId3"), sessionIdUser); +// merge = catalogManager.getInterpretationManager().merge(studyFqn, ca.getId(), interpretation.getId(), interpretationAux.getId(), +// Collections.singletonList("variantId3"), token); // assertEquals(1, merge.getNumUpdated()); // -// first = catalogManager.getInterpretationManager().get(STUDY, interpretation.getId(), QueryOptions.empty(), sessionIdUser).first(); +// first = catalogManager.getInterpretationManager().get(studyFqn, interpretation.getId(), QueryOptions.empty(), token).first(); // assertEquals(3, first.getMethods().size()); // assertEquals(3, first.getPrimaryFindings().size()); // assertEquals(Arrays.asList("method1", "method2"), first.getPrimaryFindings().get(0).getInterpretationMethodNames()); @@ -2066,38 +2215,38 @@ public void updateInterpretationFindingsTest() throws CatalogException { public void searchInterpretationVersion() throws CatalogException { ClinicalAnalysis ca = createDummyEnvironment(true, false).first(); - catalogManager.getInterpretationManager().create(STUDY, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.PRIMARY, - QueryOptions.empty(), sessionIdUser); + catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.PRIMARY, + QueryOptions.empty(), ownerToken); - InterpretationUpdateParams params = new InterpretationUpdateParams().setAnalyst(new ClinicalAnalystParam("user2")); - OpenCGAResult result = catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getId() + ".1", - params, null, QueryOptions.empty(), sessionIdUser); + InterpretationUpdateParams params = new InterpretationUpdateParams().setAnalyst(new ClinicalAnalystParam(normalUserId2)); + OpenCGAResult result = catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getId() + ".1", + params, null, QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumUpdated()); QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, InterpretationDBAdaptor.QueryParams.VERSION.key()); - result = catalogManager.getInterpretationManager().get(STUDY, Collections.singletonList(ca.getId() + ".1"), - new Query(Constants.ALL_VERSIONS, true), options, false, sessionIdUser); + result = catalogManager.getInterpretationManager().get(studyFqn, Collections.singletonList(ca.getId() + ".1"), + new Query(Constants.ALL_VERSIONS, true), options, false, ownerToken); assertEquals(2, result.getNumResults()); - result = catalogManager.getInterpretationManager().get(STUDY, Collections.singletonList(ca.getId() + ".1"), - new Query(InterpretationDBAdaptor.QueryParams.VERSION.key(), "1,2"), options, false, sessionIdUser); + result = catalogManager.getInterpretationManager().get(studyFqn, Collections.singletonList(ca.getId() + ".1"), + new Query(InterpretationDBAdaptor.QueryParams.VERSION.key(), "1,2"), options, false, ownerToken); assertEquals(2, result.getNumResults()); - result = catalogManager.getInterpretationManager().get(STUDY, Collections.singletonList(ca.getId() + ".1"), - new Query(InterpretationDBAdaptor.QueryParams.VERSION.key(), "All"), options, false, sessionIdUser); + result = catalogManager.getInterpretationManager().get(studyFqn, Collections.singletonList(ca.getId() + ".1"), + new Query(InterpretationDBAdaptor.QueryParams.VERSION.key(), "All"), options, false, ownerToken); assertEquals(2, result.getNumResults()); try { - catalogManager.getInterpretationManager().get(STUDY, Arrays.asList(ca.getId() + ".1", ca.getId() + ".2"), - new Query(Constants.ALL_VERSIONS, true), options, false, sessionIdUser); + catalogManager.getInterpretationManager().get(studyFqn, Arrays.asList(ca.getId() + ".1", ca.getId() + ".2"), + new Query(Constants.ALL_VERSIONS, true), options, false, ownerToken); fail("The previous call should fail because it should not be possible to fetch all versions of multiple interpretations"); } catch (CatalogException e) { assertTrue(e.getMessage().contains("multiple")); } try { - catalogManager.getInterpretationManager().get(STUDY, Arrays.asList(ca.getId() + ".1", "interpretation2"), - new Query(InterpretationDBAdaptor.QueryParams.VERSION.key(), "1"), options, false, sessionIdUser); + catalogManager.getInterpretationManager().get(studyFqn, Arrays.asList(ca.getId() + ".1", "interpretation2"), + new Query(InterpretationDBAdaptor.QueryParams.VERSION.key(), "1"), options, false, ownerToken); fail("The previous call should fail users cannot fetch a concrete version for multiple interpretations"); } catch (CatalogException e) { assertTrue(e.getMessage().contains("multiple")); @@ -2108,31 +2257,31 @@ public void searchInterpretationVersion() throws CatalogException { public void revertInterpretationVersion() throws CatalogException { ClinicalAnalysis ca = createDummyEnvironment(true, false).first(); - catalogManager.getInterpretationManager().create(STUDY, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.PRIMARY, - QueryOptions.empty(), sessionIdUser); + catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.PRIMARY, + QueryOptions.empty(), ownerToken); // version 2 - InterpretationUpdateParams params = new InterpretationUpdateParams().setAnalyst(new ClinicalAnalystParam("user2")); - catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getId() + ".1", params, null, QueryOptions.empty(), - sessionIdUser); + InterpretationUpdateParams params = new InterpretationUpdateParams().setAnalyst(new ClinicalAnalystParam(normalUserId2)); + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getId() + ".1", params, null, QueryOptions.empty(), + ownerToken); // version 3 params = new InterpretationUpdateParams().setComments(Collections.singletonList(new ClinicalCommentParam("my first comment", Collections.singletonList("tag1")))); - catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getId() + ".1", params, null, QueryOptions.empty(), - sessionIdUser); + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getId() + ".1", params, null, QueryOptions.empty(), + ownerToken); // version 4 params = new InterpretationUpdateParams().setComments(Collections.singletonList(new ClinicalCommentParam("my second comment", Collections.singletonList("tag2")))); - catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getId() + ".1", params, null, QueryOptions.empty(), - sessionIdUser); + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getId() + ".1", params, null, QueryOptions.empty(), + ownerToken); // Current status Interpretation interpretation = - catalogManager.getInterpretationManager().get(STUDY, ca.getId() + ".1", QueryOptions.empty(), sessionIdUser).first(); + catalogManager.getInterpretationManager().get(studyFqn, ca.getId() + ".1", QueryOptions.empty(), ownerToken).first(); assertEquals(4, interpretation.getVersion()); - assertEquals("user2", interpretation.getAnalyst().getId()); + assertEquals(normalUserId2, interpretation.getAnalyst().getId()); assertEquals(2, interpretation.getComments().size()); assertEquals(1, interpretation.getComments().get(0).getTags().size()); assertEquals("tag1", interpretation.getComments().get(0).getTags().get(0)); @@ -2141,45 +2290,45 @@ public void revertInterpretationVersion() throws CatalogException { // TEST REVERT try { - catalogManager.getInterpretationManager().revert(STUDY, ca.getId(), ca.getId() + ".1", 0, sessionIdUser); + catalogManager.getInterpretationManager().revert(studyFqn, ca.getId(), ca.getId() + ".1", 0, ownerToken); fail("A CatalogException should be raised pointing we cannot set to a version equal or inferior to 0"); } catch (CatalogException e) { } try { - catalogManager.getInterpretationManager().revert(STUDY, ca.getId(), ca.getId() + ".1", 5, sessionIdUser); + catalogManager.getInterpretationManager().revert(studyFqn, ca.getId(), ca.getId() + ".1", 5, ownerToken); fail("A CatalogException should be raised pointing we cannot set to a version above the current one"); } catch (CatalogException e) { } - OpenCGAResult result = catalogManager.getInterpretationManager().revert(STUDY, ca.getId(), ca.getId() + ".1", 2, - sessionIdUser); + OpenCGAResult result = catalogManager.getInterpretationManager().revert(studyFqn, ca.getId(), ca.getId() + ".1", 2, + ownerToken); assertEquals(1, result.getNumUpdated()); interpretation = - catalogManager.getInterpretationManager().get(STUDY, ca.getId() + ".1", QueryOptions.empty(), sessionIdUser).first(); + catalogManager.getInterpretationManager().get(studyFqn, ca.getId() + ".1", QueryOptions.empty(), ownerToken).first(); assertEquals(5, interpretation.getVersion()); - assertEquals("user2", interpretation.getAnalyst().getId()); + assertEquals(normalUserId2, interpretation.getAnalyst().getId()); assertEquals(0, interpretation.getComments().size()); - result = catalogManager.getInterpretationManager().revert(STUDY, ca.getId(), ca.getId() + ".1", 3, sessionIdUser); + result = catalogManager.getInterpretationManager().revert(studyFqn, ca.getId(), ca.getId() + ".1", 3, ownerToken); assertEquals(1, result.getNumUpdated()); interpretation = - catalogManager.getInterpretationManager().get(STUDY, ca.getId() + ".1", QueryOptions.empty(), sessionIdUser).first(); + catalogManager.getInterpretationManager().get(studyFqn, ca.getId() + ".1", QueryOptions.empty(), ownerToken).first(); assertEquals(6, interpretation.getVersion()); - assertEquals("user2", interpretation.getAnalyst().getId()); + assertEquals(normalUserId2, interpretation.getAnalyst().getId()); assertEquals(1, interpretation.getComments().size()); assertEquals(1, interpretation.getComments().get(0).getTags().size()); assertEquals("tag1", interpretation.getComments().get(0).getTags().get(0)); - result = catalogManager.getInterpretationManager().revert(STUDY, ca.getId(), ca.getId() + ".1", 4, sessionIdUser); + result = catalogManager.getInterpretationManager().revert(studyFqn, ca.getId(), ca.getId() + ".1", 4, ownerToken); assertEquals(1, result.getNumUpdated()); interpretation = - catalogManager.getInterpretationManager().get(STUDY, ca.getId() + ".1", QueryOptions.empty(), sessionIdUser).first(); + catalogManager.getInterpretationManager().get(studyFqn, ca.getId() + ".1", QueryOptions.empty(), ownerToken).first(); assertEquals(7, interpretation.getVersion()); - assertEquals("user2", interpretation.getAnalyst().getId()); + assertEquals(normalUserId2, interpretation.getAnalyst().getId()); assertEquals(2, interpretation.getComments().size()); assertEquals(1, interpretation.getComments().get(0).getTags().size()); assertEquals("tag1", interpretation.getComments().get(0).getTags().get(0)); @@ -2187,12 +2336,12 @@ public void revertInterpretationVersion() throws CatalogException { assertEquals("tag2", interpretation.getComments().get(1).getTags().get(0)); Query query = new Query(Constants.ALL_VERSIONS, true); - result = catalogManager.getInterpretationManager().get(STUDY, Collections.singletonList(ca.getId() + ".1"), query, - QueryOptions.empty(), false, sessionIdUser); + result = catalogManager.getInterpretationManager().get(studyFqn, Collections.singletonList(ca.getId() + ".1"), query, + QueryOptions.empty(), false, ownerToken); assertEquals(7, result.getNumResults()); - ClinicalAnalysis clinicalAnalysis = catalogManager.getClinicalAnalysisManager().get(STUDY, ca.getId(), QueryOptions.empty(), - sessionIdUser).first(); + ClinicalAnalysis clinicalAnalysis = catalogManager.getClinicalAnalysisManager().get(studyFqn, ca.getId(), QueryOptions.empty(), + ownerToken).first(); assertEquals(8, clinicalAnalysis.getAudit().size()); assertEquals(7, clinicalAnalysis.getInterpretation().getVersion()); } @@ -2201,19 +2350,19 @@ public void revertInterpretationVersion() throws CatalogException { public void updateInterpretationTest() throws CatalogException { ClinicalAnalysis ca = createDummyEnvironment(true, false).first(); - catalogManager.getInterpretationManager().create(STUDY, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.PRIMARY, - QueryOptions.empty(), sessionIdUser); + catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.PRIMARY, + QueryOptions.empty(), ownerToken); - catalogManager.getInterpretationManager().create(STUDY, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.SECONDARY, - QueryOptions.empty(), sessionIdUser); + catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.SECONDARY, + QueryOptions.empty(), ownerToken); - catalogManager.getInterpretationManager().create(STUDY, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.SECONDARY, - QueryOptions.empty(), sessionIdUser); + catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.SECONDARY, + QueryOptions.empty(), ownerToken); - catalogManager.getInterpretationManager().create(STUDY, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.SECONDARY, - QueryOptions.empty(), sessionIdUser); + catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.SECONDARY, + QueryOptions.empty(), ownerToken); - ca = catalogManager.getClinicalAnalysisManager().get(STUDY, ca.getId(), QueryOptions.empty(), sessionIdUser).first(); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, ca.getId(), QueryOptions.empty(), ownerToken).first(); assertNotNull(ca.getInterpretation()); assertEquals(5, ca.getAudit().size()); assertEquals(ClinicalAudit.Action.CREATE_INTERPRETATION, ca.getAudit().get(4).getAction()); @@ -2223,26 +2372,26 @@ public void updateInterpretationTest() throws CatalogException { assertEquals(ca.getId() + ".3", ca.getSecondaryInterpretations().get(1).getId()); assertEquals(ca.getId() + ".4", ca.getSecondaryInterpretations().get(2).getId()); - InterpretationUpdateParams params = new InterpretationUpdateParams().setAnalyst(new ClinicalAnalystParam("user2")); - OpenCGAResult result = catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getId() + ".1", - params, null, QueryOptions.empty(), sessionIdUser); + InterpretationUpdateParams params = new InterpretationUpdateParams().setAnalyst(new ClinicalAnalystParam(normalUserId2)); + OpenCGAResult result = catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getId() + ".1", + params, null, QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumUpdated()); - ca = catalogManager.getClinicalAnalysisManager().get(STUDY, ca.getId(), QueryOptions.empty(), sessionIdUser).first(); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, ca.getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(6, ca.getAudit().size()); assertEquals(ClinicalAudit.Action.UPDATE_INTERPRETATION, ca.getAudit().get(5).getAction()); assertNotNull(ca.getInterpretation().getAnalyst()); - assertEquals("user2", ca.getInterpretation().getAnalyst().getId()); + assertEquals(normalUserId2, ca.getInterpretation().getAnalyst().getId()); assertEquals(2, ca.getInterpretation().getVersion()); // Update a secondary interpretation params = new InterpretationUpdateParams() .setDescription("my description"); - result = catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getId() + ".3", params, null, QueryOptions.empty(), - sessionIdUser); + result = catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getId() + ".3", params, null, QueryOptions.empty(), + ownerToken); assertEquals(1, result.getNumUpdated()); - ca = catalogManager.getClinicalAnalysisManager().get(STUDY, ca.getId(), QueryOptions.empty(), sessionIdUser).first(); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, ca.getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(7, ca.getAudit().size()); assertEquals(ClinicalAudit.Action.UPDATE_INTERPRETATION, ca.getAudit().get(6).getAction()); assertEquals("my description", ca.getSecondaryInterpretations().get(1).getDescription()); @@ -2250,13 +2399,13 @@ public void updateInterpretationTest() throws CatalogException { // Scalate secondary interpretation to primary and delete params = new InterpretationUpdateParams(); - result = catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getId() + ".3", params, - ParamUtils.SaveInterpretationAs.PRIMARY, QueryOptions.empty(), sessionIdUser); + result = catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getId() + ".3", params, + ParamUtils.SaveInterpretationAs.PRIMARY, QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumUpdated()); - catalogManager.getInterpretationManager().delete(STUDY, ca.getId(), Collections.singletonList(ca.getId() + ".1"), sessionIdUser); + catalogManager.getInterpretationManager().delete(studyFqn, ca.getId(), Collections.singletonList(ca.getId() + ".1"), ownerToken); - ca = catalogManager.getClinicalAnalysisManager().get(STUDY, ca.getId(), QueryOptions.empty(), sessionIdUser).first(); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, ca.getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(10, ca.getAudit().size()); assertEquals(ClinicalAudit.Action.UPDATE_INTERPRETATION, ca.getAudit().get(7).getAction()); assertEquals(ClinicalAudit.Action.SWAP_INTERPRETATION, ca.getAudit().get(8).getAction()); @@ -2270,11 +2419,11 @@ public void updateInterpretationTest() throws CatalogException { // Scalate secondary interpretation to primary params = new InterpretationUpdateParams(); - result = catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getId() + ".4", params, - ParamUtils.SaveInterpretationAs.PRIMARY, QueryOptions.empty(), sessionIdUser); + result = catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getId() + ".4", params, + ParamUtils.SaveInterpretationAs.PRIMARY, QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumUpdated()); - ca = catalogManager.getClinicalAnalysisManager().get(STUDY, ca.getId(), QueryOptions.empty(), sessionIdUser).first(); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, ca.getId(), QueryOptions.empty(), ownerToken).first(); assertNotNull(ca.getInterpretation()); assertEquals(12, ca.getAudit().size()); assertEquals(ClinicalAudit.Action.UPDATE_INTERPRETATION, ca.getAudit().get(10).getAction()); @@ -2287,11 +2436,11 @@ public void updateInterpretationTest() throws CatalogException { assertEquals(2, ca.getSecondaryInterpretations().get(1).getVersion()); // Scalate secondary interpretation to primary - result = catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getId() + ".3", params, - ParamUtils.SaveInterpretationAs.PRIMARY, QueryOptions.empty(), sessionIdUser); + result = catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getId() + ".3", params, + ParamUtils.SaveInterpretationAs.PRIMARY, QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumUpdated()); - ca = catalogManager.getClinicalAnalysisManager().get(STUDY, ca.getId(), QueryOptions.empty(), sessionIdUser).first(); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, ca.getId(), QueryOptions.empty(), ownerToken).first(); assertNotNull(ca.getInterpretation()); assertEquals(ca.getId() + ".3", ca.getInterpretation().getId()); assertEquals(2, ca.getInterpretation().getVersion()); @@ -2302,11 +2451,11 @@ public void updateInterpretationTest() throws CatalogException { // Move primary to secondary params = new InterpretationUpdateParams(); - result = catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getId() + ".3", params, - ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), sessionIdUser); + result = catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getId() + ".3", params, + ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumUpdated()); - ca = catalogManager.getClinicalAnalysisManager().get(STUDY, ca.getId(), QueryOptions.empty(), sessionIdUser).first(); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, ca.getId(), QueryOptions.empty(), ownerToken).first(); assertNull(ca.getInterpretation()); assertEquals(3, ca.getSecondaryInterpretations().size()); assertEquals(ca.getId() + ".2", ca.getSecondaryInterpretations().get(0).getId()); @@ -2316,11 +2465,11 @@ public void updateInterpretationTest() throws CatalogException { // Scalate to primary params = new InterpretationUpdateParams(); - result = catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getId() + ".2", params, - ParamUtils.SaveInterpretationAs.PRIMARY, QueryOptions.empty(), sessionIdUser); + result = catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getId() + ".2", params, + ParamUtils.SaveInterpretationAs.PRIMARY, QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumUpdated()); - ca = catalogManager.getClinicalAnalysisManager().get(STUDY, ca.getId(), QueryOptions.empty(), sessionIdUser).first(); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, ca.getId(), QueryOptions.empty(), ownerToken).first(); assertNotNull(ca.getInterpretation()); assertEquals(ca.getId() + ".2", ca.getInterpretation().getId()); assertEquals(1, ca.getInterpretation().getVersion()); @@ -2330,11 +2479,11 @@ public void updateInterpretationTest() throws CatalogException { // Move primary to secondary params = new InterpretationUpdateParams(); - result = catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getId() + ".2", params, - ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), sessionIdUser); + result = catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getId() + ".2", params, + ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumUpdated()); - ca = catalogManager.getClinicalAnalysisManager().get(STUDY, ca.getId(), QueryOptions.empty(), sessionIdUser).first(); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, ca.getId(), QueryOptions.empty(), ownerToken).first(); assertNull(ca.getInterpretation()); assertEquals(3, ca.getSecondaryInterpretations().size()); assertEquals(ca.getId() + ".4", ca.getSecondaryInterpretations().get(0).getId()); @@ -2344,11 +2493,11 @@ public void updateInterpretationTest() throws CatalogException { // Scalate to primary and keep params = new InterpretationUpdateParams(); - result = catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getId() + ".2", params, - ParamUtils.SaveInterpretationAs.PRIMARY, QueryOptions.empty(), sessionIdUser); + result = catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getId() + ".2", params, + ParamUtils.SaveInterpretationAs.PRIMARY, QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumUpdated()); - ca = catalogManager.getClinicalAnalysisManager().get(STUDY, ca.getId(), QueryOptions.empty(), sessionIdUser).first(); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, ca.getId(), QueryOptions.empty(), ownerToken).first(); assertNotNull(ca.getInterpretation()); assertEquals(ca.getId() + ".2", ca.getInterpretation().getId()); assertEquals(1, ca.getInterpretation().getVersion()); @@ -2358,11 +2507,11 @@ public void updateInterpretationTest() throws CatalogException { // Move primary to secondary params = new InterpretationUpdateParams(); - result = catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getId() + ".2", params, - ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), sessionIdUser); + result = catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getId() + ".2", params, + ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumUpdated()); - ca = catalogManager.getClinicalAnalysisManager().get(STUDY, ca.getId(), QueryOptions.empty(), sessionIdUser).first(); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, ca.getId(), QueryOptions.empty(), ownerToken).first(); assertNull(ca.getInterpretation()); assertEquals(3, ca.getSecondaryInterpretations().size()); assertEquals(ca.getId() + ".4", ca.getSecondaryInterpretations().get(0).getId()); @@ -2372,11 +2521,11 @@ public void updateInterpretationTest() throws CatalogException { // Scalate to primary params = new InterpretationUpdateParams(); - result = catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getId() + ".2", params, - ParamUtils.SaveInterpretationAs.PRIMARY, QueryOptions.empty(), sessionIdUser); + result = catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getId() + ".2", params, + ParamUtils.SaveInterpretationAs.PRIMARY, QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumUpdated()); - ca = catalogManager.getClinicalAnalysisManager().get(STUDY, ca.getId(), QueryOptions.empty(), sessionIdUser).first(); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, ca.getId(), QueryOptions.empty(), ownerToken).first(); assertNotNull(ca.getInterpretation()); assertEquals(ca.getId() + ".2", ca.getInterpretation().getId()); assertEquals(1, ca.getInterpretation().getVersion()); @@ -2389,19 +2538,19 @@ public void updateInterpretationTest() throws CatalogException { public void updatePrimarySecondaryLockedInterpretationTest() throws CatalogException { ClinicalAnalysis ca = createDummyEnvironment(true, false).first(); - catalogManager.getInterpretationManager().create(STUDY, ca.getId(), new Interpretation(), - ParamUtils.SaveInterpretationAs.PRIMARY, QueryOptions.empty(), sessionIdUser); + catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation(), + ParamUtils.SaveInterpretationAs.PRIMARY, QueryOptions.empty(), ownerToken); - catalogManager.getInterpretationManager().create(STUDY, ca.getId(), new Interpretation().setLocked(true), - ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), sessionIdUser); + catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation().setLocked(true), + ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), ownerToken); - catalogManager.getInterpretationManager().create(STUDY, ca.getId(), new Interpretation().setLocked(true), - ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), sessionIdUser); + catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation().setLocked(true), + ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), ownerToken); - catalogManager.getInterpretationManager().create(STUDY, ca.getId(), new Interpretation().setLocked(true), - ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), sessionIdUser); + catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation().setLocked(true), + ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), ownerToken); - ca = catalogManager.getClinicalAnalysisManager().get(STUDY, ca.getId(), QueryOptions.empty(), sessionIdUser).first(); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, ca.getId(), QueryOptions.empty(), ownerToken).first(); assertNotNull(ca.getInterpretation()); assertFalse(ca.getInterpretation().isLocked()); assertEquals(5, ca.getAudit().size()); @@ -2417,13 +2566,13 @@ public void updatePrimarySecondaryLockedInterpretationTest() throws CatalogExcep // Scalate secondary interpretation to primary and delete InterpretationUpdateParams params = new InterpretationUpdateParams(); - OpenCGAResult result = catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getId() + ".3", - params, ParamUtils.SaveInterpretationAs.PRIMARY, QueryOptions.empty(), sessionIdUser); + OpenCGAResult result = catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getId() + ".3", + params, ParamUtils.SaveInterpretationAs.PRIMARY, QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumUpdated()); - catalogManager.getInterpretationManager().delete(STUDY, ca.getId(), Collections.singletonList(ca.getId() + ".1"), sessionIdUser); + catalogManager.getInterpretationManager().delete(studyFqn, ca.getId(), Collections.singletonList(ca.getId() + ".1"), ownerToken); - ca = catalogManager.getClinicalAnalysisManager().get(STUDY, ca.getId(), QueryOptions.empty(), sessionIdUser).first(); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, ca.getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(8, ca.getAudit().size()); assertNotNull(ca.getInterpretation()); assertEquals(ca.getId() + ".3", ca.getInterpretation().getId()); @@ -2433,11 +2582,11 @@ public void updatePrimarySecondaryLockedInterpretationTest() throws CatalogExcep // Scalate secondary interpretation to primary params = new InterpretationUpdateParams(); - result = catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getId() + ".4", params, - ParamUtils.SaveInterpretationAs.PRIMARY, QueryOptions.empty(), sessionIdUser); + result = catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getId() + ".4", params, + ParamUtils.SaveInterpretationAs.PRIMARY, QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumUpdated()); - ca = catalogManager.getClinicalAnalysisManager().get(STUDY, ca.getId(), QueryOptions.empty(), sessionIdUser).first(); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, ca.getId(), QueryOptions.empty(), ownerToken).first(); assertNotNull(ca.getInterpretation()); assertEquals(ca.getId() + ".4", ca.getInterpretation().getId()); assertEquals(2, ca.getSecondaryInterpretations().size()); @@ -2445,11 +2594,11 @@ public void updatePrimarySecondaryLockedInterpretationTest() throws CatalogExcep assertEquals(ca.getId() + ".3", ca.getSecondaryInterpretations().get(1).getId()); // Scalate secondary interpretation to primary - result = catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getId() + ".3", params, - ParamUtils.SaveInterpretationAs.PRIMARY, QueryOptions.empty(), sessionIdUser); + result = catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getId() + ".3", params, + ParamUtils.SaveInterpretationAs.PRIMARY, QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumUpdated()); - ca = catalogManager.getClinicalAnalysisManager().get(STUDY, ca.getId(), QueryOptions.empty(), sessionIdUser).first(); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, ca.getId(), QueryOptions.empty(), ownerToken).first(); assertNotNull(ca.getInterpretation()); assertEquals(ca.getId() + ".3", ca.getInterpretation().getId()); assertEquals(2, ca.getSecondaryInterpretations().size()); @@ -2458,11 +2607,11 @@ public void updatePrimarySecondaryLockedInterpretationTest() throws CatalogExcep // Move primary to secondary params = new InterpretationUpdateParams(); - result = catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getId() + ".3", params, - ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), sessionIdUser); + result = catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getId() + ".3", params, + ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumUpdated()); - ca = catalogManager.getClinicalAnalysisManager().get(STUDY, ca.getId(), QueryOptions.empty(), sessionIdUser).first(); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, ca.getId(), QueryOptions.empty(), ownerToken).first(); assertNull(ca.getInterpretation()); assertEquals(3, ca.getSecondaryInterpretations().size()); assertEquals(ca.getId() + ".2", ca.getSecondaryInterpretations().get(0).getId()); @@ -2471,11 +2620,11 @@ public void updatePrimarySecondaryLockedInterpretationTest() throws CatalogExcep // Scalate to primary params = new InterpretationUpdateParams(); - result = catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getId() + ".2", params, - ParamUtils.SaveInterpretationAs.PRIMARY, QueryOptions.empty(), sessionIdUser); + result = catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getId() + ".2", params, + ParamUtils.SaveInterpretationAs.PRIMARY, QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumUpdated()); - ca = catalogManager.getClinicalAnalysisManager().get(STUDY, ca.getId(), QueryOptions.empty(), sessionIdUser).first(); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, ca.getId(), QueryOptions.empty(), ownerToken).first(); assertNotNull(ca.getInterpretation()); assertEquals(ca.getId() + ".2", ca.getInterpretation().getId()); assertEquals(2, ca.getSecondaryInterpretations().size()); @@ -2484,11 +2633,11 @@ public void updatePrimarySecondaryLockedInterpretationTest() throws CatalogExcep // Move primary to secondary params = new InterpretationUpdateParams(); - result = catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getId() + ".2", params, - ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), sessionIdUser); + result = catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getId() + ".2", params, + ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumUpdated()); - ca = catalogManager.getClinicalAnalysisManager().get(STUDY, ca.getId(), QueryOptions.empty(), sessionIdUser).first(); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, ca.getId(), QueryOptions.empty(), ownerToken).first(); assertNull(ca.getInterpretation()); assertEquals(3, ca.getSecondaryInterpretations().size()); assertEquals(ca.getId() + ".4", ca.getSecondaryInterpretations().get(0).getId()); @@ -2497,11 +2646,11 @@ public void updatePrimarySecondaryLockedInterpretationTest() throws CatalogExcep // Scalate to primary and keep params = new InterpretationUpdateParams(); - result = catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getId() + ".2", params, - ParamUtils.SaveInterpretationAs.PRIMARY, QueryOptions.empty(), sessionIdUser); + result = catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getId() + ".2", params, + ParamUtils.SaveInterpretationAs.PRIMARY, QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumUpdated()); - ca = catalogManager.getClinicalAnalysisManager().get(STUDY, ca.getId(), QueryOptions.empty(), sessionIdUser).first(); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, ca.getId(), QueryOptions.empty(), ownerToken).first(); assertNotNull(ca.getInterpretation()); assertEquals(ca.getId() + ".2", ca.getInterpretation().getId()); assertEquals(2, ca.getSecondaryInterpretations().size()); @@ -2510,11 +2659,11 @@ public void updatePrimarySecondaryLockedInterpretationTest() throws CatalogExcep // Move primary to secondary params = new InterpretationUpdateParams(); - result = catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getId() + ".2", params, - ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), sessionIdUser); + result = catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getId() + ".2", params, + ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumUpdated()); - ca = catalogManager.getClinicalAnalysisManager().get(STUDY, ca.getId(), QueryOptions.empty(), sessionIdUser).first(); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, ca.getId(), QueryOptions.empty(), ownerToken).first(); assertNull(ca.getInterpretation()); assertEquals(3, ca.getSecondaryInterpretations().size()); assertEquals(ca.getId() + ".4", ca.getSecondaryInterpretations().get(0).getId()); @@ -2523,11 +2672,11 @@ public void updatePrimarySecondaryLockedInterpretationTest() throws CatalogExcep // Scalate to primary params = new InterpretationUpdateParams(); - result = catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getId() + ".2", params, - ParamUtils.SaveInterpretationAs.PRIMARY, QueryOptions.empty(), sessionIdUser); + result = catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getId() + ".2", params, + ParamUtils.SaveInterpretationAs.PRIMARY, QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumUpdated()); - ca = catalogManager.getClinicalAnalysisManager().get(STUDY, ca.getId(), QueryOptions.empty(), sessionIdUser).first(); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, ca.getId(), QueryOptions.empty(), ownerToken).first(); assertNotNull(ca.getInterpretation()); assertEquals(ca.getId() + ".2", ca.getInterpretation().getId()); assertEquals(2, ca.getSecondaryInterpretations().size()); @@ -2539,23 +2688,23 @@ public void updatePrimarySecondaryLockedInterpretationTest() throws CatalogExcep public void deleteInterpretationTest() throws CatalogException { ClinicalAnalysis ca = createDummyEnvironment(true, false).first(); - catalogManager.getInterpretationManager().create(STUDY, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.PRIMARY, - QueryOptions.empty(), sessionIdUser); + catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.PRIMARY, + QueryOptions.empty(), ownerToken); - catalogManager.getInterpretationManager().create(STUDY, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.SECONDARY, - QueryOptions.empty(), sessionIdUser); + catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.SECONDARY, + QueryOptions.empty(), ownerToken); - catalogManager.getInterpretationManager().create(STUDY, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.SECONDARY, - QueryOptions.empty(), sessionIdUser); + catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.SECONDARY, + QueryOptions.empty(), ownerToken); - catalogManager.getInterpretationManager().create(STUDY, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.SECONDARY, - QueryOptions.empty(), sessionIdUser); + catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.SECONDARY, + QueryOptions.empty(), ownerToken); // We update interpretation 1 so a new version is generated - catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getId() + ".1", new InterpretationUpdateParams() - .setDescription("my description"), null, QueryOptions.empty(), sessionIdUser); + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getId() + ".1", new InterpretationUpdateParams() + .setDescription("my description"), null, QueryOptions.empty(), ownerToken); - ca = catalogManager.getClinicalAnalysisManager().get(STUDY, ca.getId(), QueryOptions.empty(), sessionIdUser).first(); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, ca.getId(), QueryOptions.empty(), ownerToken).first(); assertNotNull(ca.getInterpretation()); assertEquals(ca.getId() + ".1", ca.getInterpretation().getId()); assertEquals(3, ca.getSecondaryInterpretations().size()); @@ -2563,18 +2712,18 @@ public void deleteInterpretationTest() throws CatalogException { assertEquals(ca.getId() + ".3", ca.getSecondaryInterpretations().get(1).getId()); assertEquals(ca.getId() + ".4", ca.getSecondaryInterpretations().get(2).getId()); - OpenCGAResult delete = catalogManager.getInterpretationManager().delete(STUDY, ca.getId(), - Arrays.asList(ca.getId() + ".1", ca.getId() + ".3"), sessionIdUser); + OpenCGAResult delete = catalogManager.getInterpretationManager().delete(studyFqn, ca.getId(), + Arrays.asList(ca.getId() + ".1", ca.getId() + ".3"), ownerToken); assertEquals(2, delete.getNumDeleted()); Query query = new Query() .append(InterpretationDBAdaptor.QueryParams.ID.key(), Arrays.asList(ca.getId() + ".1", ca.getId() + ".3")) .append(Constants.ALL_VERSIONS, true); - OpenCGAResult search = catalogManager.getInterpretationManager().search(STUDY, query, QueryOptions.empty(), - sessionIdUser); + OpenCGAResult search = catalogManager.getInterpretationManager().search(studyFqn, query, QueryOptions.empty(), + ownerToken); assertEquals(0, search.getNumResults()); - ca = catalogManager.getClinicalAnalysisManager().get(STUDY, ca.getId(), QueryOptions.empty(), sessionIdUser).first(); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, ca.getId(), QueryOptions.empty(), ownerToken).first(); assertNull(ca.getInterpretation()); assertEquals(2, ca.getSecondaryInterpretations().size()); assertEquals(ca.getId() + ".2", ca.getSecondaryInterpretations().get(0).getId()); @@ -2582,14 +2731,14 @@ public void deleteInterpretationTest() throws CatalogException { query = new Query() .append(InterpretationDBAdaptor.QueryParams.ID.key(), Arrays.asList(ca.getId() + ".1", ca.getId() + ".3")); - OpenCGAResult result = catalogManager.getInterpretationManager().search(STUDY, query, QueryOptions.empty(), - sessionIdUser); + OpenCGAResult result = catalogManager.getInterpretationManager().search(studyFqn, query, QueryOptions.empty(), + ownerToken); assertEquals(0, result.getNumResults()); query = new Query() .append(InterpretationDBAdaptor.QueryParams.ID.key(), Arrays.asList(ca.getId() + ".1", ca.getId() + ".3")) .append(InterpretationDBAdaptor.QueryParams.DELETED.key(), true); - result = catalogManager.getInterpretationManager().search(STUDY, query, QueryOptions.empty(), sessionIdUser); + result = catalogManager.getInterpretationManager().search(studyFqn, query, QueryOptions.empty(), ownerToken); assertEquals(3, result.getNumResults()); } @@ -2598,9 +2747,9 @@ public void searchClinicalAnalysisByProband() throws CatalogException { createDummyEnvironment(true, false); createDummyEnvironment(false, false); - OpenCGAResult search = catalogManager.getClinicalAnalysisManager().search(STUDY, + OpenCGAResult search = catalogManager.getClinicalAnalysisManager().search(studyFqn, new Query(ParamConstants.CLINICAL_PROBAND_PARAM, "^chil"), - new QueryOptions(QueryOptions.INCLUDE, ClinicalAnalysisDBAdaptor.QueryParams.PROBAND_ID.key()), sessionIdUser); + new QueryOptions(QueryOptions.INCLUDE, ClinicalAnalysisDBAdaptor.QueryParams.PROBAND_ID.key()), ownerToken); assertEquals(2, search.getNumResults()); assertTrue(StringUtils.isNotEmpty(search.first().getProband().getId())); assertTrue(StringUtils.isEmpty(search.first().getProband().getName())); @@ -2611,109 +2760,109 @@ public void searchClinicalAnalysisByStatus() throws CatalogException { createDummyEnvironment(true, false); createDummyEnvironment(false, false); - OpenCGAResult search = catalogManager.getClinicalAnalysisManager().search(STUDY, - new Query(ParamConstants.STATUS_PARAM, ClinicalAnalysisStatus.DONE), - new QueryOptions(QueryOptions.INCLUDE, ClinicalAnalysisDBAdaptor.QueryParams.PROBAND_ID.key()), sessionIdUser); + OpenCGAResult search = catalogManager.getClinicalAnalysisManager().search(studyFqn, + new Query(ParamConstants.STATUS_PARAM, "DONE"), + new QueryOptions(QueryOptions.INCLUDE, ClinicalAnalysisDBAdaptor.QueryParams.PROBAND_ID.key()), ownerToken); assertEquals(0, search.getNumResults()); - search = catalogManager.getClinicalAnalysisManager().search(STUDY, - new Query(ParamConstants.STATUS_PARAM, ClinicalAnalysisStatus.READY_FOR_INTERPRETATION), - new QueryOptions(), sessionIdUser); + search = catalogManager.getClinicalAnalysisManager().search(studyFqn, + new Query(ParamConstants.STATUS_PARAM, "READY_FOR_INTERPRETATION"), + new QueryOptions(), ownerToken); assertEquals(2, search.getNumResults()); for (ClinicalAnalysis result : search.getResults()) { - assertEquals(ClinicalAnalysisStatus.READY_FOR_INTERPRETATION, result.getStatus().getId()); + assertEquals("READY_FOR_INTERPRETATION", result.getStatus().getId()); } - catalogManager.getClinicalAnalysisManager().update(STUDY, search.first().getId(), - new ClinicalAnalysisUpdateParams().setStatus(new StatusParam(ClinicalAnalysisStatus.REJECTED)), QueryOptions.empty(), - sessionIdUser); - search = catalogManager.getClinicalAnalysisManager().search(STUDY, - new Query(ParamConstants.STATUS_PARAM, ClinicalAnalysisStatus.READY_FOR_INTERPRETATION), - new QueryOptions(), sessionIdUser); + catalogManager.getClinicalAnalysisManager().update(studyFqn, search.first().getId(), + new ClinicalAnalysisUpdateParams().setStatus(new StatusParam("REJECTED")), QueryOptions.empty(), + ownerToken); + search = catalogManager.getClinicalAnalysisManager().search(studyFqn, + new Query(ParamConstants.STATUS_PARAM, "READY_FOR_INTERPRETATION"), + new QueryOptions(), ownerToken); assertEquals(1, search.getNumResults()); for (ClinicalAnalysis result : search.getResults()) { - assertEquals(ClinicalAnalysisStatus.READY_FOR_INTERPRETATION, result.getStatus().getId()); + assertEquals("READY_FOR_INTERPRETATION", result.getStatus().getId()); } - search = catalogManager.getClinicalAnalysisManager().search(STUDY, - new Query(ParamConstants.STATUS_PARAM, ClinicalAnalysisStatus.REJECTED), - new QueryOptions(), sessionIdUser); + search = catalogManager.getClinicalAnalysisManager().search(studyFqn, + new Query(ParamConstants.STATUS_PARAM, "REJECTED"), + new QueryOptions(), ownerToken); assertEquals(1, search.getNumResults()); for (ClinicalAnalysis result : search.getResults()) { - assertEquals(ClinicalAnalysisStatus.REJECTED, result.getStatus().getId()); + assertEquals("REJECTED", result.getStatus().getId()); } } @Test public void deleteClinicalAnalysisTest() throws CatalogException { DataResult dummyEnvironment = createDummyEnvironment(true, false); - OpenCGAResult delete = catalogManager.getClinicalAnalysisManager().delete(STUDY, - Collections.singletonList(dummyEnvironment.first().getId()), null, sessionIdUser); + OpenCGAResult delete = catalogManager.getClinicalAnalysisManager().delete(studyFqn, + Collections.singletonList(dummyEnvironment.first().getId()), null, ownerToken); assertEquals(1, delete.getNumDeleted()); - OpenCGAResult clinicalResult = catalogManager.getClinicalAnalysisManager().get(STUDY, + OpenCGAResult clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, Collections.singletonList(dummyEnvironment.first().getId()), - new Query(ClinicalAnalysisDBAdaptor.QueryParams.DELETED.key(), true), new QueryOptions(), false, sessionIdUser); + new Query(ClinicalAnalysisDBAdaptor.QueryParams.DELETED.key(), true), new QueryOptions(), false, ownerToken); assertEquals(1, clinicalResult.getNumResults()); thrown.expect(CatalogException.class); thrown.expectMessage("not found"); - catalogManager.getClinicalAnalysisManager().get(STUDY, dummyEnvironment.first().getId(), new QueryOptions(), sessionIdUser); + catalogManager.getClinicalAnalysisManager().get(studyFqn, dummyEnvironment.first().getId(), new QueryOptions(), ownerToken); } @Test public void deleteClinicalAnalysisWithEmptyInterpretations() throws CatalogException { ClinicalAnalysis ca = createDummyEnvironment(true, false).first(); - catalogManager.getInterpretationManager().create(STUDY, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.PRIMARY, - QueryOptions.empty(), sessionIdUser); + catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.PRIMARY, + QueryOptions.empty(), ownerToken); - catalogManager.getInterpretationManager().create(STUDY, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.SECONDARY, - QueryOptions.empty(), sessionIdUser); + catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.SECONDARY, + QueryOptions.empty(), ownerToken); - catalogManager.getInterpretationManager().create(STUDY, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.SECONDARY, - QueryOptions.empty(), sessionIdUser); + catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.SECONDARY, + QueryOptions.empty(), ownerToken); - catalogManager.getInterpretationManager().create(STUDY, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.SECONDARY, - QueryOptions.empty(), sessionIdUser); + catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.SECONDARY, + QueryOptions.empty(), ownerToken); - catalogManager.getClinicalAnalysisManager().delete(STUDY, Collections.singletonList(ca.getId()), null, sessionIdUser); + catalogManager.getClinicalAnalysisManager().delete(studyFqn, Collections.singletonList(ca.getId()), null, ownerToken); assertEquals(0, - catalogManager.getClinicalAnalysisManager().search(STUDY, new Query(ClinicalAnalysisDBAdaptor.QueryParams.ID.key(), - ca.getId()), QueryOptions.empty(), sessionIdUser).getNumResults()); + catalogManager.getClinicalAnalysisManager().search(studyFqn, new Query(ClinicalAnalysisDBAdaptor.QueryParams.ID.key(), + ca.getId()), QueryOptions.empty(), ownerToken).getNumResults()); assertEquals(0, - catalogManager.getInterpretationManager().search(STUDY, new Query(InterpretationDBAdaptor.QueryParams.ID.key(), - ca.getId() + ".1"), QueryOptions.empty(), sessionIdUser).getNumResults()); + catalogManager.getInterpretationManager().search(studyFqn, new Query(InterpretationDBAdaptor.QueryParams.ID.key(), + ca.getId() + ".1"), QueryOptions.empty(), ownerToken).getNumResults()); assertEquals(0, - catalogManager.getInterpretationManager().search(STUDY, new Query(InterpretationDBAdaptor.QueryParams.ID.key(), - ca.getId() + ".2"), QueryOptions.empty(), sessionIdUser).getNumResults()); + catalogManager.getInterpretationManager().search(studyFqn, new Query(InterpretationDBAdaptor.QueryParams.ID.key(), + ca.getId() + ".2"), QueryOptions.empty(), ownerToken).getNumResults()); assertEquals(0, - catalogManager.getInterpretationManager().search(STUDY, new Query(InterpretationDBAdaptor.QueryParams.ID.key(), - ca.getId() + ".3"), QueryOptions.empty(), sessionIdUser).getNumResults()); + catalogManager.getInterpretationManager().search(studyFqn, new Query(InterpretationDBAdaptor.QueryParams.ID.key(), + ca.getId() + ".3"), QueryOptions.empty(), ownerToken).getNumResults()); assertEquals(0, - catalogManager.getInterpretationManager().search(STUDY, new Query(InterpretationDBAdaptor.QueryParams.ID.key(), - ca.getId() + ".4"), QueryOptions.empty(), sessionIdUser).getNumResults()); + catalogManager.getInterpretationManager().search(studyFqn, new Query(InterpretationDBAdaptor.QueryParams.ID.key(), + ca.getId() + ".4"), QueryOptions.empty(), ownerToken).getNumResults()); } @Test public void deleteClinicalAnalysisWithInterpretations() throws CatalogException { ClinicalAnalysis ca = createDummyEnvironment(true, false).first(); - catalogManager.getInterpretationManager().create(STUDY, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.PRIMARY, - QueryOptions.empty(), sessionIdUser); + catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.PRIMARY, + QueryOptions.empty(), ownerToken); - catalogManager.getInterpretationManager().create(STUDY, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.SECONDARY, - QueryOptions.empty(), sessionIdUser); + catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.SECONDARY, + QueryOptions.empty(), ownerToken); - catalogManager.getInterpretationManager().create(STUDY, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.SECONDARY, - QueryOptions.empty(), sessionIdUser); + catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.SECONDARY, + QueryOptions.empty(), ownerToken); - catalogManager.getInterpretationManager().create(STUDY, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.SECONDARY, - QueryOptions.empty(), sessionIdUser); + catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.SECONDARY, + QueryOptions.empty(), ownerToken); // Add finding to interpretation - catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getId() + ".1", + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getId() + ".1", new InterpretationUpdateParams().setPrimaryFindings(Collections.singletonList( new ClinicalVariant(VariantAvro.newBuilder() .setChromosome("1") @@ -2726,33 +2875,33 @@ public void deleteClinicalAnalysisWithInterpretations() throws CatalogException .setType(VariantType.SNV) .setStudies(Collections.emptyList()) .build()) - )), null, QueryOptions.empty(), sessionIdUser); + )), null, QueryOptions.empty(), ownerToken); try { - catalogManager.getClinicalAnalysisManager().delete(STUDY, Collections.singletonList(ca.getId()), null, sessionIdUser); + catalogManager.getClinicalAnalysisManager().delete(studyFqn, Collections.singletonList(ca.getId()), null, ownerToken); fail("It should not allow deleting Clinical Analyses with interpretations containing primary findings"); } catch (CatalogException e) { assertTrue(e.getMessage().contains("findings")); } - OpenCGAResult delete = catalogManager.getClinicalAnalysisManager().delete(STUDY, Collections.singletonList(ca.getId()), - new QueryOptions(Constants.FORCE, true), sessionIdUser); + OpenCGAResult delete = catalogManager.getClinicalAnalysisManager().delete(studyFqn, Collections.singletonList(ca.getId()), + new QueryOptions(Constants.FORCE, true), ownerToken); assertEquals(1, delete.getNumDeleted()); - OpenCGAResult clinicalResult = catalogManager.getClinicalAnalysisManager().get(STUDY, + OpenCGAResult clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, Collections.singletonList(ca.getId()), - new Query(ClinicalAnalysisDBAdaptor.QueryParams.DELETED.key(), true), new QueryOptions(), false, sessionIdUser); + new Query(ClinicalAnalysisDBAdaptor.QueryParams.DELETED.key(), true), new QueryOptions(), false, ownerToken); assertEquals(1, clinicalResult.getNumResults()); - assertEquals(0, catalogManager.getInterpretationManager().search(STUDY, + assertEquals(0, catalogManager.getInterpretationManager().search(studyFqn, new Query(InterpretationDBAdaptor.QueryParams.ID.key(), Arrays.asList(ca.getId() + ".1", ca.getId() + ".2", - ca.getId() + ".3", ca.getId() + ".4")), QueryOptions.empty(), sessionIdUser).getNumResults()); + ca.getId() + ".3", ca.getId() + ".4")), QueryOptions.empty(), ownerToken).getNumResults()); // Old interpretations were deleted - assertEquals(5, catalogManager.getInterpretationManager().search(STUDY, new Query() + assertEquals(5, catalogManager.getInterpretationManager().search(studyFqn, new Query() .append(InterpretationDBAdaptor.QueryParams.ID.key(), Arrays.asList(ca.getId() + ".1", ca.getId() + ".2", ca.getId() + ".3", ca.getId() + ".4")) - .append(InterpretationDBAdaptor.QueryParams.DELETED.key(), true), QueryOptions.empty(), sessionIdUser) + .append(InterpretationDBAdaptor.QueryParams.DELETED.key(), true), QueryOptions.empty(), ownerToken) .getNumResults()); } @@ -2760,23 +2909,23 @@ public void deleteClinicalAnalysisWithInterpretations() throws CatalogException public void deleteLockedClinicalAnalysis() throws CatalogException { ClinicalAnalysis ca = createDummyEnvironment(true, false).first(); - catalogManager.getClinicalAnalysisManager().update(STUDY, ca.getId(), new ClinicalAnalysisUpdateParams().setLocked(true), - QueryOptions.empty(), sessionIdUser); + catalogManager.getClinicalAnalysisManager().update(studyFqn, ca.getId(), new ClinicalAnalysisUpdateParams().setLocked(true), + QueryOptions.empty(), ownerToken); try { - catalogManager.getClinicalAnalysisManager().delete(STUDY, Collections.singletonList(ca.getId()), null, sessionIdUser); + catalogManager.getClinicalAnalysisManager().delete(studyFqn, Collections.singletonList(ca.getId()), null, ownerToken); fail("It should not allow deleting locked Clinical Analyses"); } catch (CatalogException e) { assertTrue(e.getMessage().contains("locked")); } - OpenCGAResult delete = catalogManager.getClinicalAnalysisManager().delete(STUDY, Collections.singletonList(ca.getId()), - new QueryOptions(Constants.FORCE, true), sessionIdUser); + OpenCGAResult delete = catalogManager.getClinicalAnalysisManager().delete(studyFqn, Collections.singletonList(ca.getId()), + new QueryOptions(Constants.FORCE, true), ownerToken); assertEquals(1, delete.getNumDeleted()); - OpenCGAResult clinicalResult = catalogManager.getClinicalAnalysisManager().get(STUDY, + OpenCGAResult clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, Collections.singletonList(ca.getId()), new Query(ClinicalAnalysisDBAdaptor.QueryParams.DELETED.key(), true), - new QueryOptions(), false, sessionIdUser); + new QueryOptions(), false, ownerToken); assertEquals(1, clinicalResult.getNumResults()); } @@ -2785,17 +2934,17 @@ public void searchByDisorderTest() throws CatalogException { DataResult dummyEnvironment = createDummyEnvironment(true, false); ClinicalAnalysisUpdateParams updateParams = new ClinicalAnalysisUpdateParams() .setDisorder(new DisorderReferenceParam("dis1")); - catalogManager.getClinicalAnalysisManager().update(STUDY, dummyEnvironment.first().getId(), updateParams, QueryOptions.empty(), - sessionIdUser); + catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), updateParams, QueryOptions.empty(), + ownerToken); createDummyEnvironment(false, false); - OpenCGAResult result = catalogManager.getClinicalAnalysisManager().search(STUDY, - new Query(ParamConstants.CLINICAL_DISORDER_PARAM, "dis1"), QueryOptions.empty(), sessionIdUser); + OpenCGAResult result = catalogManager.getClinicalAnalysisManager().search(studyFqn, + new Query(ParamConstants.CLINICAL_DISORDER_PARAM, "dis1"), QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumResults()); assertEquals(dummyEnvironment.first().getId(), result.first().getId()); - result = catalogManager.getClinicalAnalysisManager().search(STUDY, new Query(), QueryOptions.empty(), sessionIdUser); + result = catalogManager.getClinicalAnalysisManager().search(studyFqn, new Query(), QueryOptions.empty(), ownerToken); assertEquals(2, result.getNumResults()); } @@ -2806,10 +2955,10 @@ public void updateDisorder() throws CatalogException { ClinicalAnalysisUpdateParams updateParams = new ClinicalAnalysisUpdateParams() .setDisorder(new DisorderReferenceParam("dis1")); - catalogManager.getClinicalAnalysisManager().update(STUDY, dummyEnvironment.first().getId(), updateParams, QueryOptions.empty(), - sessionIdUser); - OpenCGAResult result1 = catalogManager.getClinicalAnalysisManager().get(STUDY, dummyEnvironment.first().getId(), - new QueryOptions(), sessionIdUser); + catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), updateParams, QueryOptions.empty(), + ownerToken); + OpenCGAResult result1 = catalogManager.getClinicalAnalysisManager().get(studyFqn, dummyEnvironment.first().getId(), + new QueryOptions(), ownerToken); assertEquals("dis1", result1.first().getDisorder().getId()); assertEquals("OT", result1.first().getDisorder().getSource()); @@ -2818,8 +2967,8 @@ public void updateDisorder() throws CatalogException { .setDisorder(new DisorderReferenceParam("non_existing")); thrown.expect(CatalogException.class); thrown.expectMessage("proband disorders"); - catalogManager.getClinicalAnalysisManager().update(STUDY, dummyEnvironment.first().getId(), updateParams, QueryOptions.empty(), - sessionIdUser); + catalogManager.getClinicalAnalysisManager().update(studyFqn, dummyEnvironment.first().getId(), updateParams, QueryOptions.empty(), + ownerToken); } @Test @@ -2847,8 +2996,8 @@ public void checkFamilyMembersOrder() throws CatalogException { .setDueDate("20180510100000") .setProband(new Individual().setId("child1")); clinicalAnalysis.setFamily(dummyFamily.first()); - DataResult clinicalAnalysisDataResult = catalogManager.getClinicalAnalysisManager().create(STUDY, - clinicalAnalysis, INCLUDE_RESULT, sessionIdUser); + DataResult clinicalAnalysisDataResult = catalogManager.getClinicalAnalysisManager().create(studyFqn, + clinicalAnalysis, INCLUDE_RESULT, ownerToken); assertTrue(clinicalAnalysisDataResult.first().getFamily().getMembers().stream().map(Individual::getId).collect(Collectors.toList()) .containsAll(dummyFamily.first().getMembers().stream().map(Individual::getId).collect(Collectors.toList()))); @@ -2858,9 +3007,8 @@ public void checkFamilyMembersOrder() throws CatalogException { @Test public void createClinicalAnalysisWithPanels() throws CatalogException { - catalogManager.getPanelManager().importFromSource(STUDY, "gene-census", "", sessionIdUser); - Panel panel = catalogManager.getPanelManager().search(STUDY, new Query(), QueryOptions.empty(), sessionIdUser).first(); - + catalogManager.getPanelManager().importFromSource(studyFqn, "gene-census", "", ownerToken); + Panel panel = catalogManager.getPanelManager().search(studyFqn, new Query(), QueryOptions.empty(), ownerToken).first(); DataResult dummyFamily = createDummyFamily(); // Leave only sample2 for child1 in family for (Individual member : dummyFamily.first().getMembers()) { @@ -2874,7 +3022,6 @@ public void createClinicalAnalysisWithPanels() throws CatalogException { member.setSamples(null); } } - ClinicalAnalysis clinicalAnalysis = new ClinicalAnalysis() .setId("analysis").setDescription("My description").setType(ClinicalAnalysis.Type.FAMILY) .setDueDate("20180510100000") @@ -2882,9 +3029,8 @@ public void createClinicalAnalysisWithPanels() throws CatalogException { .setFamily(dummyFamily.first()) .setProband(new Individual().setId("child1")); - ClinicalAnalysis ca = catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, INCLUDE_RESULT, - sessionIdUser).first(); - + ClinicalAnalysis ca = catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, INCLUDE_RESULT, + ownerToken).first(); assertEquals(1, ca.getPanels().size()); assertEquals(panel.getId(), ca.getPanels().get(0).getId()); assertEquals(panel.getName(), ca.getPanels().get(0).getName()); @@ -2894,17 +3040,17 @@ public void createClinicalAnalysisWithPanels() throws CatalogException { @Test public void createInterpretationWithPanels() throws CatalogException { - catalogManager.getPanelManager().importFromSource(STUDY, "gene-census", "", sessionIdUser); - Panel panel = catalogManager.getPanelManager().search(STUDY, new Query(), QueryOptions.empty(), sessionIdUser).first(); + catalogManager.getPanelManager().importFromSource(studyFqn, "gene-census", "", ownerToken); + Panel panel = catalogManager.getPanelManager().search(studyFqn, new Query(), QueryOptions.empty(), ownerToken).first(); ClinicalAnalysis ca = createDummyEnvironment(true, false).first(); Interpretation interpretation = new Interpretation() .setPanels(Collections.singletonList(new Panel().setId(panel.getId()))); - interpretation = catalogManager.getInterpretationManager().create(STUDY, ca.getId(), interpretation, - ParamUtils.SaveInterpretationAs.PRIMARY, INCLUDE_RESULT, sessionIdUser).first(); - interpretation = catalogManager.getInterpretationManager().get(STUDY, interpretation.getId(), QueryOptions.empty(), sessionIdUser) + interpretation = catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), interpretation, + ParamUtils.SaveInterpretationAs.PRIMARY, INCLUDE_RESULT, ownerToken).first(); + interpretation = catalogManager.getInterpretationManager().get(studyFqn, interpretation.getId(), QueryOptions.empty(), ownerToken) .first(); assertEquals(1, interpretation.getPanels().size()); @@ -2913,8 +3059,8 @@ public void createInterpretationWithPanels() throws CatalogException { @Test public void updatePanelsInClinicalAnalysis() throws CatalogException { - catalogManager.getPanelManager().importFromSource(STUDY, "gene-census", "", sessionIdUser); - Panel panel = catalogManager.getPanelManager().search(STUDY, new Query(), QueryOptions.empty(), sessionIdUser).first(); + catalogManager.getPanelManager().importFromSource(studyFqn, "gene-census", "", ownerToken); + Panel panel = catalogManager.getPanelManager().search(studyFqn, new Query(), QueryOptions.empty(), ownerToken).first(); DataResult dummyFamily = createDummyFamily(); // Leave only sample2 for child1 in family @@ -2937,13 +3083,13 @@ public void updatePanelsInClinicalAnalysis() throws CatalogException { .setProband(new Individual().setId("child1")); // Create without a panel and update the panel - catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, QueryOptions.empty(), sessionIdUser).first(); - catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysis.getId(), + catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, QueryOptions.empty(), ownerToken).first(); + catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams().setPanels(Collections.singletonList(new PanelReferenceParam(panel.getId()))), - QueryOptions.empty(), sessionIdUser); + QueryOptions.empty(), ownerToken); - ClinicalAnalysis ca = catalogManager.getClinicalAnalysisManager().get(STUDY, clinicalAnalysis.getId(), QueryOptions.empty(), - sessionIdUser).first(); + ClinicalAnalysis ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), + ownerToken).first(); assertEquals(1, ca.getPanels().size()); assertEquals(panel.getId(), ca.getPanels().get(0).getId()); @@ -2977,41 +3123,41 @@ public void testQueriesInFamilyCase() throws CatalogException { .setDueDate("20180510100000") .setProband(new Individual().setId("child1")); clinicalAnalysis.setFamily(dummyFamily.first()); - catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, QueryOptions.empty(), sessionIdUser); + catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, QueryOptions.empty(), ownerToken); - catalogManager.getFamilyManager().update(STUDY, dummyFamily.first().getId(), new FamilyUpdateParams() - .setId("familyId"), QueryOptions.empty(), sessionIdUser); + catalogManager.getFamilyManager().update(studyFqn, dummyFamily.first().getId(), new FamilyUpdateParams() + .setId("familyId"), QueryOptions.empty(), ownerToken); QueryOptions includeClinicalIds = ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS; // Query by members Query query = new Query(ClinicalAnalysisDBAdaptor.QueryParams.INDIVIDUAL.key(), "child3"); - OpenCGAResult search = catalogManager.getClinicalAnalysisManager().search(STUDY, query, includeClinicalIds, - sessionIdUser); + OpenCGAResult search = catalogManager.getClinicalAnalysisManager().search(studyFqn, query, includeClinicalIds, + ownerToken); assertEquals(1, search.getNumResults()); query = new Query(ClinicalAnalysisDBAdaptor.QueryParams.INDIVIDUAL.key(), "child1"); - search = catalogManager.getClinicalAnalysisManager().search(STUDY, query, includeClinicalIds, sessionIdUser); + search = catalogManager.getClinicalAnalysisManager().search(studyFqn, query, includeClinicalIds, ownerToken); assertEquals(1, search.getNumResults()); query = new Query(ClinicalAnalysisDBAdaptor.QueryParams.INDIVIDUAL.key(), "child4"); - search = catalogManager.getClinicalAnalysisManager().search(STUDY, query, includeClinicalIds, sessionIdUser); + search = catalogManager.getClinicalAnalysisManager().search(studyFqn, query, includeClinicalIds, ownerToken); assertEquals(0, search.getNumResults()); // Query by samples query = new Query(ParamConstants.CLINICAL_SAMPLE_PARAM, "sample2"); - search = catalogManager.getClinicalAnalysisManager().search(STUDY, query, includeClinicalIds, sessionIdUser); + search = catalogManager.getClinicalAnalysisManager().search(studyFqn, query, includeClinicalIds, ownerToken); assertEquals(1, search.getNumResults()); query = new Query(ParamConstants.CLINICAL_SAMPLE_PARAM, "sample5"); - search = catalogManager.getClinicalAnalysisManager().search(STUDY, query, includeClinicalIds, sessionIdUser); + search = catalogManager.getClinicalAnalysisManager().search(studyFqn, query, includeClinicalIds, ownerToken); assertEquals(1, search.getNumResults()); query = new Query(ParamConstants.CLINICAL_SAMPLE_PARAM, "sample4"); - search = catalogManager.getClinicalAnalysisManager().search(STUDY, query, includeClinicalIds, sessionIdUser); + search = catalogManager.getClinicalAnalysisManager().search(studyFqn, query, includeClinicalIds, ownerToken); assertEquals(0, search.getNumResults()); query = new Query(ClinicalAnalysisDBAdaptor.QueryParams.FAMILY.key(), "familyId"); - search = catalogManager.getClinicalAnalysisManager().search(STUDY, query, includeClinicalIds, sessionIdUser); + search = catalogManager.getClinicalAnalysisManager().search(studyFqn, query, includeClinicalIds, ownerToken); assertEquals(1, search.getNumResults()); } @@ -3021,28 +3167,28 @@ public void testQueriesInCancerCase() throws CatalogException { sample.setSomatic(true); Individual individual = DummyModelUtils.getDummyIndividual("individual", SexOntologyTermAnnotation.initMale(), Collections.singletonList(sample), null, null); - catalogManager.getIndividualManager().create(STUDY, individual, QueryOptions.empty(), sessionIdUser); + catalogManager.getIndividualManager().create(studyFqn, individual, QueryOptions.empty(), ownerToken); ClinicalAnalysis clinicalAnalysis = new ClinicalAnalysis() .setId("analysis").setDescription("My description").setType(ClinicalAnalysis.Type.CANCER) .setDueDate("20180510100000") .setProband(new Individual().setId(individual.getId())); - catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, QueryOptions.empty(), sessionIdUser); + catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, QueryOptions.empty(), ownerToken); // Update to force a version increment and therefore, an update over the case - catalogManager.getSampleManager().update(STUDY, sample.getId(), new SampleUpdateParams().setDescription("descr"), - QueryOptions.empty(), sessionIdUser); + catalogManager.getSampleManager().update(studyFqn, sample.getId(), new SampleUpdateParams().setDescription("descr"), + QueryOptions.empty(), ownerToken); QueryOptions includeClinicalIds = ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS; // Query by members Query query = new Query(ClinicalAnalysisDBAdaptor.QueryParams.INDIVIDUAL.key(), "individual"); - OpenCGAResult search = catalogManager.getClinicalAnalysisManager().search(STUDY, query, includeClinicalIds, - sessionIdUser); + OpenCGAResult search = catalogManager.getClinicalAnalysisManager().search(studyFqn, query, includeClinicalIds, + ownerToken); assertEquals(1, search.getNumResults()); assertEquals(clinicalAnalysis.getId(), search.first().getId()); query = new Query(ParamConstants.CLINICAL_SAMPLE_PARAM, "sample"); - search = catalogManager.getClinicalAnalysisManager().search(STUDY, query, includeClinicalIds, sessionIdUser); + search = catalogManager.getClinicalAnalysisManager().search(studyFqn, query, includeClinicalIds, ownerToken); assertEquals(1, search.getNumResults()); assertEquals(clinicalAnalysis.getId(), search.first().getId()); } @@ -3074,7 +3220,7 @@ public void sampleNotFoundInMember() throws CatalogException { clinicalAnalysis.setFamily(dummyFamily.first()); thrown.expect(CatalogException.class); thrown.expectMessage("could not be found in member"); - catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, QueryOptions.empty(), sessionIdUser); + catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, QueryOptions.empty(), ownerToken); } @Test @@ -3093,7 +3239,7 @@ public void checkMoreThanOneSample() throws CatalogException { clinicalAnalysis.setFamily(dummyFamily.first()); thrown.expect(CatalogException.class); thrown.expectMessage("More than one sample"); - catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, QueryOptions.empty(), sessionIdUser); + catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, QueryOptions.empty(), ownerToken); } @Test @@ -3104,7 +3250,7 @@ public void createClinicalAnalysisWithoutFamily() throws CatalogException { thrown.expect(CatalogException.class); thrown.expectMessage("missing"); - catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, QueryOptions.empty(), sessionIdUser); + catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, QueryOptions.empty(), ownerToken); } @Test @@ -3115,7 +3261,7 @@ public void createClinicalAnalysisWithoutProband() throws CatalogException { thrown.expect(CatalogException.class); thrown.expectMessage("missing"); - catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, QueryOptions.empty(), sessionIdUser); + catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, QueryOptions.empty(), ownerToken); } @Test @@ -3126,14 +3272,14 @@ public void createClinicalAnalysisWithoutType() throws CatalogException { thrown.expect(CatalogException.class); thrown.expectMessage("missing"); - catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, QueryOptions.empty(), sessionIdUser); + catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, QueryOptions.empty(), ownerToken); } private List createPanels(int nPanels) throws CatalogException { List panelList = new ArrayList<>(nPanels); for (int i = 0; i < nPanels; i++) { - panelList.add(catalogManager.getPanelManager().create(STUDY, new Panel().setId("panel" + i), INCLUDE_RESULT, - sessionIdUser).first()); + panelList.add(catalogManager.getPanelManager().create(studyFqn, new Panel().setId("panel" + i), INCLUDE_RESULT, + ownerToken).first()); } return panelList; } @@ -3141,11 +3287,11 @@ private List createPanels(int nPanels) throws CatalogException { @Test public void createClinicalAnalysisWithPanelsTest() throws CatalogException { List panels = createPanels(2); - Individual proband = catalogManager.getIndividualManager().create(STUDY, + Individual proband = catalogManager.getIndividualManager().create(studyFqn, new Individual() .setId("proband") .setSamples(Collections.singletonList(new Sample().setId("sample"))), - INCLUDE_RESULT, sessionIdUser).first(); + INCLUDE_RESULT, ownerToken).first(); ClinicalAnalysis clinicalAnalysis = new ClinicalAnalysis() .setId("analysis") @@ -3154,7 +3300,7 @@ public void createClinicalAnalysisWithPanelsTest() throws CatalogException { .setPanels(panels); OpenCGAResult result = - catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, INCLUDE_RESULT, sessionIdUser); + catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumResults()); assertEquals(2, result.first().getPanels().size()); for (Panel panel : result.first().getPanels()) { @@ -3165,28 +3311,28 @@ public void createClinicalAnalysisWithPanelsTest() throws CatalogException { assertNotNull(panel.getId()); assertNotNull(panel.getName()); } - assertFalse(result.first().isPanelLock()); + assertFalse(result.first().isPanelLocked()); } @Test public void fetchInterpretationWithFullPanelInformationTest() throws CatalogException { List panels = createPanels(2); - Individual proband = catalogManager.getIndividualManager().create(STUDY, + Individual proband = catalogManager.getIndividualManager().create(studyFqn, new Individual() .setId("proband") .setSamples(Collections.singletonList(new Sample().setId("sample"))), - INCLUDE_RESULT, sessionIdUser).first(); + INCLUDE_RESULT, ownerToken).first(); ClinicalAnalysis clinicalAnalysis = new ClinicalAnalysis() .setId("analysis") .setType(ClinicalAnalysis.Type.SINGLE) .setProband(proband) .setPanels(panels); OpenCGAResult result = - catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, INCLUDE_RESULT, sessionIdUser); + catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, INCLUDE_RESULT, ownerToken); - Interpretation interpretation = catalogManager.getInterpretationManager().search(STUDY, + Interpretation interpretation = catalogManager.getInterpretationManager().search(studyFqn, new Query(InterpretationDBAdaptor.QueryParams.CLINICAL_ANALYSIS_ID.key(), clinicalAnalysis.getId()), QueryOptions.empty(), - sessionIdUser).first(); + ownerToken).first(); assertEquals(2, interpretation.getPanels().size()); for (Panel panel : interpretation.getPanels()) { assertNotNull(panel.getId()); @@ -3197,11 +3343,11 @@ public void fetchInterpretationWithFullPanelInformationTest() throws CatalogExce @Test public void updatePanelsActionTest() throws CatalogException { List panels = createPanels(5); - Individual proband = catalogManager.getIndividualManager().create(STUDY, + Individual proband = catalogManager.getIndividualManager().create(studyFqn, new Individual() .setId("proband") .setSamples(Collections.singletonList(new Sample().setId("sample"))), - INCLUDE_RESULT, sessionIdUser).first(); + INCLUDE_RESULT, ownerToken).first(); ClinicalAnalysis clinicalAnalysis = new ClinicalAnalysis() .setId("analysis") @@ -3210,17 +3356,17 @@ public void updatePanelsActionTest() throws CatalogException { .setProband(proband); OpenCGAResult result = - catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, INCLUDE_RESULT, sessionIdUser); + catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumResults()); assertEquals(2, result.first().getPanels().size()); Map actionMap = new HashMap<>(); actionMap.put(ClinicalAnalysisDBAdaptor.QueryParams.PANELS.key(), ParamUtils.BasicUpdateAction.ADD); QueryOptions options = new QueryOptions(Constants.ACTIONS, actionMap); - catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() + catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() .setPanels(Collections.singletonList(new PanelReferenceParam(panels.get(2).getId()))), - options, sessionIdUser); - result = catalogManager.getClinicalAnalysisManager().get(STUDY, clinicalAnalysis.getId(), QueryOptions.empty(), sessionIdUser); + options, ownerToken); + result = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumResults()); assertEquals(3, result.first().getPanels().size()); assertTrue(panels.subList(0, 3).stream().map(Panel::getId).collect(Collectors.toList()).containsAll( @@ -3229,13 +3375,13 @@ public void updatePanelsActionTest() throws CatalogException { actionMap = new HashMap<>(); actionMap.put(ClinicalAnalysisDBAdaptor.QueryParams.PANELS.key(), ParamUtils.BasicUpdateAction.REMOVE); options = new QueryOptions(Constants.ACTIONS, actionMap); - catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() + catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() .setPanels(Arrays.asList( new PanelReferenceParam(panels.get(0).getId()), new PanelReferenceParam(panels.get(2).getId())) ), - options, sessionIdUser); - result = catalogManager.getClinicalAnalysisManager().get(STUDY, clinicalAnalysis.getId(), QueryOptions.empty(), sessionIdUser); + options, ownerToken); + result = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumResults()); assertEquals(1, result.first().getPanels().size()); assertEquals(panels.get(1).getId(), result.first().getPanels().get(0).getId()); @@ -3243,13 +3389,13 @@ public void updatePanelsActionTest() throws CatalogException { actionMap = new HashMap<>(); actionMap.put(ClinicalAnalysisDBAdaptor.QueryParams.PANELS.key(), ParamUtils.BasicUpdateAction.SET); options = new QueryOptions(Constants.ACTIONS, actionMap); - catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() + catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() .setPanels(Arrays.asList( new PanelReferenceParam(panels.get(3).getId()), new PanelReferenceParam(panels.get(4).getId())) ), - options, sessionIdUser); - result = catalogManager.getClinicalAnalysisManager().get(STUDY, clinicalAnalysis.getId(), QueryOptions.empty(), sessionIdUser); + options, ownerToken); + result = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumResults()); assertEquals(2, result.first().getPanels().size()); assertTrue(panels.subList(3, 5).stream().map(Panel::getId).collect(Collectors.toList()).containsAll( @@ -3260,11 +3406,11 @@ public void updatePanelsActionTest() throws CatalogException { @Test public void updateInterpretationPanelsActionTest() throws CatalogException { List panels = createPanels(5); - Individual proband = catalogManager.getIndividualManager().create(STUDY, + Individual proband = catalogManager.getIndividualManager().create(studyFqn, new Individual() .setId("proband") .setSamples(Collections.singletonList(new Sample().setId("sample"))), - INCLUDE_RESULT, sessionIdUser).first(); + INCLUDE_RESULT, ownerToken).first(); ClinicalAnalysis clinicalAnalysis = new ClinicalAnalysis() .setId("analysis") @@ -3273,7 +3419,7 @@ public void updateInterpretationPanelsActionTest() throws CatalogException { .setProband(proband); OpenCGAResult result = - catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, INCLUDE_RESULT, sessionIdUser); + catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumResults()); assertEquals(0, result.first().getPanels().size()); @@ -3283,10 +3429,10 @@ public void updateInterpretationPanelsActionTest() throws CatalogException { QueryOptions options = new QueryOptions() .append(Constants.ACTIONS, actionMap) .append(ParamConstants.INCLUDE_RESULT_PARAM, true); - OpenCGAResult interpretation = catalogManager.getInterpretationManager().update(STUDY, clinicalAnalysis.getId(), + OpenCGAResult interpretation = catalogManager.getInterpretationManager().update(studyFqn, clinicalAnalysis.getId(), intepretationId, new InterpretationUpdateParams() .setPanels(panels.subList(0, 2).stream().map((p) -> new PanelReferenceParam(p.getId())).collect(Collectors.toList())), - null, options, sessionIdUser); + null, options, ownerToken); assertEquals(2, interpretation.first().getPanels().size()); actionMap = new HashMap<>(); @@ -3294,10 +3440,10 @@ intepretationId, new InterpretationUpdateParams() options = new QueryOptions() .append(Constants.ACTIONS, actionMap) .append(ParamConstants.INCLUDE_RESULT_PARAM, true); - interpretation = catalogManager.getInterpretationManager().update(STUDY, clinicalAnalysis.getId(), + interpretation = catalogManager.getInterpretationManager().update(studyFqn, clinicalAnalysis.getId(), intepretationId, new InterpretationUpdateParams() .setPanels(Collections.singletonList(new PanelReferenceParam(panels.get(2).getId()))), - null, options, sessionIdUser); + null, options, ownerToken); assertEquals(1, interpretation.getNumResults()); assertEquals(3, interpretation.first().getPanels().size()); assertTrue(panels.subList(0, 3).stream().map(Panel::getId).collect(Collectors.toList()).containsAll( @@ -3308,13 +3454,13 @@ intepretationId, new InterpretationUpdateParams() options = new QueryOptions() .append(Constants.ACTIONS, actionMap) .append(ParamConstants.INCLUDE_RESULT_PARAM, true); - interpretation = catalogManager.getInterpretationManager().update(STUDY, clinicalAnalysis.getId(), + interpretation = catalogManager.getInterpretationManager().update(studyFqn, clinicalAnalysis.getId(), intepretationId, new InterpretationUpdateParams() .setPanels(Arrays.asList( new PanelReferenceParam(panels.get(0).getId()), new PanelReferenceParam(panels.get(2).getId())) ), - null, options, sessionIdUser); + null, options, ownerToken); assertEquals(1, interpretation.getNumResults()); assertEquals(1, interpretation.first().getPanels().size()); assertEquals(panels.get(1).getId(), interpretation.first().getPanels().get(0).getId()); @@ -3324,13 +3470,13 @@ intepretationId, new InterpretationUpdateParams() options = new QueryOptions() .append(Constants.ACTIONS, actionMap) .append(ParamConstants.INCLUDE_RESULT_PARAM, true); - interpretation = catalogManager.getInterpretationManager().update(STUDY, clinicalAnalysis.getId(), + interpretation = catalogManager.getInterpretationManager().update(studyFqn, clinicalAnalysis.getId(), intepretationId, new InterpretationUpdateParams() .setPanels(Arrays.asList( new PanelReferenceParam(panels.get(3).getId()), new PanelReferenceParam(panels.get(4).getId())) ), - null, options, sessionIdUser); + null, options, ownerToken); assertEquals(1, interpretation.getNumResults()); assertEquals(2, interpretation.first().getPanels().size()); assertTrue(panels.subList(3, 5).stream().map(Panel::getId).collect(Collectors.toList()).containsAll( @@ -3341,11 +3487,11 @@ intepretationId, new InterpretationUpdateParams() @Test public void updatePanelsAndPanelLockFromClinicalAnalysisTest() throws CatalogException { List panels = createPanels(2); - Individual proband = catalogManager.getIndividualManager().create(STUDY, + Individual proband = catalogManager.getIndividualManager().create(studyFqn, new Individual() .setId("proband") .setSamples(Collections.singletonList(new Sample().setId("sample"))), - INCLUDE_RESULT, sessionIdUser).first(); + INCLUDE_RESULT, ownerToken).first(); ClinicalAnalysis clinicalAnalysis = new ClinicalAnalysis() .setId("analysis") @@ -3353,63 +3499,63 @@ public void updatePanelsAndPanelLockFromClinicalAnalysisTest() throws CatalogExc .setProband(proband); OpenCGAResult result = - catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, true, INCLUDE_RESULT, sessionIdUser); + catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, true, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumResults()); assertEquals(0, result.first().getPanels().size()); - assertFalse(result.first().isPanelLock()); + assertFalse(result.first().isPanelLocked()); Map actionMap = new HashMap<>(); actionMap.put(ClinicalAnalysisDBAdaptor.QueryParams.PANELS.key(), ParamUtils.BasicUpdateAction.SET); QueryOptions updateOptions = new QueryOptions(Constants.ACTIONS, actionMap); try { - catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() + catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() .setPanels(panels.stream().map(p -> new PanelReferenceParam(p.getId())).collect(Collectors.toList())) - .setPanelLock(true), - updateOptions, sessionIdUser); + .setPanelLocked(true), + updateOptions, ownerToken); fail("Updating panels and setting panellock to true in one call should not be accepted"); } catch (CatalogException e) { assertTrue(e.getMessage().contains("not allowed")); } - catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() + catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() .setPanels(panels.stream().map(p -> new PanelReferenceParam(p.getId())).collect(Collectors.toList())), - updateOptions, sessionIdUser); - catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() - .setPanelLock(true), - updateOptions, sessionIdUser); - result = catalogManager.getClinicalAnalysisManager().get(STUDY, clinicalAnalysis.getId(), QueryOptions.empty(), sessionIdUser); + updateOptions, ownerToken); + catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() + .setPanelLocked(true), + updateOptions, ownerToken); + result = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumResults()); assertEquals(2, result.first().getPanels().size()); - assertTrue(result.first().isPanelLock()); + assertTrue(result.first().isPanelLocked()); - catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() + catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() .setPanels(Collections.singletonList(new PanelReferenceParam(panels.get(0).getId()))) - .setPanelLock(false), - updateOptions, sessionIdUser); - result = catalogManager.getClinicalAnalysisManager().get(STUDY, clinicalAnalysis.getId(), QueryOptions.empty(), sessionIdUser); + .setPanelLocked(false), + updateOptions, ownerToken); + result = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumResults()); assertEquals(1, result.first().getPanels().size()); - assertFalse(result.first().isPanelLock()); + assertFalse(result.first().isPanelLocked()); - catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() - .setPanelLock(true), - updateOptions, sessionIdUser); + catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() + .setPanelLocked(true), + updateOptions, ownerToken); thrown.expect(CatalogException.class); thrown.expectMessage("panelLock"); - catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() + catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() .setPanels(panels.stream().map(p -> new PanelReferenceParam(p.getId())).collect(Collectors.toList())), - updateOptions, sessionIdUser); + updateOptions, ownerToken); } @Test public void updatePanelsAndPanelLockFromClinicalAnalysisWithInterpretationTest() throws CatalogException { List panels = createPanels(2); - Individual proband = catalogManager.getIndividualManager().create(STUDY, + Individual proband = catalogManager.getIndividualManager().create(studyFqn, new Individual() .setId("proband") .setSamples(Collections.singletonList(new Sample().setId("sample"))), - INCLUDE_RESULT, sessionIdUser).first(); + INCLUDE_RESULT, ownerToken).first(); ClinicalAnalysis clinicalAnalysis = new ClinicalAnalysis() .setId("analysis") @@ -3417,27 +3563,27 @@ public void updatePanelsAndPanelLockFromClinicalAnalysisWithInterpretationTest() .setProband(proband); OpenCGAResult result = - catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, null, INCLUDE_RESULT, sessionIdUser); + catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, null, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumResults()); assertEquals(0, result.first().getPanels().size()); - assertFalse(result.first().isPanelLock()); + assertFalse(result.first().isPanelLocked()); thrown.expect(CatalogException.class); thrown.expectMessage("not allowed"); - catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() + catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() .setPanels(panels.stream().map(p -> new PanelReferenceParam(p.getId())).collect(Collectors.toList())) - .setPanelLock(true), - QueryOptions.empty(), sessionIdUser); + .setPanelLocked(true), + QueryOptions.empty(), ownerToken); } @Test public void setPanelLockWithInterpretationWithNoPanelsTest() throws CatalogException { List panels = createPanels(2); - Individual proband = catalogManager.getIndividualManager().create(STUDY, + Individual proband = catalogManager.getIndividualManager().create(studyFqn, new Individual() .setId("proband") .setSamples(Collections.singletonList(new Sample().setId("sample"))), - INCLUDE_RESULT, sessionIdUser).first(); + INCLUDE_RESULT, ownerToken).first(); ClinicalAnalysis clinicalAnalysis = new ClinicalAnalysis() .setId("analysis") @@ -3445,33 +3591,33 @@ public void setPanelLockWithInterpretationWithNoPanelsTest() throws CatalogExcep .setProband(proband); OpenCGAResult result = - catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, null, INCLUDE_RESULT, sessionIdUser); + catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, null, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumResults()); assertEquals(0, result.first().getPanels().size()); - assertFalse(result.first().isPanelLock()); + assertFalse(result.first().isPanelLocked()); - catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() + catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() .setPanels(panels.stream().map(p -> new PanelReferenceParam(p.getId())).collect(Collectors.toList())), - QueryOptions.empty(), sessionIdUser); - clinicalAnalysis = catalogManager.getClinicalAnalysisManager().get(STUDY, clinicalAnalysis.getId(), QueryOptions.empty(), sessionIdUser).first(); + QueryOptions.empty(), ownerToken); + clinicalAnalysis = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(2, clinicalAnalysis.getPanels().size()); assertEquals(0, clinicalAnalysis.getInterpretation().getPanels().size()); thrown.expect(CatalogException.class); thrown.expectMessage("any of the case panels"); - catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() - .setPanelLock(true), - QueryOptions.empty(), sessionIdUser); + catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() + .setPanelLocked(true), + QueryOptions.empty(), ownerToken); } @Test public void setPanelLockWithInterpretationWithPanelSubsetTest() throws CatalogException { List panels = createPanels(2); - Individual proband = catalogManager.getIndividualManager().create(STUDY, + Individual proband = catalogManager.getIndividualManager().create(studyFqn, new Individual() .setId("proband") .setSamples(Collections.singletonList(new Sample().setId("sample"))), - INCLUDE_RESULT, sessionIdUser).first(); + INCLUDE_RESULT, ownerToken).first(); ClinicalAnalysis clinicalAnalysis = new ClinicalAnalysis() .setId("analysis") @@ -3479,32 +3625,32 @@ public void setPanelLockWithInterpretationWithPanelSubsetTest() throws CatalogEx .setProband(proband); OpenCGAResult result = - catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, null, INCLUDE_RESULT, sessionIdUser); + catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, null, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumResults()); assertEquals(0, result.first().getPanels().size()); - assertFalse(result.first().isPanelLock()); + assertFalse(result.first().isPanelLocked()); - catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() + catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() .setPanels(panels.stream().map(p -> new PanelReferenceParam(p.getId())).collect(Collectors.toList())), - QueryOptions.empty(), sessionIdUser); - clinicalAnalysis = catalogManager.getClinicalAnalysisManager().get(STUDY, clinicalAnalysis.getId(), QueryOptions.empty(), sessionIdUser).first(); + QueryOptions.empty(), ownerToken); + clinicalAnalysis = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(2, clinicalAnalysis.getPanels().size()); assertEquals(0, clinicalAnalysis.getInterpretation().getPanels().size()); - catalogManager.getInterpretationManager().update(STUDY, clinicalAnalysis.getId(), clinicalAnalysis.getInterpretation().getId(), + catalogManager.getInterpretationManager().update(studyFqn, clinicalAnalysis.getId(), clinicalAnalysis.getInterpretation().getId(), new InterpretationUpdateParams() .setPanels(Collections.singletonList(new PanelReferenceParam(panels.get(0).getId()))), null, - QueryOptions.empty(), sessionIdUser); - clinicalAnalysis = catalogManager.getClinicalAnalysisManager().get(STUDY, clinicalAnalysis.getId(), QueryOptions.empty(), sessionIdUser).first(); - assertFalse(clinicalAnalysis.isPanelLock()); + QueryOptions.empty(), ownerToken); + clinicalAnalysis = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), ownerToken).first(); + assertFalse(clinicalAnalysis.isPanelLocked()); assertEquals(2, clinicalAnalysis.getPanels().size()); assertEquals(1, clinicalAnalysis.getInterpretation().getPanels().size()); - catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() - .setPanelLock(true), - QueryOptions.empty(), sessionIdUser); - clinicalAnalysis = catalogManager.getClinicalAnalysisManager().get(STUDY, clinicalAnalysis.getId(), QueryOptions.empty(), sessionIdUser).first(); - assertTrue(clinicalAnalysis.isPanelLock()); + catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() + .setPanelLocked(true), + QueryOptions.empty(), ownerToken); + clinicalAnalysis = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), ownerToken).first(); + assertTrue(clinicalAnalysis.isPanelLocked()); assertEquals(2, clinicalAnalysis.getPanels().size()); assertEquals(1, clinicalAnalysis.getInterpretation().getPanels().size()); } @@ -3512,11 +3658,11 @@ public void setPanelLockWithInterpretationWithPanelSubsetTest() throws CatalogEx @Test public void setPanelLockWithInterpretationWithDifferentPanelsTest() throws CatalogException { List panels = createPanels(2); - Individual proband = catalogManager.getIndividualManager().create(STUDY, + Individual proband = catalogManager.getIndividualManager().create(studyFqn, new Individual() .setId("proband") .setSamples(Collections.singletonList(new Sample().setId("sample"))), - INCLUDE_RESULT, sessionIdUser).first(); + INCLUDE_RESULT, ownerToken).first(); ClinicalAnalysis clinicalAnalysis = new ClinicalAnalysis() .setId("analysis") @@ -3524,41 +3670,41 @@ public void setPanelLockWithInterpretationWithDifferentPanelsTest() throws Catal .setProband(proband); OpenCGAResult result = - catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, null, INCLUDE_RESULT, sessionIdUser); + catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, null, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumResults()); assertEquals(0, result.first().getPanels().size()); - assertFalse(result.first().isPanelLock()); + assertFalse(result.first().isPanelLocked()); - catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() + catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() .setPanels(Collections.singletonList(new PanelReferenceParam(panels.get(0).getId()))), - QueryOptions.empty(), sessionIdUser); - clinicalAnalysis = catalogManager.getClinicalAnalysisManager().get(STUDY, clinicalAnalysis.getId(), QueryOptions.empty(), sessionIdUser).first(); + QueryOptions.empty(), ownerToken); + clinicalAnalysis = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(1, clinicalAnalysis.getPanels().size()); assertEquals(0, clinicalAnalysis.getInterpretation().getPanels().size()); - catalogManager.getInterpretationManager().update(STUDY, clinicalAnalysis.getId(), clinicalAnalysis.getInterpretation().getId(), + catalogManager.getInterpretationManager().update(studyFqn, clinicalAnalysis.getId(), clinicalAnalysis.getInterpretation().getId(), new InterpretationUpdateParams() .setPanels(panels.stream().map(p -> new PanelReferenceParam(p.getId())).collect(Collectors.toList())), null, - QueryOptions.empty(), sessionIdUser); - clinicalAnalysis = catalogManager.getClinicalAnalysisManager().get(STUDY, clinicalAnalysis.getId(), QueryOptions.empty(), sessionIdUser).first(); + QueryOptions.empty(), ownerToken); + clinicalAnalysis = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(1, clinicalAnalysis.getPanels().size()); assertEquals(2, clinicalAnalysis.getInterpretation().getPanels().size()); thrown.expect(CatalogException.class); thrown.expectMessage("not defined by the case"); - catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() - .setPanelLock(true), - QueryOptions.empty(), sessionIdUser); + catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() + .setPanelLocked(true), + QueryOptions.empty(), ownerToken); } @Test public void updatePanelsFromClinicalAnalysisWithPanelLockTest() throws CatalogException { List panels = createPanels(3); - Individual proband = catalogManager.getIndividualManager().create(STUDY, + Individual proband = catalogManager.getIndividualManager().create(studyFqn, new Individual() .setId("proband") .setSamples(Collections.singletonList(new Sample().setId("sample"))), - INCLUDE_RESULT, sessionIdUser).first(); + INCLUDE_RESULT, ownerToken).first(); ClinicalAnalysis clinicalAnalysis = new ClinicalAnalysis() .setId("analysis") @@ -3567,36 +3713,36 @@ public void updatePanelsFromClinicalAnalysisWithPanelLockTest() throws CatalogEx .setPanels(panels.subList(0, 2)); OpenCGAResult result = - catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, INCLUDE_RESULT, sessionIdUser); + catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumResults()); assertEquals(2, result.first().getPanels().size()); - assertFalse(result.first().isPanelLock()); + assertFalse(result.first().isPanelLocked()); - Interpretation interpretation = catalogManager.getInterpretationManager().create(STUDY, clinicalAnalysis.getId(), - new Interpretation(), ParamUtils.SaveInterpretationAs.PRIMARY, INCLUDE_RESULT, sessionIdUser).first(); + Interpretation interpretation = catalogManager.getInterpretationManager().create(studyFqn, clinicalAnalysis.getId(), + new Interpretation(), ParamUtils.SaveInterpretationAs.PRIMARY, INCLUDE_RESULT, ownerToken).first(); assertEquals(2, interpretation.getPanels().size()); // Set panelLock to true - catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() - .setPanelLock(true), QueryOptions.empty(), sessionIdUser); - clinicalAnalysis = catalogManager.getClinicalAnalysisManager().get(STUDY, clinicalAnalysis.getId(), QueryOptions.empty(), sessionIdUser).first(); - assertTrue(clinicalAnalysis.isPanelLock()); + catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() + .setPanelLocked(true), QueryOptions.empty(), ownerToken); + clinicalAnalysis = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), ownerToken).first(); + assertTrue(clinicalAnalysis.isPanelLocked()); thrown.expect(CatalogException.class); thrown.expectMessage("panelLock"); - catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() + catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() .setPanels(Collections.singletonList(new PanelReferenceParam(panels.get(2).getId()))), - QueryOptions.empty(), sessionIdUser); + QueryOptions.empty(), ownerToken); } @Test public void updatePanelLockWithDifferentPanels() throws CatalogException { List panels = createPanels(3); - Individual proband = catalogManager.getIndividualManager().create(STUDY, + Individual proband = catalogManager.getIndividualManager().create(studyFqn, new Individual() .setId("proband") .setSamples(Collections.singletonList(new Sample().setId("sample"))), - INCLUDE_RESULT, sessionIdUser).first(); + INCLUDE_RESULT, ownerToken).first(); ClinicalAnalysis clinicalAnalysis = new ClinicalAnalysis() .setId("analysis") @@ -3605,87 +3751,87 @@ public void updatePanelLockWithDifferentPanels() throws CatalogException { .setPanels(panels.subList(0, 2)); OpenCGAResult result = - catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, INCLUDE_RESULT, sessionIdUser); + catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumResults()); assertEquals(2, result.first().getPanels().size()); - assertFalse(result.first().isPanelLock()); + assertFalse(result.first().isPanelLocked()); - Interpretation interpretation = catalogManager.getInterpretationManager().create(STUDY, clinicalAnalysis.getId(), - new Interpretation(), ParamUtils.SaveInterpretationAs.PRIMARY, INCLUDE_RESULT, sessionIdUser).first(); + Interpretation interpretation = catalogManager.getInterpretationManager().create(studyFqn, clinicalAnalysis.getId(), + new Interpretation(), ParamUtils.SaveInterpretationAs.PRIMARY, INCLUDE_RESULT, ownerToken).first(); assertEquals(2, interpretation.getPanels().size()); // Set panelLock to true - catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() - .setPanelLock(true), QueryOptions.empty(), sessionIdUser); - clinicalAnalysis = catalogManager.getClinicalAnalysisManager().get(STUDY, clinicalAnalysis.getId(), QueryOptions.empty(), sessionIdUser).first(); - assertTrue(clinicalAnalysis.isPanelLock()); + catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() + .setPanelLocked(true), QueryOptions.empty(), ownerToken); + clinicalAnalysis = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), ownerToken).first(); + assertTrue(clinicalAnalysis.isPanelLocked()); // Set panelLock to false - catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() - .setPanelLock(false), QueryOptions.empty(), sessionIdUser); - clinicalAnalysis = catalogManager.getClinicalAnalysisManager().get(STUDY, clinicalAnalysis.getId(), QueryOptions.empty(), sessionIdUser).first(); - assertFalse(clinicalAnalysis.isPanelLock()); + catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() + .setPanelLocked(false), QueryOptions.empty(), ownerToken); + clinicalAnalysis = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), ownerToken).first(); + assertFalse(clinicalAnalysis.isPanelLocked()); Map actionMap = new HashMap<>(); actionMap.put(ClinicalAnalysisDBAdaptor.QueryParams.PANELS.key(), ParamUtils.BasicUpdateAction.SET); QueryOptions options = new QueryOptions(Constants.ACTIONS, actionMap); - catalogManager.getInterpretationManager().update(STUDY, clinicalAnalysis.getId(), interpretation.getId(), + catalogManager.getInterpretationManager().update(studyFqn, clinicalAnalysis.getId(), interpretation.getId(), new InterpretationUpdateParams().setPanels(Collections.singletonList(new PanelReferenceParam(panels.get(2).getId()))), - null, options, sessionIdUser); - interpretation = catalogManager.getInterpretationManager().get(STUDY, interpretation.getId(), QueryOptions.empty(), sessionIdUser).first(); + null, options, ownerToken); + interpretation = catalogManager.getInterpretationManager().get(studyFqn, interpretation.getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(1, interpretation.getPanels().size()); assertEquals(panels.get(2).getId(), interpretation.getPanels().get(0).getId()); thrown.expect(CatalogException.class); thrown.expectMessage("panels"); // Set panelLock to true - catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() - .setPanelLock(true), QueryOptions.empty(), sessionIdUser); - clinicalAnalysis = catalogManager.getClinicalAnalysisManager().get(STUDY, clinicalAnalysis.getId(), QueryOptions.empty(), sessionIdUser).first(); - assertTrue(clinicalAnalysis.isPanelLock()); + catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() + .setPanelLocked(true), QueryOptions.empty(), ownerToken); + clinicalAnalysis = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), ownerToken).first(); + assertTrue(clinicalAnalysis.isPanelLocked()); } @Test public void updatePanelsFromInterpretationWithLockedCATest() throws CatalogException { List panels = createPanels(3); - Individual proband = catalogManager.getIndividualManager().create(STUDY, + Individual proband = catalogManager.getIndividualManager().create(studyFqn, new Individual() .setId("proband") .setSamples(Collections.singletonList(new Sample().setId("sample"))), - INCLUDE_RESULT, sessionIdUser).first(); + INCLUDE_RESULT, ownerToken).first(); ClinicalAnalysis clinicalAnalysis = new ClinicalAnalysis() .setId("analysis") .setType(ClinicalAnalysis.Type.SINGLE) .setProband(proband) - .setPanelLock(true) + .setPanelLocked(true) .setPanels(panels.subList(0, 2)); OpenCGAResult result = - catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, INCLUDE_RESULT, sessionIdUser); + catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumResults()); assertEquals(2, result.first().getPanels().size()); - assertTrue(result.first().isPanelLock()); + assertTrue(result.first().isPanelLocked()); - Interpretation interpretation = catalogManager.getInterpretationManager().create(STUDY, clinicalAnalysis.getId(), - new Interpretation(), ParamUtils.SaveInterpretationAs.PRIMARY, INCLUDE_RESULT, sessionIdUser).first(); + Interpretation interpretation = catalogManager.getInterpretationManager().create(studyFqn, clinicalAnalysis.getId(), + new Interpretation(), ParamUtils.SaveInterpretationAs.PRIMARY, INCLUDE_RESULT, ownerToken).first(); assertEquals(2, interpretation.getPanels().size()); thrown.expect(CatalogException.class); thrown.expectMessage("panelLock"); - catalogManager.getInterpretationManager().update(STUDY, clinicalAnalysis.getId(), interpretation.getId(), + catalogManager.getInterpretationManager().update(studyFqn, clinicalAnalysis.getId(), interpretation.getId(), new InterpretationUpdateParams().setPanels(Collections.singletonList(new PanelReferenceParam(panels.get(2).getId()))), - null, QueryOptions.empty(), sessionIdUser); + null, QueryOptions.empty(), ownerToken); } @Test public void updatePanelsFromInterpretationWithUnlockedCATest() throws CatalogException { List panels = createPanels(3); - Individual proband = catalogManager.getIndividualManager().create(STUDY, + Individual proband = catalogManager.getIndividualManager().create(studyFqn, new Individual() .setId("proband") .setSamples(Collections.singletonList(new Sample().setId("sample"))), - INCLUDE_RESULT, sessionIdUser).first(); + INCLUDE_RESULT, ownerToken).first(); ClinicalAnalysis clinicalAnalysis = new ClinicalAnalysis() .setId("analysis") @@ -3694,21 +3840,21 @@ public void updatePanelsFromInterpretationWithUnlockedCATest() throws CatalogExc .setPanels(panels.subList(0, 2)); OpenCGAResult result = - catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, INCLUDE_RESULT, sessionIdUser); + catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumResults()); assertEquals(2, result.first().getPanels().size()); - assertFalse(result.first().isPanelLock()); + assertFalse(result.first().isPanelLocked()); - Interpretation interpretation = catalogManager.getInterpretationManager().create(STUDY, clinicalAnalysis.getId(), - new Interpretation(), ParamUtils.SaveInterpretationAs.PRIMARY, INCLUDE_RESULT, sessionIdUser).first(); + Interpretation interpretation = catalogManager.getInterpretationManager().create(studyFqn, clinicalAnalysis.getId(), + new Interpretation(), ParamUtils.SaveInterpretationAs.PRIMARY, INCLUDE_RESULT, ownerToken).first(); assertEquals(2, interpretation.getPanels().size()); // Ensure this fails - catalogManager.getInterpretationManager().update(STUDY, clinicalAnalysis.getId(), interpretation.getId(), + catalogManager.getInterpretationManager().update(studyFqn, clinicalAnalysis.getId(), interpretation.getId(), new InterpretationUpdateParams().setPanels(Collections.singletonList(new PanelReferenceParam(panels.get(2).getId()))), - null, QueryOptions.empty(), sessionIdUser); - interpretation = catalogManager.getInterpretationManager().get(STUDY, interpretation.getId(), QueryOptions.empty(), - sessionIdUser).first(); + null, QueryOptions.empty(), ownerToken); + interpretation = catalogManager.getInterpretationManager().get(studyFqn, interpretation.getId(), QueryOptions.empty(), + ownerToken).first(); assertEquals(3, interpretation.getPanels().size()); } @@ -3718,21 +3864,21 @@ public void createClinicalAnalysisWithFiles() throws CatalogException { Individual individual = new Individual() .setId("proband") .setSamples(Collections.singletonList(new Sample().setId("sample"))); - catalogManager.getIndividualManager().create(STUDY, individual, QueryOptions.empty(), sessionIdUser); + catalogManager.getIndividualManager().create(studyFqn, individual, QueryOptions.empty(), ownerToken); // Register and associate files to sample "sample" List files = registerDummyFiles(); for (File file : files) { - catalogManager.getFileManager().update(STUDY, file.getPath(), - new FileUpdateParams().setSampleIds(Collections.singletonList("sample")), QueryOptions.empty(), sessionIdUser); + catalogManager.getFileManager().update(studyFqn, file.getPath(), + new FileUpdateParams().setSampleIds(Collections.singletonList("sample")), QueryOptions.empty(), ownerToken); } ClinicalAnalysis clinicalAnalysis = new ClinicalAnalysis() .setId("Clinical") .setType(ClinicalAnalysis.Type.SINGLE) .setProband(individual); - OpenCGAResult clinical = catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, - INCLUDE_RESULT, sessionIdUser); + OpenCGAResult clinical = catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, + INCLUDE_RESULT, ownerToken); assertEquals(1, clinical.getNumResults()); assertEquals(4, clinical.first().getFiles().size()); for (File file : clinical.first().getFiles()) { @@ -3746,13 +3892,13 @@ public void updateClinicalAnalysisFiles() throws CatalogException { Individual individual = new Individual() .setId("proband") .setSamples(Collections.singletonList(new Sample().setId("sample"))); - catalogManager.getIndividualManager().create(STUDY, individual, QueryOptions.empty(), sessionIdUser); + catalogManager.getIndividualManager().create(studyFqn, individual, QueryOptions.empty(), ownerToken); // Register and associate files to sample "sample" List files = registerDummyFiles(); for (File file : files) { - catalogManager.getFileManager().update(STUDY, file.getPath(), - new FileUpdateParams().setSampleIds(Collections.singletonList("sample")), QueryOptions.empty(), sessionIdUser); + catalogManager.getFileManager().update(studyFqn, file.getPath(), + new FileUpdateParams().setSampleIds(Collections.singletonList("sample")), QueryOptions.empty(), ownerToken); } List fileRefs = files.stream().map(f -> new FileReferenceParam(f.getPath())).collect(Collectors.toList()); @@ -3760,18 +3906,18 @@ public void updateClinicalAnalysisFiles() throws CatalogException { .setId("Clinical") .setType(ClinicalAnalysis.Type.SINGLE) .setProband(individual); - OpenCGAResult clinical = catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, - INCLUDE_RESULT, sessionIdUser); + OpenCGAResult clinical = catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, + INCLUDE_RESULT, ownerToken); assertEquals(1, clinical.getNumResults()); assertEquals(4, clinical.first().getFiles().size()); // Remove first and last file ObjectMap actionMap = new ObjectMap(ClinicalAnalysisDBAdaptor.QueryParams.FILES.key(), ParamUtils.BasicUpdateAction.REMOVE); QueryOptions options = new QueryOptions(Constants.ACTIONS, actionMap); - catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysis.getId(), - new ClinicalAnalysisUpdateParams().setFiles(Arrays.asList(fileRefs.get(0), fileRefs.get(3))), options, sessionIdUser); - ClinicalAnalysis ca = catalogManager.getClinicalAnalysisManager().get(STUDY, clinicalAnalysis.getId(), QueryOptions.empty(), - sessionIdUser).first(); + catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), + new ClinicalAnalysisUpdateParams().setFiles(Arrays.asList(fileRefs.get(0), fileRefs.get(3))), options, ownerToken); + ClinicalAnalysis ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), + ownerToken).first(); assertEquals(2, ca.getFiles().size()); assertTrue(files.subList(1, 3).stream().map(File::getPath).collect(Collectors.toSet()) .containsAll(ca.getFiles().stream().map(File::getPath).collect(Collectors.toSet()))); @@ -3779,9 +3925,9 @@ public void updateClinicalAnalysisFiles() throws CatalogException { // Add first file again actionMap = new ObjectMap(ClinicalAnalysisDBAdaptor.QueryParams.FILES.key(), ParamUtils.BasicUpdateAction.ADD); options = new QueryOptions(Constants.ACTIONS, actionMap); - catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysis.getId(), - new ClinicalAnalysisUpdateParams().setFiles(Collections.singletonList(fileRefs.get(0))), options, sessionIdUser); - ca = catalogManager.getClinicalAnalysisManager().get(STUDY, clinicalAnalysis.getId(), QueryOptions.empty(), sessionIdUser).first(); + catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), + new ClinicalAnalysisUpdateParams().setFiles(Collections.singletonList(fileRefs.get(0))), options, ownerToken); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(3, ca.getFiles().size()); assertTrue(files.subList(0, 3).stream().map(File::getPath).collect(Collectors.toSet()) .containsAll(ca.getFiles().stream().map(File::getPath).collect(Collectors.toSet()))); @@ -3789,9 +3935,9 @@ public void updateClinicalAnalysisFiles() throws CatalogException { // Set file 3 and 4 actionMap = new ObjectMap(ClinicalAnalysisDBAdaptor.QueryParams.FILES.key(), ParamUtils.BasicUpdateAction.SET); options = new QueryOptions(Constants.ACTIONS, actionMap); - catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysis.getId(), - new ClinicalAnalysisUpdateParams().setFiles(Arrays.asList(fileRefs.get(2), fileRefs.get(3))), options, sessionIdUser); - ca = catalogManager.getClinicalAnalysisManager().get(STUDY, clinicalAnalysis.getId(), QueryOptions.empty(), sessionIdUser).first(); + catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), + new ClinicalAnalysisUpdateParams().setFiles(Arrays.asList(fileRefs.get(2), fileRefs.get(3))), options, ownerToken); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, clinicalAnalysis.getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(2, ca.getFiles().size()); assertTrue(files.subList(2, 4).stream().map(File::getPath).collect(Collectors.toSet()) .containsAll(ca.getFiles().stream().map(File::getPath).collect(Collectors.toSet()))); @@ -3804,19 +3950,19 @@ public void fetchCasesWithSameProbandAndDifferentSample() throws CatalogExceptio Individual proband = DummyModelUtils.getDummyIndividual("proband", SexOntologyTermAnnotation.initMale(), Arrays.asList(sample1, sample2), null, null); - catalogManager.getIndividualManager().create(STUDY, proband, QueryOptions.empty(), sessionIdUser); + catalogManager.getIndividualManager().create(studyFqn, proband, QueryOptions.empty(), ownerToken); Individual probandCopy = JacksonUtils.copy(proband, Individual.class); probandCopy.setSamples(Collections.singletonList(proband.getSamples().get(0))); ClinicalAnalysis case1 = DummyModelUtils.getDummyClinicalAnalysis("case1", probandCopy, null, null); - catalogManager.getClinicalAnalysisManager().create(STUDY, case1, QueryOptions.empty(), sessionIdUser); + catalogManager.getClinicalAnalysisManager().create(studyFqn, case1, QueryOptions.empty(), ownerToken); probandCopy.setSamples(Collections.singletonList(proband.getSamples().get(1))); ClinicalAnalysis case2 = DummyModelUtils.getDummyClinicalAnalysis("case2", probandCopy, null, null); - catalogManager.getClinicalAnalysisManager().create(STUDY, case2, QueryOptions.empty(), sessionIdUser); + catalogManager.getClinicalAnalysisManager().create(studyFqn, case2, QueryOptions.empty(), ownerToken); - OpenCGAResult result = catalogManager.getClinicalAnalysisManager().search(STUDY, new Query(), - QueryOptions.empty(), sessionIdUser); + OpenCGAResult result = catalogManager.getClinicalAnalysisManager().search(studyFqn, new Query(), + QueryOptions.empty(), ownerToken); assertEquals(2, result.getNumResults()); assertEquals(case1.getId(), result.getResults().get(0).getId()); assertEquals(proband.getId(), result.getResults().get(0).getProband().getId()); @@ -3828,6 +3974,42 @@ public void fetchCasesWithSameProbandAndDifferentSample() throws CatalogExceptio assertEquals(proband.getSamples().get(1).getId(), result.getResults().get(1).getProband().getSamples().get(0).getId()); } + @Test + public void loadClinicalAnalysesTest() throws CatalogException, IOException { + String fileStr = "clinical_analyses.json.gz"; + File file; + try (InputStream stream = getClass().getResourceAsStream("/biofiles/" + fileStr)) { + file = catalogManager.getFileManager().upload(studyFqn, stream, new File().setPath("biofiles/" + fileStr), false, true, false, ownerToken).first(); + } + + Path filePath = Paths.get(file.getUri()); + + System.out.println("Loading clinical analyses file: " + filePath + " ...."); + ClinicalAnalysisLoadResult loadResult = catalogManager.getClinicalAnalysisManager().load(studyFqn, filePath, ownerToken); + System.out.println(loadResult); + + assertEquals(1, loadResult.getFailures().size()); + + String ca1Id = "SAP-45016-1"; + String ca2Id = "OPA-6607-1"; + + Query query = new Query(); + OpenCGAResult result = catalogManager.getClinicalAnalysisManager().search(studyFqn, query, QueryOptions.empty(), + ownerToken); + assertTrue(result.getResults().stream().map(ca -> ca.getId()).collect(Collectors.toList()).contains(ca1Id)); + assertTrue(result.getResults().stream().map(ca -> ca.getId()).collect(Collectors.toList()).contains(ca2Id)); + + query.put("id", ca1Id); + ClinicalAnalysis clinicalAnalysis = catalogManager.getClinicalAnalysisManager().search(studyFqn, query, QueryOptions.empty(), + ownerToken).first(); + assertEquals(ca1Id, clinicalAnalysis.getId()); + + query.put("id", ca2Id); + clinicalAnalysis = catalogManager.getClinicalAnalysisManager().search(studyFqn, query, QueryOptions.empty(), + ownerToken).first(); + assertEquals(ca2Id, clinicalAnalysis.getId()); + } + // Annotation sets @Test public void searchByInternalAnnotationSetTest() throws CatalogException { @@ -3835,7 +4017,7 @@ public void searchByInternalAnnotationSetTest() throws CatalogException { variables.add(new Variable().setId("a").setType(Variable.VariableType.STRING)); variables.add(new Variable().setId("b").setType(Variable.VariableType.MAP_INTEGER).setAllowedKeys(Arrays.asList("b1", "b2"))); VariableSet variableSet = new VariableSet("myInternalVset", "", false, false, true, "", variables, null, 1, null); - catalogManager.getStudyManager().createVariableSet(STUDY, variableSet, sessionIdUser); + catalogManager.getStudyManager().createVariableSet(studyFqn, variableSet, ownerToken); Map annotations = new HashMap<>(); annotations.put("a", "hello"); @@ -3848,8 +4030,8 @@ public void searchByInternalAnnotationSetTest() throws CatalogException { AnnotationSet annotationSet2 = new AnnotationSet("annSet2", variableSet.getId(), annotations); DataResult clinicalAnalysisDataResult = createDummyEnvironment(true, true); - ClinicalAnalysis clinicalAnalysis = catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysisDataResult.first().getId(), - new ClinicalAnalysisUpdateParams().setAnnotationSets(Arrays.asList(annotationSet, annotationSet2)), INCLUDE_RESULT, sessionIdUser).first(); + ClinicalAnalysis clinicalAnalysis = catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysisDataResult.first().getId(), + new ClinicalAnalysisUpdateParams().setAnnotationSets(Arrays.asList(annotationSet, annotationSet2)), INCLUDE_RESULT, ownerToken).first(); assertEquals(0, clinicalAnalysis.getAnnotationSets().size()); // Create a different case with different annotations @@ -3864,53 +4046,53 @@ public void searchByInternalAnnotationSetTest() throws CatalogException { annotationSet2 = new AnnotationSet("annSet2", variableSet.getId(), annotations); DataResult clinicalAnalysisDataResult2 = createDummyEnvironment(false, true); - ClinicalAnalysis clinicalAnalysis2 = catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysisDataResult2.first().getId(), - new ClinicalAnalysisUpdateParams().setAnnotationSets(Arrays.asList(annotationSet, annotationSet2)), INCLUDE_RESULT, sessionIdUser).first(); + ClinicalAnalysis clinicalAnalysis2 = catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysisDataResult2.first().getId(), + new ClinicalAnalysisUpdateParams().setAnnotationSets(Arrays.asList(annotationSet, annotationSet2)), INCLUDE_RESULT, ownerToken).first(); assertEquals(0, clinicalAnalysis2.getAnnotationSets().size()); // Query by one of the annotations Query query = new Query(Constants.ANNOTATION, "myInternalVset:a=hello"); - assertEquals(1, catalogManager.getClinicalAnalysisManager().count(STUDY, query, sessionIdUser).getNumMatches()); - assertEquals(clinicalAnalysis.getId(), catalogManager.getClinicalAnalysisManager().search(STUDY, query, ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS, sessionIdUser).first() + assertEquals(1, catalogManager.getClinicalAnalysisManager().count(studyFqn, query, ownerToken).getNumMatches()); + assertEquals(clinicalAnalysis.getId(), catalogManager.getClinicalAnalysisManager().search(studyFqn, query, ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS, ownerToken).first() .getId()); query = new Query(Constants.ANNOTATION, "myInternalVset:b.b1=" + (Integer.MAX_VALUE + 1L)); - assertEquals(1, catalogManager.getClinicalAnalysisManager().count(STUDY, query, sessionIdUser).getNumMatches()); - assertEquals(clinicalAnalysis.getId(), catalogManager.getClinicalAnalysisManager().search(STUDY, query, ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS, sessionIdUser).first() + assertEquals(1, catalogManager.getClinicalAnalysisManager().count(studyFqn, query, ownerToken).getNumMatches()); + assertEquals(clinicalAnalysis.getId(), catalogManager.getClinicalAnalysisManager().search(studyFqn, query, ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS, ownerToken).first() .getId()); query = new Query(Constants.ANNOTATION, "b.b1=14"); - assertEquals(1, catalogManager.getClinicalAnalysisManager().count(STUDY, query, sessionIdUser).getNumMatches()); - assertEquals(clinicalAnalysis2.getId(), catalogManager.getClinicalAnalysisManager().search(STUDY, query, ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS, sessionIdUser).first() + assertEquals(1, catalogManager.getClinicalAnalysisManager().count(studyFqn, query, ownerToken).getNumMatches()); + assertEquals(clinicalAnalysis2.getId(), catalogManager.getClinicalAnalysisManager().search(studyFqn, query, ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS, ownerToken).first() .getId()); query = new Query(Constants.ANNOTATION, "a=goodbye"); - assertEquals(1, catalogManager.getClinicalAnalysisManager().count(STUDY, query, sessionIdUser).getNumMatches()); - assertEquals(clinicalAnalysis2.getId(), catalogManager.getClinicalAnalysisManager().search(STUDY, query, ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS, sessionIdUser).first() + assertEquals(1, catalogManager.getClinicalAnalysisManager().count(studyFqn, query, ownerToken).getNumMatches()); + assertEquals(clinicalAnalysis2.getId(), catalogManager.getClinicalAnalysisManager().search(studyFqn, query, ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS, ownerToken).first() .getId()); // Update sample annotation to be exactly the same as sample2 ObjectMap action = new ObjectMap(ClinicalAnalysisDBAdaptor.QueryParams.ANNOTATION_SETS.key(), ParamUtils.BasicUpdateAction.SET); QueryOptions options = new QueryOptions(Constants.ACTIONS, action); - catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysis.getId(), - new ClinicalAnalysisUpdateParams().setAnnotationSets(Arrays.asList(annotationSet, annotationSet2)), options, sessionIdUser); + catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), + new ClinicalAnalysisUpdateParams().setAnnotationSets(Arrays.asList(annotationSet, annotationSet2)), options, ownerToken); query = new Query(Constants.ANNOTATION, "myInternalVset:a=hello"); - assertEquals(0, catalogManager.getClinicalAnalysisManager().count(STUDY, query, sessionIdUser).getNumMatches()); + assertEquals(0, catalogManager.getClinicalAnalysisManager().count(studyFqn, query, ownerToken).getNumMatches()); query = new Query(Constants.ANNOTATION, "myInternalVset:b.b1=4"); - assertEquals(0, catalogManager.getClinicalAnalysisManager().count(STUDY, query, sessionIdUser).getNumMatches()); + assertEquals(0, catalogManager.getClinicalAnalysisManager().count(studyFqn, query, ownerToken).getNumMatches()); query = new Query(Constants.ANNOTATION, "b.b1=14"); - assertEquals(2, catalogManager.getClinicalAnalysisManager().count(STUDY, query, sessionIdUser).getNumMatches()); + assertEquals(2, catalogManager.getClinicalAnalysisManager().count(studyFqn, query, ownerToken).getNumMatches()); assertTrue(Arrays.asList(clinicalAnalysis.getId(), clinicalAnalysis2.getId()) - .containsAll(catalogManager.getClinicalAnalysisManager().search(STUDY, query, ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS, sessionIdUser) + .containsAll(catalogManager.getClinicalAnalysisManager().search(studyFqn, query, ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS, ownerToken) .getResults().stream().map(ClinicalAnalysis::getId).collect(Collectors.toList()))); query = new Query(Constants.ANNOTATION, "a=goodbye"); - assertEquals(2, catalogManager.getClinicalAnalysisManager().count(STUDY, query, sessionIdUser).getNumMatches()); + assertEquals(2, catalogManager.getClinicalAnalysisManager().count(studyFqn, query, ownerToken).getNumMatches()); assertTrue(Arrays.asList(clinicalAnalysis.getId(), clinicalAnalysis2.getId()) - .containsAll(catalogManager.getClinicalAnalysisManager().search(STUDY, query, ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS, sessionIdUser) + .containsAll(catalogManager.getClinicalAnalysisManager().search(studyFqn, query, ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS, ownerToken) .getResults().stream().map(ClinicalAnalysis::getId).collect(Collectors.toList()))); } @@ -3925,8 +4107,8 @@ public void testSearchAnnotation() throws CatalogException { "", null, Collections.emptyMap())); variables.add(new Variable("OTHER", "", "", Variable.VariableType.OBJECT, null, false, false, null, null, 1, "", "", null, Collections.emptyMap())); - VariableSet vs1 = catalogManager.getStudyManager().createVariableSet(STUDY, "vs1", "vs1", false, false, "", null, variables, - Collections.singletonList(VariableSet.AnnotableDataModels.CLINICAL_ANALYSIS), sessionIdUser).first(); + VariableSet vs1 = catalogManager.getStudyManager().createVariableSet(studyFqn, "vs1", "vs1", false, false, "", null, variables, + Collections.singletonList(VariableSet.AnnotableDataModels.CLINICAL_ANALYSIS), ownerToken).first(); ObjectMap annotations = new ObjectMap() .append("var_name", "Joe") @@ -3936,18 +4118,17 @@ public void testSearchAnnotation() throws CatalogException { DataResult clinicalAnalysisDataResult = createDummyEnvironment(true, true); createDummyEnvironment(false, true); - catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysisDataResult.first().getId(), + catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysisDataResult.first().getId(), new ClinicalAnalysisUpdateParams().setAnnotationSets(Collections.singletonList(annotationSet)), QueryOptions.empty(), - sessionIdUser); + ownerToken); Query query = new Query(Constants.ANNOTATION, "var_name=Joe;" + vs1.getId() + ":AGE=25"); - DataResult annotDataResult = catalogManager.getClinicalAnalysisManager().search(STUDY, query, - QueryOptions.empty(), sessionIdUser); + DataResult annotDataResult = catalogManager.getClinicalAnalysisManager().search(studyFqn, query, + QueryOptions.empty(), ownerToken); assertEquals(1, annotDataResult.getNumResults()); query.put(Constants.ANNOTATION, "var_name=Joe;" + vs1.getId() + ":AGE=23"); - annotDataResult = catalogManager.getClinicalAnalysisManager().search(STUDY, query, QueryOptions.empty(), sessionIdUser); + annotDataResult = catalogManager.getClinicalAnalysisManager().search(studyFqn, query, QueryOptions.empty(), ownerToken); assertEquals(0, annotDataResult.getNumResults()); } - } diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FamilyManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FamilyManagerTest.java index 4d082414de8..670ead3493e 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FamilyManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FamilyManagerTest.java @@ -18,12 +18,9 @@ import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.StringUtils; -import org.junit.After; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; -import org.junit.rules.ExpectedException; import org.opencb.biodata.models.clinical.ClinicalComment; import org.opencb.biodata.models.clinical.Disorder; import org.opencb.biodata.models.clinical.Phenotype; @@ -31,7 +28,6 @@ import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.commons.test.GenericTest; import org.opencb.opencga.TestParamConstants; import org.opencb.opencga.catalog.db.api.FamilyDBAdaptor; import org.opencb.opencga.catalog.db.api.IndividualDBAdaptor; @@ -54,7 +50,6 @@ import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.sample.SamplePermissions; import org.opencb.opencga.core.models.sample.SampleReferenceParam; -import org.opencb.opencga.core.models.user.Account; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.MediumTests; @@ -68,43 +63,14 @@ * Created by pfurio on 11/05/17. */ @Category(MediumTests.class) -public class FamilyManagerTest extends GenericTest { +public class FamilyManagerTest extends AbstractManagerTest { - public final static String STUDY = "user@1000G:phase1"; - @Rule - public ExpectedException thrown = ExpectedException.none(); - - @Rule - public CatalogManagerExternalResource catalogManagerResource = new CatalogManagerExternalResource(); - - protected CatalogManager catalogManager; private FamilyManager familyManager; - private String opencgaToken; - protected String sessionIdUser; - - private static final QueryOptions INCLUDE_RESULT = new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true); @Before - public void setUp() throws IOException, CatalogException { - catalogManager = catalogManagerResource.getCatalogManager(); + public void setUp() throws Exception { + super.setUp(); familyManager = catalogManager.getFamilyManager(); - setUpCatalogManager(catalogManager); - } - - public void setUpCatalogManager(CatalogManager catalogManager) throws CatalogException { - opencgaToken = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); - - catalogManager.getUserManager().create("user", "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, "", null, - Account.AccountType.FULL, opencgaToken); - sessionIdUser = catalogManager.getUserManager().login("user", TestParamConstants.PASSWORD).getToken(); - - String projectId = catalogManager.getProjectManager().create("1000G", "Project about some genomes", "", "Homo sapiens", - null, "GRCh38", INCLUDE_RESULT, sessionIdUser).first().getId(); - catalogManager.getStudyManager().create(projectId, "phase1", null, "Phase 1", "Done", null, null, null, null, null, sessionIdUser); - } - - @After - public void tearDown() throws Exception { } @Test @@ -165,16 +131,16 @@ public void deleteFamilyTest() throws CatalogException { assertEquals(familyId, member.getFamilyIds().get(0)); } - catalogManager.getFamilyManager().delete(STUDY, Collections.singletonList(familyId), QueryOptions.empty(), sessionIdUser); + catalogManager.getFamilyManager().delete(studyFqn, Collections.singletonList(familyId), QueryOptions.empty(), ownerToken); try { - catalogManager.getFamilyManager().get(STUDY, familyId, QueryOptions.empty(), sessionIdUser); + catalogManager.getFamilyManager().get(studyFqn, familyId, QueryOptions.empty(), ownerToken); fail("Family should not exist"); } catch (CatalogException e) { // empty block } List members = familyDataResult.first().getMembers().stream().map(Individual::getId).collect(Collectors.toList()); - OpenCGAResult result = catalogManager.getIndividualManager().get(STUDY, members, QueryOptions.empty(), sessionIdUser); + OpenCGAResult result = catalogManager.getIndividualManager().get(studyFqn, members, QueryOptions.empty(), ownerToken); for (Individual member : result.getResults()) { assertTrue(member.getFamilyIds().isEmpty()); @@ -184,28 +150,28 @@ public void deleteFamilyTest() throws CatalogException { @Test public void deleteWithClinicalAnalysisTest() throws CatalogException { Sample sample = new Sample().setId("sample1"); - catalogManager.getSampleManager().create(STUDY, sample, QueryOptions.empty(), sessionIdUser); + catalogManager.getSampleManager().create(studyFqn, sample, QueryOptions.empty(), ownerToken); sample = new Sample().setId("sample2"); - catalogManager.getSampleManager().create(STUDY, sample, QueryOptions.empty(), sessionIdUser); + catalogManager.getSampleManager().create(studyFqn, sample, QueryOptions.empty(), ownerToken); sample = new Sample().setId("sample3"); - catalogManager.getSampleManager().create(STUDY, sample, QueryOptions.empty(), sessionIdUser); + catalogManager.getSampleManager().create(studyFqn, sample, QueryOptions.empty(), ownerToken); sample = new Sample().setId("sample4"); - catalogManager.getSampleManager().create(STUDY, sample, QueryOptions.empty(), sessionIdUser); + catalogManager.getSampleManager().create(studyFqn, sample, QueryOptions.empty(), ownerToken); Individual individual = new Individual() .setId("proband") .setDisorders(Collections.singletonList(new Disorder().setId("disorder"))); - catalogManager.getIndividualManager().create(STUDY, individual, Arrays.asList("sample1", "sample2"), QueryOptions.empty(), - sessionIdUser); + catalogManager.getIndividualManager().create(studyFqn, individual, Arrays.asList("sample1", "sample2"), QueryOptions.empty(), + ownerToken); individual = new Individual().setId("father"); - catalogManager.getIndividualManager().create(STUDY, individual, Arrays.asList("sample3"), QueryOptions.empty(), sessionIdUser); + catalogManager.getIndividualManager().create(studyFqn, individual, Arrays.asList("sample3"), QueryOptions.empty(), ownerToken); Family family = new Family().setId("family"); - catalogManager.getFamilyManager().create(STUDY, family, Arrays.asList("proband", "father"), QueryOptions.empty(), sessionIdUser); + catalogManager.getFamilyManager().create(studyFqn, family, Arrays.asList("proband", "father"), QueryOptions.empty(), ownerToken); family.setMembers(Arrays.asList( new Individual().setId("proband").setSamples(Collections.singletonList(new Sample().setId("sample2"))), @@ -218,26 +184,26 @@ public void deleteWithClinicalAnalysisTest() throws CatalogException { .setFamily(family) .setLocked(true) .setType(ClinicalAnalysis.Type.FAMILY); - catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, QueryOptions.empty(), sessionIdUser); + catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, QueryOptions.empty(), ownerToken); try { - catalogManager.getFamilyManager().delete(STUDY, Collections.singletonList(family.getId()), QueryOptions.empty(), sessionIdUser); + catalogManager.getFamilyManager().delete(studyFqn, Collections.singletonList(family.getId()), QueryOptions.empty(), ownerToken); fail("Clinical is locked. It should not delete anything"); } catch (CatalogException e) { System.out.println(e.getMessage()); // empty block } - OpenCGAResult result = catalogManager.getFamilyManager().get(STUDY, family.getId(), QueryOptions.empty(), sessionIdUser); + OpenCGAResult result = catalogManager.getFamilyManager().get(studyFqn, family.getId(), QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumResults()); - catalogManager.getClinicalAnalysisManager().update(STUDY, clinicalAnalysis.getId(), + catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis.getId(), new ClinicalAnalysisUpdateParams() .setLocked(false), - QueryOptions.empty(), sessionIdUser); + QueryOptions.empty(), ownerToken); try { - catalogManager.getFamilyManager().delete(STUDY, Collections.singletonList(family.getId()), QueryOptions.empty(), sessionIdUser); + catalogManager.getFamilyManager().delete(studyFqn, Collections.singletonList(family.getId()), QueryOptions.empty(), ownerToken); fail("Clinical is not locked. It should not delete anything either"); } catch (CatalogException e) { System.out.println(e.getMessage()); @@ -254,7 +220,7 @@ public void updateFamilyReferencesInIndividualTest() throws CatalogException { } // Create a new individual - catalogManager.getIndividualManager().create(STUDY, new Individual().setId("john"), QueryOptions.empty(), sessionIdUser); + catalogManager.getIndividualManager().create(studyFqn, new Individual().setId("john"), QueryOptions.empty(), ownerToken); FamilyUpdateParams updateParams = new FamilyUpdateParams() .setMembers(Arrays.asList( new IndividualReferenceParam().setId("john"), @@ -263,8 +229,8 @@ public void updateFamilyReferencesInIndividualTest() throws CatalogException { new IndividualReferenceParam().setId("child1") )); - familyManager.update(STUDY, familyDataResult.first().getId(), updateParams, QueryOptions.empty(), sessionIdUser); - Family family = familyManager.get(STUDY, familyDataResult.first().getId(), QueryOptions.empty(), sessionIdUser).first(); + familyManager.update(studyFqn, familyDataResult.first().getId(), updateParams, QueryOptions.empty(), ownerToken); + Family family = familyManager.get(studyFqn, familyDataResult.first().getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(4, family.getMembers().size()); assertTrue(Arrays.asList("john", "father", "mother", "child1") .containsAll(family.getMembers().stream().map(Individual::getId).collect(Collectors.toList()))); @@ -274,16 +240,16 @@ public void updateFamilyReferencesInIndividualTest() throws CatalogException { } // Check removed members no longer belong to the family - List individualList = catalogManager.getIndividualManager().get(STUDY, Arrays.asList("child2", "child3"), - QueryOptions.empty(), sessionIdUser).getResults(); + List individualList = catalogManager.getIndividualManager().get(studyFqn, Arrays.asList("child2", "child3"), + QueryOptions.empty(), ownerToken).getResults(); assertEquals(2, individualList.size()); for (Individual individual : individualList) { assertEquals(0, individual.getFamilyIds().size()); } createDummyFamily("Other-Family-Name", false); - individualList = catalogManager.getIndividualManager().get(STUDY, Arrays.asList("john", "father", "mother", "child1", "child2", - "child3"), QueryOptions.empty(), sessionIdUser).getResults(); + individualList = catalogManager.getIndividualManager().get(studyFqn, Arrays.asList("john", "father", "mother", "child1", "child2", + "child3"), QueryOptions.empty(), ownerToken).getResults(); for (Individual individual : individualList) { switch (individual.getId()) { case "john": @@ -322,10 +288,10 @@ public void createComplexFamily() throws CatalogException { .setFather(father).setMother(mother).setSex(SexOntologyTermAnnotation.initFemale()); Individual sibling = new Individual().setId("sibling").setFather(father).setMother(mother); - catalogManager.getFamilyManager().create(STUDY, new Family().setId("family").setMembers( + catalogManager.getFamilyManager().create(studyFqn, new Family().setId("family").setMembers( Arrays.asList(paternalGrandfather, paternalGrandmother, maternalGrandfather, maternalGrandmother, mother, father, proband, - brother, sister, sibling)), QueryOptions.empty(), sessionIdUser); - OpenCGAResult family = catalogManager.getFamilyManager().get(STUDY, "family", QueryOptions.empty(), sessionIdUser); + brother, sister, sibling)), QueryOptions.empty(), ownerToken); + OpenCGAResult family = catalogManager.getFamilyManager().get(studyFqn, "family", QueryOptions.empty(), ownerToken); Map> roles = family.first().getRoles(); assertEquals(10, family.first().getMembers().size()); @@ -441,10 +407,10 @@ public void updateFamilyRoles() throws CatalogException { Individual sister = new Individual().setId("sister").setSex(SexOntologyTermAnnotation.initFemale()); Individual sibling = new Individual().setId("sibling"); - catalogManager.getFamilyManager().create(STUDY, new Family().setId("family").setMembers( + catalogManager.getFamilyManager().create(studyFqn, new Family().setId("family").setMembers( Arrays.asList(paternalGrandfather, paternalGrandmother, maternalGrandfather, maternalGrandmother, mother, father, proband, - brother, sister, sibling)), QueryOptions.empty(), sessionIdUser); - OpenCGAResult family = catalogManager.getFamilyManager().get(STUDY, "family", QueryOptions.empty(), sessionIdUser); + brother, sister, sibling)), QueryOptions.empty(), ownerToken); + OpenCGAResult family = catalogManager.getFamilyManager().get(studyFqn, "family", QueryOptions.empty(), ownerToken); Map> roles = family.first().getRoles(); assertEquals(10, family.first().getMembers().size()); for (Map.Entry> entry : family.first().getRoles().entrySet()) { @@ -454,26 +420,26 @@ public void updateFamilyRoles() throws CatalogException { IndividualUpdateParams updateParams = new IndividualUpdateParams() .setFather(new IndividualReferenceParam().setId(paternalGrandfather.getId())) .setMother(new IndividualReferenceParam().setId(paternalGrandmother.getId())); - catalogManager.getIndividualManager().update(STUDY, father.getId(), updateParams, QueryOptions.empty(), sessionIdUser); + catalogManager.getIndividualManager().update(studyFqn, father.getId(), updateParams, QueryOptions.empty(), ownerToken); updateParams = new IndividualUpdateParams() .setFather(new IndividualReferenceParam().setId(maternalGrandfather.getId())) .setMother(new IndividualReferenceParam().setId(maternalGrandmother.getId())); - catalogManager.getIndividualManager().update(STUDY, mother.getId(), updateParams, QueryOptions.empty(), sessionIdUser); + catalogManager.getIndividualManager().update(studyFqn, mother.getId(), updateParams, QueryOptions.empty(), ownerToken); updateParams = new IndividualUpdateParams() .setFather(new IndividualReferenceParam().setId(father.getId())) .setMother(new IndividualReferenceParam().setId(mother.getId())); - catalogManager.getIndividualManager().update(STUDY, proband.getId(), updateParams, QueryOptions.empty(), sessionIdUser); - catalogManager.getIndividualManager().update(STUDY, brother.getId(), updateParams, QueryOptions.empty(), sessionIdUser); - catalogManager.getIndividualManager().update(STUDY, sister.getId(), updateParams, QueryOptions.empty(), sessionIdUser); - catalogManager.getIndividualManager().update(STUDY, sibling.getId(), updateParams, QueryOptions.empty(), sessionIdUser); + catalogManager.getIndividualManager().update(studyFqn, proband.getId(), updateParams, QueryOptions.empty(), ownerToken); + catalogManager.getIndividualManager().update(studyFqn, brother.getId(), updateParams, QueryOptions.empty(), ownerToken); + catalogManager.getIndividualManager().update(studyFqn, sister.getId(), updateParams, QueryOptions.empty(), ownerToken); + catalogManager.getIndividualManager().update(studyFqn, sibling.getId(), updateParams, QueryOptions.empty(), ownerToken); -// catalogManager.getFamilyManager().update(STUDY, family.first().getId(), null, -// new QueryOptions(ParamConstants.FAMILY_UPDATE_ROLES_PARAM, true), sessionIdUser); +// catalogManager.getFamilyManager().update(studyFqn, family.first().getId(), null, +// new QueryOptions(ParamConstants.FAMILY_UPDATE_ROLES_PARAM, true), token); // Roles should have been automatically updated containing up to date roles - family = catalogManager.getFamilyManager().get(STUDY, "family", QueryOptions.empty(), sessionIdUser); + family = catalogManager.getFamilyManager().get(studyFqn, "family", QueryOptions.empty(), ownerToken); roles = family.first().getRoles(); assertEquals(10, family.first().getMembers().size()); Map pGrandfather = roles.get("p_grandfather"); @@ -581,8 +547,8 @@ public void searchFamily() throws CatalogException { createDummyFamily("Martinez", false); createDummyFamily("Furio", false); - OpenCGAResult search = catalogManager.getFamilyManager().search(STUDY, - new Query(FamilyDBAdaptor.QueryParams.ID.key(), "~^Mart"), QueryOptions.empty(), sessionIdUser); + OpenCGAResult search = catalogManager.getFamilyManager().search(studyFqn, + new Query(FamilyDBAdaptor.QueryParams.ID.key(), "~^Mart"), QueryOptions.empty(), ownerToken); assertEquals(2, search.getNumResults()); for (Family result : search.getResults()) { assertTrue(result.getId().startsWith("Mart")); @@ -621,19 +587,19 @@ public void testRoles() throws CatalogException { public void testPropagateFamilyPermission() throws CatalogException { createDummyFamily("Martinez-Martinez", true); - catalogManager.getUserManager().create("user2", "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, "", null, - Account.AccountType.GUEST, opencgaToken); - String token = catalogManager.getUserManager().login("user2", TestParamConstants.PASSWORD).getToken(); + catalogManager.getUserManager().create("user2", "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, organizationId, null, + opencgaToken); + String token = catalogManager.getUserManager().login(organizationId, "user2", TestParamConstants.PASSWORD).getToken(); try { - familyManager.get(STUDY, "Martinez-Martinez", QueryOptions.empty(), token); + familyManager.get(studyFqn, "Martinez-Martinez", QueryOptions.empty(), token); fail("Expected authorization exception. user2 should not be able to see the study"); } catch (CatalogAuthorizationException ignored) { } - familyManager.updateAcl(STUDY, new FamilyAclParams("VIEW", "Martinez-Martinez", null, null, FamilyAclParams.Propagate.NO), "user2", - ParamUtils.AclAction.SET, sessionIdUser); - DataResult familyDataResult = familyManager.get(STUDY, "Martinez-Martinez", QueryOptions.empty(), token); + familyManager.updateAcl(studyFqn, new FamilyAclParams("VIEW", "Martinez-Martinez", null, null, FamilyAclParams.Propagate.NO), "user2", + ParamUtils.AclAction.SET, ownerToken); + DataResult familyDataResult = familyManager.get(studyFqn, "Martinez-Martinez", QueryOptions.empty(), token); assertEquals(1, familyDataResult.getNumResults()); assertEquals(0, familyDataResult.first().getMembers().size()); int nsamples = 0; @@ -642,9 +608,9 @@ public void testPropagateFamilyPermission() throws CatalogException { } assertEquals(0, nsamples); - familyManager.updateAcl(STUDY, new FamilyAclParams("VIEW", "Martinez-Martinez", null, null, FamilyAclParams.Propagate.YES), "user2", - ParamUtils.AclAction.SET, sessionIdUser); - familyDataResult = familyManager.get(STUDY, "Martinez-Martinez", QueryOptions.empty(), token); + familyManager.updateAcl(studyFqn, new FamilyAclParams("VIEW", "Martinez-Martinez", null, null, FamilyAclParams.Propagate.YES), "user2", + ParamUtils.AclAction.SET, ownerToken); + familyDataResult = familyManager.get(studyFqn, "Martinez-Martinez", QueryOptions.empty(), token); assertEquals(1, familyDataResult.getNumResults()); assertEquals(5, familyDataResult.first().getMembers().size()); List sampleList = new ArrayList<>(3); @@ -653,16 +619,16 @@ public void testPropagateFamilyPermission() throws CatalogException { } assertEquals(3, sampleList.size()); - OpenCGAResult> acls = catalogManager.getSampleManager().getAcls(STUDY, - sampleList.stream().map(Sample::getId).collect(Collectors.toList()), "user2", false, sessionIdUser); + OpenCGAResult> acls = catalogManager.getSampleManager().getAcls(studyFqn, + sampleList.stream().map(Sample::getId).collect(Collectors.toList()), "user2", false, token); for (AclEntryList result : acls.getResults()) { assertTrue(result.getAcl().get(0).getPermissions().contains(SamplePermissions.VIEW)); assertFalse(result.getAcl().get(0).getPermissions().contains(SamplePermissions.VIEW_VARIANTS)); } - familyManager.updateAcl(STUDY, new FamilyAclParams("VIEW", "Martinez-Martinez", null, null, - FamilyAclParams.Propagate.YES_AND_VARIANT_VIEW), "user2", ParamUtils.AclAction.SET, sessionIdUser); - familyDataResult = familyManager.get(STUDY, "Martinez-Martinez", QueryOptions.empty(), token); + familyManager.updateAcl(studyFqn, new FamilyAclParams("VIEW", "Martinez-Martinez", null, null, + FamilyAclParams.Propagate.YES_AND_VARIANT_VIEW), "user2", ParamUtils.AclAction.SET, ownerToken); + familyDataResult = familyManager.get(studyFqn, "Martinez-Martinez", QueryOptions.empty(), token); assertEquals(1, familyDataResult.getNumResults()); assertEquals(5, familyDataResult.first().getMembers().size()); sampleList = new ArrayList<>(3); @@ -671,8 +637,8 @@ public void testPropagateFamilyPermission() throws CatalogException { } assertEquals(3, sampleList.size()); - acls = catalogManager.getSampleManager().getAcls(STUDY, sampleList.stream().map(Sample::getId).collect(Collectors.toList()), - "user2", false, sessionIdUser); + acls = catalogManager.getSampleManager().getAcls(studyFqn, sampleList.stream().map(Sample::getId).collect(Collectors.toList()), + "user2", false, token); for (AclEntryList result : acls.getResults()) { assertTrue(result.getAcl().get(0).getPermissions().contains(SamplePermissions.VIEW)); assertTrue(result.getAcl().get(0).getPermissions().contains(SamplePermissions.VIEW_VARIANTS)); @@ -683,43 +649,43 @@ public void testPropagateFamilyPermission() throws CatalogException { public void getFamilyWithOnlyAllowedMembers2() throws CatalogException, IOException { createDummyFamily("Martinez-Martinez", true); - catalogManager.getUserManager().create("user2", "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, "", null, - Account.AccountType.GUEST, opencgaToken); - String token = catalogManager.getUserManager().login("user2", TestParamConstants.PASSWORD).getToken(); + catalogManager.getUserManager().create("user2", "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, organizationId, null, + opencgaToken); + String token = catalogManager.getUserManager().login(organizationId, "user2", TestParamConstants.PASSWORD).getToken(); try { - familyManager.get(STUDY, "Martinez-Martinez", QueryOptions.empty(), token); + familyManager.get(studyFqn, "Martinez-Martinez", QueryOptions.empty(), token); fail("Expected authorization exception. user2 should not be able to see the study"); } catch (CatalogAuthorizationException ignored) { } - familyManager.updateAcl(STUDY, new FamilyAclParams("VIEW", "Martinez-Martinez", null, null, FamilyAclParams.Propagate.NO), "user2", - ParamUtils.AclAction.SET, sessionIdUser); - DataResult familyDataResult = familyManager.get(STUDY, "Martinez-Martinez", QueryOptions.empty(), token); + familyManager.updateAcl(studyFqn, new FamilyAclParams("VIEW", "Martinez-Martinez", null, null, FamilyAclParams.Propagate.NO), "user2", + ParamUtils.AclAction.SET, ownerToken); + DataResult familyDataResult = familyManager.get(studyFqn, "Martinez-Martinez", QueryOptions.empty(), token); assertEquals(1, familyDataResult.getNumResults()); assertEquals(0, familyDataResult.first().getMembers().size()); - catalogManager.getIndividualManager().updateAcl(STUDY, Collections.singletonList("child2"), "user2", - new IndividualAclParams("", "VIEW"), ParamUtils.AclAction.SET, false, sessionIdUser); - familyDataResult = familyManager.get(STUDY, "Martinez-Martinez", QueryOptions.empty(), token); + catalogManager.getIndividualManager().updateAcl(studyFqn, Collections.singletonList("child2"), "user2", + new IndividualAclParams("", "VIEW"), ParamUtils.AclAction.SET, false, ownerToken); + familyDataResult = familyManager.get(studyFqn, "Martinez-Martinez", QueryOptions.empty(), token); assertEquals(1, familyDataResult.getNumResults()); assertEquals(1, familyDataResult.first().getMembers().size()); assertEquals("child2", familyDataResult.first().getMembers().get(0).getId()); - catalogManager.getIndividualManager().updateAcl(STUDY, Collections.singletonList("child3"), "user2", - new IndividualAclParams("", "VIEW"), ParamUtils.AclAction.SET, false, sessionIdUser); - familyDataResult = familyManager.get(STUDY, "Martinez-Martinez", QueryOptions.empty(), token); + catalogManager.getIndividualManager().updateAcl(studyFqn, Collections.singletonList("child3"), "user2", + new IndividualAclParams("", "VIEW"), ParamUtils.AclAction.SET, false, ownerToken); + familyDataResult = familyManager.get(studyFqn, "Martinez-Martinez", QueryOptions.empty(), token); assertEquals(1, familyDataResult.getNumResults()); assertEquals(2, familyDataResult.first().getMembers().size()); assertEquals("child2", familyDataResult.first().getMembers().get(0).getId()); assertEquals("child3", familyDataResult.first().getMembers().get(1).getId()); - familyDataResult = familyManager.get(STUDY, "Martinez-Martinez", + familyDataResult = familyManager.get(studyFqn, "Martinez-Martinez", new QueryOptions(QueryOptions.EXCLUDE, FamilyDBAdaptor.QueryParams.MEMBERS.key()), token); assertEquals(1, familyDataResult.getNumResults()); assertEquals(null, familyDataResult.first().getMembers()); - familyDataResult = familyManager.get(STUDY, "Martinez-Martinez", + familyDataResult = familyManager.get(studyFqn, "Martinez-Martinez", new QueryOptions(QueryOptions.INCLUDE, FamilyDBAdaptor.QueryParams.MEMBERS.key() + "." + IndividualDBAdaptor.QueryParams.NAME.key()), token); @@ -735,28 +701,28 @@ public void getFamilyWithOnlyAllowedMembers2() throws CatalogException, IOExcept @Test public void testUpdateWithLockedClinicalAnalysis() throws CatalogException { Sample sample = new Sample().setId("sample1"); - catalogManager.getSampleManager().create(STUDY, sample, QueryOptions.empty(), sessionIdUser); + catalogManager.getSampleManager().create(studyFqn, sample, QueryOptions.empty(), ownerToken); sample = new Sample().setId("sample2"); - catalogManager.getSampleManager().create(STUDY, sample, QueryOptions.empty(), sessionIdUser); + catalogManager.getSampleManager().create(studyFqn, sample, QueryOptions.empty(), ownerToken); sample = new Sample().setId("sample3"); - catalogManager.getSampleManager().create(STUDY, sample, QueryOptions.empty(), sessionIdUser); + catalogManager.getSampleManager().create(studyFqn, sample, QueryOptions.empty(), ownerToken); sample = new Sample().setId("sample4"); - catalogManager.getSampleManager().create(STUDY, sample, QueryOptions.empty(), sessionIdUser); + catalogManager.getSampleManager().create(studyFqn, sample, QueryOptions.empty(), ownerToken); Individual individual = new Individual() .setId("proband") .setDisorders(Collections.singletonList(new Disorder().setId("disorder"))); - catalogManager.getIndividualManager().create(STUDY, individual, Arrays.asList("sample1", "sample2"), QueryOptions.empty(), - sessionIdUser); + catalogManager.getIndividualManager().create(studyFqn, individual, Arrays.asList("sample1", "sample2"), QueryOptions.empty(), + ownerToken); individual = new Individual().setId("father"); - catalogManager.getIndividualManager().create(STUDY, individual, Arrays.asList("sample3"), QueryOptions.empty(), sessionIdUser); + catalogManager.getIndividualManager().create(studyFqn, individual, Arrays.asList("sample3"), QueryOptions.empty(), ownerToken); Family family = new Family().setId("family"); - catalogManager.getFamilyManager().create(STUDY, family, Arrays.asList("proband", "father"), QueryOptions.empty(), sessionIdUser); + catalogManager.getFamilyManager().create(studyFqn, family, Arrays.asList("proband", "father"), QueryOptions.empty(), ownerToken); family.setMembers(Arrays.asList( new Individual().setId("proband").setSamples(Collections.singletonList(new Sample().setId("sample2"))), @@ -768,7 +734,7 @@ public void testUpdateWithLockedClinicalAnalysis() throws CatalogException { .setProband(new Individual().setId("proband")) .setFamily(family) .setType(ClinicalAnalysis.Type.FAMILY); - catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, QueryOptions.empty(), sessionIdUser); + catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, QueryOptions.empty(), ownerToken); // We will create another clinical analysis with the same information. In this test, we will not lock clinical2 clinicalAnalysis = new ClinicalAnalysis() @@ -776,55 +742,55 @@ public void testUpdateWithLockedClinicalAnalysis() throws CatalogException { .setProband(new Individual().setId("proband")) .setFamily(family) .setType(ClinicalAnalysis.Type.FAMILY); - catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, QueryOptions.empty(), sessionIdUser); + catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, QueryOptions.empty(), ownerToken); // Update family not used in Clinical Analysis - catalogManager.getFamilyManager().update(STUDY, "family", new FamilyUpdateParams() + catalogManager.getFamilyManager().update(studyFqn, "family", new FamilyUpdateParams() .setDescription(RandomStringUtils.randomAlphanumeric(10)), - new QueryOptions(), sessionIdUser); + new QueryOptions(), ownerToken); - Family familyResult = catalogManager.getFamilyManager().get(STUDY, "family", QueryOptions.empty(), sessionIdUser).first(); + Family familyResult = catalogManager.getFamilyManager().get(studyFqn, "family", QueryOptions.empty(), ownerToken).first(); assertEquals(2, familyResult.getVersion()); assertEquals(2, familyResult.getMembers().size()); assertEquals(2, familyResult.getMembers().get(0).getVersion()); assertEquals(2, familyResult.getMembers().get(1).getVersion()); - ClinicalAnalysis clinicalResult = catalogManager.getClinicalAnalysisManager().get(STUDY, "clinical", QueryOptions.empty(), - sessionIdUser).first(); + ClinicalAnalysis clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, "clinical", QueryOptions.empty(), + ownerToken).first(); assertEquals(2, clinicalResult.getProband().getVersion()); assertEquals(2, clinicalResult.getProband().getSamples().get(0).getVersion()); // sample1 version assertEquals(2, clinicalResult.getFamily().getVersion()); assertEquals(2, clinicalResult.getFamily().getMembers().size()); - clinicalResult = catalogManager.getClinicalAnalysisManager().get(STUDY, "clinical2", QueryOptions.empty(), sessionIdUser).first(); + clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, "clinical2", QueryOptions.empty(), ownerToken).first(); assertEquals(2, clinicalResult.getProband().getVersion()); assertEquals(2, clinicalResult.getProband().getSamples().get(0).getVersion()); // sample1 version assertEquals(2, clinicalResult.getFamily().getVersion()); assertEquals(2, clinicalResult.getFamily().getMembers().size()); // proband version // LOCK CLINICAL ANALYSIS - catalogManager.getClinicalAnalysisManager().update(STUDY, "clinical", new ClinicalAnalysisUpdateParams().setLocked(true), - QueryOptions.empty(), sessionIdUser); - clinicalResult = catalogManager.getClinicalAnalysisManager().get(STUDY, "clinical", QueryOptions.empty(), sessionIdUser).first(); + catalogManager.getClinicalAnalysisManager().update(studyFqn, "clinical", new ClinicalAnalysisUpdateParams().setLocked(true), + QueryOptions.empty(), ownerToken); + clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, "clinical", QueryOptions.empty(), ownerToken).first(); assertTrue(clinicalResult.isLocked()); // Update family with version increment - catalogManager.getFamilyManager().update(STUDY, "family", new FamilyUpdateParams().setName("bl"), new QueryOptions(), - sessionIdUser); + catalogManager.getFamilyManager().update(studyFqn, "family", new FamilyUpdateParams().setName("bl"), new QueryOptions(), + ownerToken); - familyResult = catalogManager.getFamilyManager().get(STUDY, "family", QueryOptions.empty(), sessionIdUser).first(); + familyResult = catalogManager.getFamilyManager().get(studyFqn, "family", QueryOptions.empty(), ownerToken).first(); assertEquals(3, familyResult.getVersion()); assertEquals(2, familyResult.getMembers().size()); assertEquals(2, familyResult.getMembers().get(0).getVersion()); assertEquals(2, familyResult.getMembers().get(1).getVersion()); - clinicalResult = catalogManager.getClinicalAnalysisManager().get(STUDY, "clinical", QueryOptions.empty(), sessionIdUser).first(); + clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, "clinical", QueryOptions.empty(), ownerToken).first(); assertEquals(2, clinicalResult.getProband().getVersion()); assertEquals(2, clinicalResult.getProband().getSamples().get(0).getVersion()); // sample1 version assertEquals(2, clinicalResult.getFamily().getVersion()); assertEquals(2, clinicalResult.getFamily().getMembers().size()); - clinicalResult = catalogManager.getClinicalAnalysisManager().get(STUDY, "clinical2", QueryOptions.empty(), sessionIdUser).first(); + clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, "clinical2", QueryOptions.empty(), ownerToken).first(); assertEquals(2, clinicalResult.getProband().getVersion()); assertEquals(2, clinicalResult.getProband().getSamples().get(0).getVersion()); // sample1 version assertEquals(3, clinicalResult.getFamily().getVersion()); @@ -837,7 +803,7 @@ public void includeMemberIdOnly() throws CatalogException { QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, FamilyDBAdaptor.QueryParams.MEMBERS.key() + "." + IndividualDBAdaptor.QueryParams.ID.key()); - DataResult family = familyManager.get(STUDY, "family", options, sessionIdUser); + DataResult family = familyManager.get(studyFqn, "family", options, ownerToken); for (Individual individual : family.first().getMembers()) { assertTrue(StringUtils.isNotEmpty(individual.getId())); @@ -889,20 +855,20 @@ public void includeMemberIdOnly() throws CatalogException { Arrays.asList(relChild1, relChild2, relChild3, relFather, relMother), "", 5, Collections.emptyList(), Collections.emptyMap()); - return familyManager.create(STUDY, family, QueryOptions.empty(), sessionIdUser); + return familyManager.create(studyFqn, family, QueryOptions.empty(), token); } * */ private DataResult createDummyFamily(String familyName, boolean createMissingMembers) throws CatalogException { if (createMissingMembers) { Sample sample1 = new Sample().setId("sample1"); - catalogManager.getSampleManager().create(STUDY, sample1, QueryOptions.empty(), sessionIdUser); + catalogManager.getSampleManager().create(studyFqn, sample1, QueryOptions.empty(), ownerToken); Sample sample2 = new Sample().setId("sample2"); - catalogManager.getSampleManager().create(STUDY, sample2, QueryOptions.empty(), sessionIdUser); + catalogManager.getSampleManager().create(studyFqn, sample2, QueryOptions.empty(), ownerToken); Sample sample3 = new Sample().setId("sample3"); - catalogManager.getSampleManager().create(STUDY, sample3, QueryOptions.empty(), sessionIdUser); + catalogManager.getSampleManager().create(studyFqn, sample3, QueryOptions.empty(), ownerToken); } Phenotype phenotype1 = new Phenotype("dis1", "Phenotype 1", "HPO"); @@ -954,18 +920,18 @@ private DataResult createDummyFamily(String familyName, boolean createMi Family family = new Family(familyName, familyName, null, null, members, "", 5, Collections.emptyList(), Collections.emptyMap()); - OpenCGAResult familyOpenCGAResult = familyManager.create(STUDY, family, memberIds, INCLUDE_RESULT, sessionIdUser); + OpenCGAResult familyOpenCGAResult = familyManager.create(studyFqn, family, memberIds, INCLUDE_RESULT, ownerToken); if (createMissingMembers) { - catalogManager.getIndividualManager().update(STUDY, relChild1.getId(), + catalogManager.getIndividualManager().update(studyFqn, relChild1.getId(), new IndividualUpdateParams().setSamples(Collections.singletonList(new SampleReferenceParam().setId("sample1"))), - QueryOptions.empty(), sessionIdUser); - catalogManager.getIndividualManager().update(STUDY, relFather.getId(), + QueryOptions.empty(), ownerToken); + catalogManager.getIndividualManager().update(studyFqn, relFather.getId(), new IndividualUpdateParams().setSamples(Collections.singletonList(new SampleReferenceParam().setId("sample2"))), - QueryOptions.empty(), sessionIdUser); - catalogManager.getIndividualManager().update(STUDY, relMother.getId(), + QueryOptions.empty(), ownerToken); + catalogManager.getIndividualManager().update(studyFqn, relMother.getId(), new IndividualUpdateParams().setSamples(Collections.singletonList(new SampleReferenceParam().setId("sample3"))), - QueryOptions.empty(), sessionIdUser); + QueryOptions.empty(), ownerToken); } return familyOpenCGAResult; @@ -979,12 +945,12 @@ public void updateFamilyDisordersWhenIndividualDisorderIsUpdated() throws Catalo List disorderList = Arrays.asList(new Disorder().setId("disorder")); IndividualUpdateParams params = new IndividualUpdateParams().setDisorders(disorderList); - catalogManager.getIndividualManager().update(STUDY, "child1", params, new QueryOptions(), sessionIdUser); - DataResult child1 = catalogManager.getIndividualManager().get(STUDY, "child1", QueryOptions.empty(), sessionIdUser); + catalogManager.getIndividualManager().update(studyFqn, "child1", params, new QueryOptions(), ownerToken); + DataResult child1 = catalogManager.getIndividualManager().get(studyFqn, "child1", QueryOptions.empty(), ownerToken); assertEquals(1, child1.first().getDisorders().size()); assertEquals(3, child1.first().getVersion()); - family = catalogManager.getFamilyManager().get(STUDY, "family", QueryOptions.empty(), sessionIdUser); + family = catalogManager.getFamilyManager().get(studyFqn, "family", QueryOptions.empty(), ownerToken); assertEquals(1, family.first().getDisorders().size()); disorderList = Collections.emptyList(); @@ -992,23 +958,23 @@ public void updateFamilyDisordersWhenIndividualDisorderIsUpdated() throws Catalo Map actionMap = new HashMap<>(); actionMap.put(IndividualDBAdaptor.QueryParams.DISORDERS.key(), ParamUtils.BasicUpdateAction.SET); QueryOptions queryOptions = new QueryOptions(Constants.ACTIONS, actionMap); - catalogManager.getIndividualManager().update(STUDY, "child1", params, queryOptions, sessionIdUser); - child1 = catalogManager.getIndividualManager().get(STUDY, "child1", QueryOptions.empty(), sessionIdUser); + catalogManager.getIndividualManager().update(studyFqn, "child1", params, queryOptions, ownerToken); + child1 = catalogManager.getIndividualManager().get(studyFqn, "child1", QueryOptions.empty(), ownerToken); assertEquals(0, child1.first().getDisorders().size()); - family = catalogManager.getFamilyManager().get(STUDY, "family", QueryOptions.empty(), sessionIdUser); + family = catalogManager.getFamilyManager().get(studyFqn, "family", QueryOptions.empty(), ownerToken); assertEquals(0, family.first().getDisorders().size()); // Now we will update increasing the version. No changes should be produced in the family disorderList = Arrays.asList(new Disorder().setId("disorder")); params.setDisorders(disorderList); - catalogManager.getIndividualManager().update(STUDY, "child1", params, new QueryOptions(), sessionIdUser); - child1 = catalogManager.getIndividualManager().get(STUDY, "child1", QueryOptions.empty(), sessionIdUser); + catalogManager.getIndividualManager().update(studyFqn, "child1", params, new QueryOptions(), ownerToken); + child1 = catalogManager.getIndividualManager().get(studyFqn, "child1", QueryOptions.empty(), ownerToken); assertEquals(1, child1.first().getDisorders().size()); assertEquals(5, child1.first().getVersion()); - family = catalogManager.getFamilyManager().get(STUDY, "family", QueryOptions.empty(), sessionIdUser); + family = catalogManager.getFamilyManager().get(studyFqn, "family", QueryOptions.empty(), ownerToken); for (Individual member : family.first().getMembers()) { if (member.getId().equals(child1.first().getId())) { assertEquals(child1.first().getVersion(), member.getVersion()); @@ -1028,10 +994,10 @@ public void disordersDistinctTest() throws CatalogException { IndividualUpdateParams params2 = new IndividualUpdateParams().setDisorders(disorderList2); - catalogManager.getIndividualManager().update(STUDY, "child1", params1, new QueryOptions(), sessionIdUser); - catalogManager.getIndividualManager().update(STUDY, "child2", params2, new QueryOptions(), sessionIdUser); + catalogManager.getIndividualManager().update(studyFqn, "child1", params1, new QueryOptions(), ownerToken); + catalogManager.getIndividualManager().update(studyFqn, "child2", params2, new QueryOptions(), ownerToken); - OpenCGAResult distinct = catalogManager.getFamilyManager().distinct(STUDY, "disorders.name", new Query(), sessionIdUser); + OpenCGAResult distinct = catalogManager.getFamilyManager().distinct(organizationId, studyFqn, "disorders.name", new Query(), ownerToken); System.out.println(distinct); assertEquals(2, distinct.getNumResults()); @@ -1046,7 +1012,7 @@ public void createFamilyDuo() throws CatalogException { .setMembers(Arrays.asList(new Individual().setId("proband").setSex(SexOntologyTermAnnotation.initMale()), new Individual().setFather(new Individual().setId("proband")).setId("child") .setSex(SexOntologyTermAnnotation.initFemale()))); - DataResult familyDataResult = familyManager.create(STUDY, family, INCLUDE_RESULT, sessionIdUser); + DataResult familyDataResult = familyManager.create(studyFqn, family, INCLUDE_RESULT, ownerToken); assertEquals(2, familyDataResult.first().getMembers().size()); } @@ -1085,7 +1051,7 @@ public void createFamilyMissingMember() throws CatalogException { Family family = new Family("Martinez-Martinez", "Martinez-Martinez", Arrays.asList(phenotype1, phenotype2), null, Arrays.asList(relFather, relChild1, relChild2), "", 3, Collections.emptyList(), Collections.emptyMap()); - family = familyManager.create(STUDY, family, new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true), sessionIdUser).first(); + family = familyManager.create(studyFqn, family, new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true), ownerToken).first(); assertEquals(3, family.getMembers().size()); assertTrue(Arrays.asList(relFather.getId(), relChild2.getId(), relChild1.getId()) .containsAll(family.getMembers().stream().map(Individual::getId).collect(Collectors.toList()))); @@ -1120,7 +1086,7 @@ public void createFamilyPhenotypeNotPassed() throws CatalogException { thrown.expect(CatalogException.class); thrown.expectMessage("not present in any member of the family"); - familyManager.create(STUDY, family, QueryOptions.empty(), sessionIdUser); + familyManager.create(studyFqn, family, QueryOptions.empty(), ownerToken); } @Test(expected = CatalogException.class) @@ -1161,14 +1127,14 @@ public void createFamilyRepeatedMember() throws CatalogException { Arrays.asList(relFather, relMother, relChild1, relChild2, relChild1), "", -1, Collections.emptyList(), Collections.emptyMap ()); - DataResult familyDataResult = familyManager.create(STUDY, family, INCLUDE_RESULT, sessionIdUser); + DataResult familyDataResult = familyManager.create(studyFqn, family, INCLUDE_RESULT, ownerToken); assertEquals(4, familyDataResult.first().getMembers().size()); } @Test public void createEmptyFamily() throws CatalogException { Family family = new Family("xxx", "xxx", null, null, null, "", -1, Collections.emptyList(), Collections.emptyMap()); - DataResult familyDataResult = familyManager.create(STUDY, family, INCLUDE_RESULT, sessionIdUser); + DataResult familyDataResult = familyManager.create(studyFqn, family, INCLUDE_RESULT, ownerToken); assertEquals(1, familyDataResult.getNumResults()); } @@ -1197,8 +1163,8 @@ public void createEmptyFamily() throws CatalogException { // ObjectMap params = new ObjectMap(jsonObjectMapper.writeValueAsString(family)); // params = new ObjectMap(FamilyDBAdaptor.QueryParams.MEMBERS.key(), params.get(FamilyDBAdaptor.QueryParams.MEMBERS.key())); // -// DataResult updatedFamily = familyManager.update(STUDY, originalFamily.first().getName(), params, QueryOptions.empty(), -// sessionIdUser); +// DataResult updatedFamily = familyManager.update(studyFqn, originalFamily.first().getName(), params, QueryOptions.empty(), +// token); // // assertEquals(3, updatedFamily.first().getMembers().size()); // // Other parameters from the family should not have been stored in the database @@ -1228,8 +1194,8 @@ public void updateFamilyMissingMember() throws CatalogException { new IndividualReferenceParam().setId("child3"), new IndividualReferenceParam().setId("father"))); - Family family = familyManager.update(STUDY, originalFamily.first().getId(), updateParams, - new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true), sessionIdUser).first(); + Family family = familyManager.update(studyFqn, originalFamily.first().getId(), updateParams, + new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true), ownerToken).first(); assertEquals(2, family.getMembers().size()); assertEquals(2, family.getRoles().size()); assertTrue(Arrays.asList("child3", "father") @@ -1243,7 +1209,7 @@ public void recalculateRoles() throws CatalogException { FamilyUpdateParams updateParams = null; QueryOptions options = new QueryOptions(ParamConstants.FAMILY_UPDATE_ROLES_PARAM, true); - assertEquals(1, familyManager.update(STUDY, originalFamily.first().getId(), updateParams, options, sessionIdUser).getNumUpdated()); + assertEquals(1, familyManager.update(studyFqn, originalFamily.first().getId(), updateParams, options, ownerToken).getNumUpdated()); } @Test @@ -1255,11 +1221,11 @@ public void updateFamilyQualityControl() throws CatalogException { FamilyUpdateParams updateParams = new FamilyUpdateParams().setQualityControl(qualityControl); - DataResult updatedFamily = familyManager.update(STUDY, originalFamily.first().getId(), - updateParams, QueryOptions.empty(), sessionIdUser); + DataResult updatedFamily = familyManager.update(studyFqn, originalFamily.first().getId(), + updateParams, QueryOptions.empty(), ownerToken); assertEquals(1, updatedFamily.getNumUpdated()); - updatedFamily = familyManager.get(STUDY, originalFamily.first().getId(), QueryOptions.empty(), sessionIdUser); + updatedFamily = familyManager.get(studyFqn, originalFamily.first().getId(), QueryOptions.empty(), ownerToken); assertTrue(Arrays.asList("file1", "file2").containsAll(updatedFamily.first().getQualityControl().getFiles())); assertEquals(1, updatedFamily.first().getQualityControl().getComments().size()); assertEquals("author", updatedFamily.first().getQualityControl().getComments().get(0).getAuthor()); @@ -1275,17 +1241,17 @@ public void incrementVersionTest() throws CatalogException { Family dummyFamily2 = DummyModelUtils.getDummyFamily(); for (int i = dummyFamily1.getMembers().size() - 1; i >= 0; i--) { - catalogManager.getIndividualManager().create(STUDY, dummyFamily1.getMembers().get(i), QueryOptions.empty(), sessionIdUser); + catalogManager.getIndividualManager().create(studyFqn, dummyFamily1.getMembers().get(i), QueryOptions.empty(), ownerToken); } List members = dummyFamily1.getMembers().stream().map(Individual::getId).collect(Collectors.toList()); dummyFamily1.setMembers(null); - catalogManager.getFamilyManager().create(STUDY, dummyFamily1, members, QueryOptions.empty(), sessionIdUser); + catalogManager.getFamilyManager().create(studyFqn, dummyFamily1, members, QueryOptions.empty(), ownerToken); dummyFamily2.setMembers(null); - catalogManager.getFamilyManager().create(STUDY, dummyFamily2, members, QueryOptions.empty(), sessionIdUser); + catalogManager.getFamilyManager().create(studyFqn, dummyFamily2, members, QueryOptions.empty(), ownerToken); - OpenCGAResult result = catalogManager.getFamilyManager().get(STUDY, Arrays.asList(dummyFamily1.getId(), - dummyFamily2.getId()), QueryOptions.empty(), sessionIdUser); + OpenCGAResult result = catalogManager.getFamilyManager().get(studyFqn, Arrays.asList(dummyFamily1.getId(), + dummyFamily2.getId()), QueryOptions.empty(), ownerToken); assertEquals(2, result.getNumResults()); for (Family family : result.getResults()) { if (family.getId().equals(dummyFamily1.getId())) { @@ -1297,12 +1263,12 @@ public void incrementVersionTest() throws CatalogException { } } - catalogManager.getFamilyManager().update(STUDY, dummyFamily1.getId(), new FamilyUpdateParams().setName("name"), - QueryOptions.empty(), sessionIdUser); - catalogManager.getFamilyManager().update(STUDY, dummyFamily1.getId(), new FamilyUpdateParams().setName("name22"), - QueryOptions.empty(), sessionIdUser); - result = catalogManager.getFamilyManager().get(STUDY, Arrays.asList(dummyFamily1.getId(), dummyFamily2.getId()), - QueryOptions.empty(), sessionIdUser); + catalogManager.getFamilyManager().update(studyFqn, dummyFamily1.getId(), new FamilyUpdateParams().setName("name"), + QueryOptions.empty(), ownerToken); + catalogManager.getFamilyManager().update(studyFqn, dummyFamily1.getId(), new FamilyUpdateParams().setName("name22"), + QueryOptions.empty(), ownerToken); + result = catalogManager.getFamilyManager().get(studyFqn, Arrays.asList(dummyFamily1.getId(), dummyFamily2.getId()), + QueryOptions.empty(), ownerToken); assertEquals(2, result.getNumResults()); assertEquals(4, result.first().getVersion()); assertEquals(1, result.getResults().get(1).getVersion()); @@ -1310,7 +1276,7 @@ public void incrementVersionTest() throws CatalogException { Query query = new Query() .append(FamilyDBAdaptor.QueryParams.ID.key(), dummyFamily1.getId()) .append(Constants.ALL_VERSIONS, true); - result = catalogManager.getFamilyManager().search(STUDY, query, QueryOptions.empty(), sessionIdUser); + result = catalogManager.getFamilyManager().search(studyFqn, query, QueryOptions.empty(), ownerToken); assertEquals(1, result.getResults().get(0).getVersion()); assertEquals(2, result.getResults().get(1).getVersion()); @@ -1327,33 +1293,33 @@ public void memberReferenceTest() throws CatalogException { Family dummyFamily2 = DummyModelUtils.getDummyFamily(); for (int i = dummyFamily1.getMembers().size() - 1; i >= 0; i--) { - catalogManager.getIndividualManager().create(STUDY, dummyFamily1.getMembers().get(i), QueryOptions.empty(), sessionIdUser); + catalogManager.getIndividualManager().create(studyFqn, dummyFamily1.getMembers().get(i), QueryOptions.empty(), ownerToken); } List members = dummyFamily1.getMembers().stream().map(Individual::getId).collect(Collectors.toList()); dummyFamily1.setMembers(null); - catalogManager.getFamilyManager().create(STUDY, dummyFamily1, members, QueryOptions.empty(), sessionIdUser); + catalogManager.getFamilyManager().create(studyFqn, dummyFamily1, members, QueryOptions.empty(), ownerToken); dummyFamily2.setMembers(null); - catalogManager.getFamilyManager().create(STUDY, dummyFamily2, members, QueryOptions.empty(), sessionIdUser); + catalogManager.getFamilyManager().create(studyFqn, dummyFamily2, members, QueryOptions.empty(), ownerToken); - OpenCGAResult result = catalogManager.getIndividualManager().get(STUDY, members, QueryOptions.empty(), sessionIdUser); + OpenCGAResult result = catalogManager.getIndividualManager().get(studyFqn, members, QueryOptions.empty(), ownerToken); for (Individual member : result.getResults()) { assertEquals(2, member.getFamilyIds().size()); assertTrue(member.getFamilyIds().containsAll(Arrays.asList(dummyFamily1.getId(), dummyFamily2.getId()))); } // Update family id - catalogManager.getFamilyManager().update(STUDY, dummyFamily1.getId(), new FamilyUpdateParams().setId("newId"), QueryOptions.empty(), - sessionIdUser); - result = catalogManager.getIndividualManager().get(STUDY, members, QueryOptions.empty(), sessionIdUser); + catalogManager.getFamilyManager().update(studyFqn, dummyFamily1.getId(), new FamilyUpdateParams().setId("newId"), QueryOptions.empty(), + ownerToken); + result = catalogManager.getIndividualManager().get(studyFqn, members, QueryOptions.empty(), ownerToken); for (Individual member : result.getResults()) { assertEquals(2, member.getFamilyIds().size()); assertTrue(member.getFamilyIds().containsAll(Arrays.asList("newId", dummyFamily2.getId()))); } // Delete family1 - catalogManager.getFamilyManager().delete(STUDY, Collections.singletonList("newId"), QueryOptions.empty(), sessionIdUser); - result = catalogManager.getIndividualManager().get(STUDY, members, QueryOptions.empty(), sessionIdUser); + catalogManager.getFamilyManager().delete(studyFqn, Collections.singletonList("newId"), QueryOptions.empty(), ownerToken); + result = catalogManager.getIndividualManager().get(studyFqn, members, QueryOptions.empty(), ownerToken); for (Individual member : result.getResults()) { assertEquals(1, member.getFamilyIds().size()); assertEquals(dummyFamily2.getId(), member.getFamilyIds().get(0)); @@ -1366,12 +1332,12 @@ public void updateInUseInCATest() throws CatalogException { Family family = DummyModelUtils.getDummyCaseFamily("family1"); for (int i = family.getMembers().size() - 1; i >= 0; i--) { - catalogManager.getIndividualManager().create(STUDY, family.getMembers().get(i), QueryOptions.empty(), sessionIdUser); + catalogManager.getIndividualManager().create(studyFqn, family.getMembers().get(i), QueryOptions.empty(), ownerToken); } List members = family.getMembers().stream().map(Individual::getId).collect(Collectors.toList()); family.setMembers(null); - catalogManager.getFamilyManager().create(STUDY, family, members, QueryOptions.empty(), sessionIdUser); + catalogManager.getFamilyManager().create(studyFqn, family, members, QueryOptions.empty(), ownerToken); // Unlocked cases ClinicalAnalysis case1 = DummyModelUtils.getDummyClinicalAnalysis(family.getMembers().get(0), family, null); @@ -1380,18 +1346,18 @@ public void updateInUseInCATest() throws CatalogException { // locked true ClinicalAnalysis case3 = DummyModelUtils.getDummyClinicalAnalysis(family.getMembers().get(0), family, null); - catalogManager.getClinicalAnalysisManager().create(STUDY, case1, QueryOptions.empty(), sessionIdUser); - catalogManager.getClinicalAnalysisManager().create(STUDY, case2, QueryOptions.empty(), sessionIdUser); - catalogManager.getClinicalAnalysisManager().create(STUDY, case3, QueryOptions.empty(), sessionIdUser); - catalogManager.getClinicalAnalysisManager().update(STUDY, case3.getId(), new ClinicalAnalysisUpdateParams().setLocked(true), - QueryOptions.empty(), sessionIdUser); + catalogManager.getClinicalAnalysisManager().create(studyFqn, case1, QueryOptions.empty(), ownerToken); + catalogManager.getClinicalAnalysisManager().create(studyFqn, case2, QueryOptions.empty(), ownerToken); + catalogManager.getClinicalAnalysisManager().create(studyFqn, case3, QueryOptions.empty(), ownerToken); + catalogManager.getClinicalAnalysisManager().update(studyFqn, case3.getId(), new ClinicalAnalysisUpdateParams().setLocked(true), + QueryOptions.empty(), ownerToken); // Update family id - catalogManager.getFamilyManager().update(STUDY, family.getId(), new FamilyUpdateParams().setId("newId"), QueryOptions.empty(), - sessionIdUser); + catalogManager.getFamilyManager().update(studyFqn, family.getId(), new FamilyUpdateParams().setId("newId"), QueryOptions.empty(), + ownerToken); - OpenCGAResult result = catalogManager.getClinicalAnalysisManager().get(STUDY, - Arrays.asList(case1.getId(), case2.getId(), case3.getId()), QueryOptions.empty(), sessionIdUser); + OpenCGAResult result = catalogManager.getClinicalAnalysisManager().get(studyFqn, + Arrays.asList(case1.getId(), case2.getId(), case3.getId()), QueryOptions.empty(), ownerToken); case1 = result.getResults().get(0); case2 = result.getResults().get(1); case3 = result.getResults().get(2); @@ -1407,12 +1373,12 @@ public void updateDeleteInUseInCATest() throws CatalogException { Family family = DummyModelUtils.getDummyCaseFamily("family1"); for (int i = family.getMembers().size() - 1; i >= 0; i--) { - catalogManager.getIndividualManager().create(STUDY, family.getMembers().get(i), QueryOptions.empty(), sessionIdUser); + catalogManager.getIndividualManager().create(studyFqn, family.getMembers().get(i), QueryOptions.empty(), ownerToken); } List members = family.getMembers().stream().map(Individual::getId).collect(Collectors.toList()); family.setMembers(null); - catalogManager.getFamilyManager().create(STUDY, family, members, QueryOptions.empty(), sessionIdUser); + catalogManager.getFamilyManager().create(studyFqn, family, members, QueryOptions.empty(), ownerToken); // Unlocked cases ClinicalAnalysis case1 = DummyModelUtils.getDummyClinicalAnalysis(family.getMembers().get(0), family, null); @@ -1421,25 +1387,25 @@ public void updateDeleteInUseInCATest() throws CatalogException { // locked true ClinicalAnalysis case3 = DummyModelUtils.getDummyClinicalAnalysis(family.getMembers().get(0), family, null); - catalogManager.getClinicalAnalysisManager().create(STUDY, case1, QueryOptions.empty(), sessionIdUser); - catalogManager.getClinicalAnalysisManager().create(STUDY, case2, QueryOptions.empty(), sessionIdUser); - catalogManager.getClinicalAnalysisManager().create(STUDY, case3, QueryOptions.empty(), sessionIdUser); - catalogManager.getClinicalAnalysisManager().update(STUDY, case3.getId(), new ClinicalAnalysisUpdateParams().setLocked(true), - QueryOptions.empty(), sessionIdUser); + catalogManager.getClinicalAnalysisManager().create(studyFqn, case1, QueryOptions.empty(), ownerToken); + catalogManager.getClinicalAnalysisManager().create(studyFqn, case2, QueryOptions.empty(), ownerToken); + catalogManager.getClinicalAnalysisManager().create(studyFqn, case3, QueryOptions.empty(), ownerToken); + catalogManager.getClinicalAnalysisManager().update(studyFqn, case3.getId(), new ClinicalAnalysisUpdateParams().setLocked(true), + QueryOptions.empty(), ownerToken); // Delete family try { - catalogManager.getFamilyManager().delete(STUDY, Collections.singletonList(family.getId()), QueryOptions.empty(), sessionIdUser); + catalogManager.getFamilyManager().delete(studyFqn, Collections.singletonList(family.getId()), QueryOptions.empty(), ownerToken); } catch (CatalogException e) { assertTrue(e.getMessage().contains("in use in Clinical Analyses")); } // unlock case3 - catalogManager.getClinicalAnalysisManager().update(STUDY, case3.getId(), new ClinicalAnalysisUpdateParams().setLocked(false), - QueryOptions.empty(), sessionIdUser); + catalogManager.getClinicalAnalysisManager().update(studyFqn, case3.getId(), + new ClinicalAnalysisUpdateParams().setLocked(false), QueryOptions.empty(), ownerToken); try { - catalogManager.getFamilyManager().delete(STUDY, Collections.singletonList(family.getId()), QueryOptions.empty(), sessionIdUser); + catalogManager.getFamilyManager().delete(studyFqn, Collections.singletonList(family.getId()), QueryOptions.empty(), ownerToken); } catch (CatalogException e) { assertTrue(e.getMessage().contains("in use in Clinical Analyses")); } @@ -1453,19 +1419,19 @@ public void updateFamilyMembers() throws CatalogException { child.setFather(father); child.setMother(mother); - catalogManager.getIndividualManager().create(STUDY, father, QueryOptions.empty(), sessionIdUser); - catalogManager.getIndividualManager().create(STUDY, mother, QueryOptions.empty(), sessionIdUser); - catalogManager.getIndividualManager().create(STUDY, child, QueryOptions.empty(), sessionIdUser); + catalogManager.getIndividualManager().create(studyFqn, father, QueryOptions.empty(), ownerToken); + catalogManager.getIndividualManager().create(studyFqn, mother, QueryOptions.empty(), ownerToken); + catalogManager.getIndividualManager().create(studyFqn, child, QueryOptions.empty(), ownerToken); Family family = DummyModelUtils.getDummyFamily("family"); family.setMembers(null); - catalogManager.getFamilyManager().create(STUDY, family, Collections.singletonList(child.getId()), QueryOptions.empty(), - sessionIdUser); + catalogManager.getFamilyManager().create(studyFqn, family, Collections.singletonList(child.getId()), QueryOptions.empty(), + ownerToken); FamilyUpdateParams updateParams = new FamilyUpdateParams().setMembers(Arrays.asList( new IndividualReferenceParam(child.getId(), child.getUuid()), new IndividualReferenceParam(father.getId(), child.getUuid()) )); - catalogManager.getFamilyManager().update(STUDY, family.getId(), updateParams, QueryOptions.empty(), sessionIdUser); + catalogManager.getFamilyManager().update(studyFqn, family.getId(), updateParams, QueryOptions.empty(), ownerToken); } } diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FileManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FileManagerTest.java index 6cf76e3ff12..5e655998abd 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FileManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FileManagerTest.java @@ -31,6 +31,7 @@ import org.opencb.opencga.catalog.db.api.FileDBAdaptor; import org.opencb.opencga.catalog.db.api.SampleDBAdaptor; import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; +import org.opencb.opencga.catalog.db.mongodb.MongoBackupUtils; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; @@ -38,6 +39,7 @@ import org.opencb.opencga.catalog.io.IOManager; import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.ParamUtils; +import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.common.UriUtils; @@ -51,7 +53,6 @@ import org.opencb.opencga.core.models.panel.Panel; import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.study.*; -import org.opencb.opencga.core.models.user.Account; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.MediumTests; @@ -88,7 +89,7 @@ public void setUp() throws Exception { super.setUp(); fileManager = catalogManager.getFileManager(); // Ensure this threshold is restored - fileManager.setFileSampleLinkThreshold(5000); + fileManager.setFileSampleLinkThreshold(organizationId, 5000); } private DataResult link(URI uriOrigin, String pathDestiny, String studyIdStr, ObjectMap params, String sessionId) @@ -99,56 +100,56 @@ private DataResult link(URI uriOrigin, String pathDestiny, String studyIdS @Test public void testCreateFileFromUnsharedStudy() throws CatalogException { try { - fileManager.create(studyFqn, new FileCreateParams() + fileManager.create(studyFqn2, new FileCreateParams() .setType(File.Type.FILE) .setPath("data/test/folder/file.txt") .setDescription("My description"), - true, sessionIdUser2); + true, normalToken1); fail("The file could be created despite not having the proper permissions."); } catch (CatalogAuthorizationException e) { - assertEquals(0, fileManager.search(studyFqn, new Query(FileDBAdaptor.QueryParams.PATH.key(), - "data/test/folder/file.txt"), null, token).getNumResults()); + assertEquals(0, fileManager.search(studyFqn2, new Query(FileDBAdaptor.QueryParams.PATH.key(), + "data/test/folder/file.txt"), null, ownerToken).getNumResults()); } } @Test public void testCreateFileFromSharedStudy() throws CatalogException { StudyAclParams aclParams = new StudyAclParams("", "analyst"); - catalogManager.getStudyManager().updateAcl(Arrays.asList(studyFqn), "user2", aclParams, ParamUtils.AclAction.ADD, token); + catalogManager.getStudyManager().updateAcl(studyFqn, normalUserId2, aclParams, ParamUtils.AclAction.ADD, ownerToken); fileManager.create(studyFqn, new FileCreateParams() .setType(File.Type.FILE) .setPath("data/test/folder/file.txt") .setDescription("My description") .setContent("blabla"), - true, sessionIdUser2); + true, normalToken2); assertEquals(1, fileManager.search(studyFqn, new Query(FileDBAdaptor.QueryParams.PATH.key(), - "data/test/folder/file.txt"), null, token).getNumResults()); + "data/test/folder/file.txt"), null, ownerToken).getNumResults()); } URI getStudyURI() throws CatalogException { return catalogManager.getStudyManager().get(studyFqn, - new QueryOptions(QueryOptions.INCLUDE, StudyDBAdaptor.QueryParams.URI.key()), token).first().getUri(); + new QueryOptions(QueryOptions.INCLUDE, StudyDBAdaptor.QueryParams.URI.key()), ownerToken).first().getUri(); } @Test public void testSearchById() throws CatalogException { Query query = new Query(FileDBAdaptor.QueryParams.ID.key(), "~/^data/"); - OpenCGAResult search = catalogManager.getFileManager().search(studyFqn, query, FileManager.INCLUDE_FILE_IDS, token); - assertEquals(6, search.getNumResults()); + OpenCGAResult search = catalogManager.getFileManager().search(studyFqn, query, FileManager.INCLUDE_FILE_IDS, ownerToken); + assertEquals(11, search.getNumResults()); } @Test public void testLinkCram() throws CatalogException { String reference = getClass().getResource("/biofiles/cram/hg19mini.fasta").getFile(); - File referenceFile = fileManager.link(studyFqn, Paths.get(reference).toUri(), "", null, token).first(); + File referenceFile = fileManager.link(studyFqn, Paths.get(reference).toUri(), "", null, ownerToken).first(); assertEquals(File.Format.FASTA, referenceFile.getFormat()); assertEquals(File.Bioformat.REFERENCE_GENOME, referenceFile.getBioformat()); SmallRelatedFileParams relatedFile = new SmallRelatedFileParams("hg19mini.fasta", FileRelatedFile.Relation.REFERENCE_GENOME); String cramFile = getClass().getResource("/biofiles/cram/cram_with_crai_index.cram").getFile(); DataResult link = fileManager.link(studyFqn, Paths.get(cramFile).toUri(), "", - new ObjectMap("relatedFiles", Collections.singletonList(relatedFile)), token); + new ObjectMap("relatedFiles", Collections.singletonList(relatedFile)), ownerToken); assertTrue(!link.first().getAttributes().isEmpty()); assertNotNull(link.first().getAttributes().get("alignmentHeader")); assertEquals(File.Format.CRAM, link.first().getFormat()); @@ -156,15 +157,15 @@ public void testLinkCram() throws CatalogException { assertEquals(referenceFile.getId(), link.first().getRelatedFiles().get(0).getFile().getId()); assertEquals(FileRelatedFile.Relation.REFERENCE_GENOME, link.first().getRelatedFiles().get(0).getRelation()); - Sample sample = catalogManager.getSampleManager().get(studyFqn, link.first().getSampleIds().get(0), QueryOptions.empty(), token).first(); + Sample sample = catalogManager.getSampleManager().get(studyFqn, link.first().getSampleIds().get(0), QueryOptions.empty(), ownerToken).first(); assertEquals("cram_with_crai_index.cram", sample.getFileIds().get(0)); } @Test public void testLinkAnalystUser() throws CatalogException { - catalogManager.getUserManager().create("analyst", "analyst", "a@mail.com", TestParamConstants.PASSWORD, "", 200000L, Account.AccountType.GUEST, opencgaToken); - catalogManager.getStudyManager().updateAcl(studyFqn, "analyst", new StudyAclParams("", "analyst"), ParamUtils.AclAction.SET, token); - String analystToken = catalogManager.getUserManager().login("analyst", TestParamConstants.PASSWORD).getToken(); + catalogManager.getUserManager().create("analyst", "analyst", "a@mail.com", TestParamConstants.PASSWORD, organizationId, 200000L, opencgaToken); + catalogManager.getStudyManager().updateAcl(studyFqn, "analyst", new StudyAclParams("", "analyst"), ParamUtils.AclAction.SET, ownerToken); + String analystToken = catalogManager.getUserManager().login(organizationId, "analyst", TestParamConstants.PASSWORD).getToken(); String reference = getClass().getResource("/biofiles/cram/hg19mini.fasta").getFile(); File referenceFile = fileManager.link(studyFqn, Paths.get(reference).toUri(), "", null, analystToken).first(); @@ -174,9 +175,9 @@ public void testLinkAnalystUser() throws CatalogException { @Test public void testLinkUserWithNoWritePermissions() throws CatalogException { - catalogManager.getUserManager().create("view_user", "view_user", "a@mail.com", TestParamConstants.PASSWORD, "", 200000L, Account.AccountType.GUEST, opencgaToken); - catalogManager.getStudyManager().updateAcl(studyFqn, "view_user", new StudyAclParams("", "view_only"), ParamUtils.AclAction.SET, token); - String analystToken = catalogManager.getUserManager().login("view_user", TestParamConstants.PASSWORD).getToken(); + catalogManager.getUserManager().create("view_user", "view_user", "a@mail.com", TestParamConstants.PASSWORD, organizationId, 200000L, opencgaToken); + catalogManager.getStudyManager().updateAcl(studyFqn, "view_user", new StudyAclParams("", "view_only"), ParamUtils.AclAction.SET, ownerToken); + String analystToken = catalogManager.getUserManager().login(organizationId, "view_user", TestParamConstants.PASSWORD).getToken(); String reference = getClass().getResource("/biofiles/cram/hg19mini.fasta").getFile(); @@ -187,37 +188,37 @@ public void testLinkUserWithNoWritePermissions() throws CatalogException { @Test public void testLinkFileWithoutReadPermissions() throws IOException, CatalogException { - java.io.File file = createDebugFile("/tmp/file_" + RandomStringUtils.randomAlphanumeric(5) + ".vcf"); + java.io.File file = MongoBackupUtils.createDebugFile("/tmp/file_" + RandomStringUtils.randomAlphanumeric(5) + ".vcf"); Files.setPosixFilePermissions(Paths.get(file.toURI()), new HashSet<>()); thrown.expect(CatalogIOException.class); thrown.expectMessage("read VariantSource"); - fileManager.link(studyFqn, new FileLinkParams().setUri(file.getPath()), false, token); + fileManager.link(studyFqn, new FileLinkParams().setUri(file.getPath()), false, ownerToken); } @Test public void filterByFormatTest() throws CatalogException { Query query = new Query(FileDBAdaptor.QueryParams.FORMAT.key(), "PLAIN"); - OpenCGAResult search = catalogManager.getFileManager().search(studyFqn, query, QueryOptions.empty(), token); - assertEquals(2, search.getNumResults()); + OpenCGAResult search = catalogManager.getFileManager().search(studyFqn, query, QueryOptions.empty(), ownerToken); + assertEquals(3, search.getNumResults()); query = new Query(FileDBAdaptor.QueryParams.FORMAT.key(), "plain"); - search = catalogManager.getFileManager().search(studyFqn, query, QueryOptions.empty(), token); + search = catalogManager.getFileManager().search(studyFqn, query, QueryOptions.empty(), ownerToken); assertEquals(0, search.getNumResults()); // Case sensitive search in lower case query = new Query(FileDBAdaptor.QueryParams.FORMAT.key(), "~/^pla/"); - search = catalogManager.getFileManager().search(studyFqn, query, QueryOptions.empty(), token); + search = catalogManager.getFileManager().search(studyFqn, query, QueryOptions.empty(), ownerToken); assertEquals(0, search.getNumResults()); // Case sensitive in upper case query = new Query(FileDBAdaptor.QueryParams.FORMAT.key(), "~/^PLA/"); - search = catalogManager.getFileManager().search(studyFqn, query, QueryOptions.empty(), token); - assertEquals(2, search.getNumResults()); + search = catalogManager.getFileManager().search(studyFqn, query, QueryOptions.empty(), ownerToken); + assertEquals(3, search.getNumResults()); // Case insensitive search query = new Query(FileDBAdaptor.QueryParams.FORMAT.key(), "~/^pla/i"); - search = catalogManager.getFileManager().search(studyFqn, query, QueryOptions.empty(), token); - assertEquals(2, search.getNumResults()); + search = catalogManager.getFileManager().search(studyFqn, query, QueryOptions.empty(), ownerToken); + assertEquals(3, search.getNumResults()); } @Test @@ -225,7 +226,7 @@ public void createDirectoryTest() throws CatalogException { FileCreateParams params = new FileCreateParams() .setType(File.Type.DIRECTORY) .setPath("/files/folder"); - File file = fileManager.create(studyFqn, params, true, token).first(); + File file = fileManager.create(studyFqn, params, true, ownerToken).first(); assertEquals(params.getPath().substring(1) + "/", file.getPath()); } @@ -236,7 +237,7 @@ public void createDirectoryFailTest() throws CatalogException { .setPath("/files/folder/"); thrown.expect(CatalogException.class); thrown.expectMessage("parents"); - fileManager.create(studyFqn, params, false, token); + fileManager.create(studyFqn, params, false, ownerToken); } @Test @@ -247,8 +248,8 @@ public void createWithBase64File1Test() throws CatalogException { .setFormat(File.Format.IMAGE) .setType(File.Type.FILE) .setPath("/files/folder/heart.png"); - File file = fileManager.create(studyFqn, params, true, token).first(); - FileContent content = fileManager.image(studyFqn, file.getPath(), token).first(); + File file = fileManager.create(studyFqn, params, true, ownerToken).first(); + FileContent content = fileManager.image(studyFqn, file.getPath(), ownerToken).first(); assertEquals(base64, content.getContent()); assertTrue(file.getTags().isEmpty()); assertNull(file.getSoftware()); @@ -267,10 +268,10 @@ public void createWithBase64File2Test() throws CatalogException { .setType(File.Type.FILE) .setFormat(File.Format.IMAGE) .setSoftware(new Software().setName("software")) - .setSampleIds(Arrays.asList(s_1, s_2)) + .setSampleIds(Arrays.asList(s_1Id, s_2Id)) .setPath("/files/folder/heart.png"); - File file = fileManager.create(studyFqn, params, true, token).first(); - FileContent content = fileManager.image(studyFqn, file.getPath(), token).first(); + File file = fileManager.create(studyFqn, params, true, ownerToken).first(); + FileContent content = fileManager.image(studyFqn, file.getPath(), ownerToken).first(); assertEquals(base64, content.getContent()); assertEquals(params.getTags().size(), file.getTags().size()); assertTrue(file.getTags().containsAll(params.getTags())); @@ -281,7 +282,7 @@ public void createWithBase64File2Test() throws CatalogException { assertTrue(file.getSampleIds().containsAll(params.getSampleIds())); OpenCGAResult sampleResult = catalogManager.getSampleManager().get(studyFqn, params.getSampleIds(), QueryOptions.empty(), - token); + ownerToken); assertEquals(2, sampleResult.getNumResults()); for (Sample sample : sampleResult.getResults()) { assertEquals(2, sample.getFileIds().size()); @@ -297,8 +298,8 @@ public void createTxtFileNoExtensionTest() throws CatalogException { .setType(File.Type.FILE) .setPath("/files/folder/file"); - File file = fileManager.create(studyFqn, params, true, token).first(); - FileContent fileContent = fileManager.head(studyFqn, file.getId(), 0, 1, token).first(); + File file = fileManager.create(studyFqn, params, true, ownerToken).first(); + FileContent fileContent = fileManager.head(studyFqn, file.getId(), 0, 1, ownerToken).first(); assertEquals(content, fileContent.getContent().trim()); } @@ -310,8 +311,8 @@ public void createTxtFileTest() throws CatalogException { .setType(File.Type.FILE) .setPath("/files/folder/file.txt"); - File file = fileManager.create(studyFqn, params, true, token).first(); - FileContent fileContent = fileManager.head(studyFqn, file.getId(), 0, 1, token).first(); + File file = fileManager.create(studyFqn, params, true, ownerToken).first(); + FileContent fileContent = fileManager.head(studyFqn, file.getId(), 0, 1, ownerToken).first(); assertEquals(content, fileContent.getContent().trim()); } @@ -324,12 +325,12 @@ public void createWithBase64FileSampleDontExistTest() throws CatalogException { .setBioformat(File.Bioformat.VARIANT) .setType(File.Type.FILE) .setSoftware(new Software().setName("software")) - .setSampleIds(Arrays.asList(s_1, "sample1")) + .setSampleIds(Arrays.asList(s_1Id, "sample1")) .setPath("/files/folder/heart.png"); thrown.expect(CatalogException.class); thrown.expectMessage("sample1"); - fileManager.create(studyFqn, params, true, token).first(); + fileManager.create(studyFqn, params, true, ownerToken).first(); } @Test @@ -343,7 +344,7 @@ public void createWithBase64FileWrongPath1Test() throws CatalogException { thrown.expect(CatalogException.class); thrown.expectMessage("type"); thrown.expectMessage("path"); - fileManager.create(studyFqn, params, true, token); + fileManager.create(studyFqn, params, true, ownerToken); } @Test @@ -353,7 +354,7 @@ public void createWithBase64FileMissingPathTest() throws CatalogException { .setContent(base64); thrown.expect(CatalogException.class); thrown.expectMessage("path"); - fileManager.create(studyFqn, params, true, token).first(); + fileManager.create(studyFqn, params, true, ownerToken).first(); } @Test @@ -363,16 +364,16 @@ public void createWithBase64FileMissingContentTest() throws CatalogException { .setPath("/files/folder/heart.png"); thrown.expect(CatalogException.class); thrown.expectMessage("content"); - fileManager.create(studyFqn, params, true, token).first(); + fileManager.create(studyFqn, params, true, ownerToken).first(); } @Test public void testUpdateRelatedFiles() throws CatalogException { FileUpdateParams updateParams = new FileUpdateParams() .setRelatedFiles(Collections.singletonList(new SmallRelatedFileParams(testFile2, FileRelatedFile.Relation.PRODUCED_FROM))); - fileManager.update(studyFqn, testFile1, updateParams, QueryOptions.empty(), token); + fileManager.update(studyFqn, testFile1, updateParams, QueryOptions.empty(), ownerToken); - File file = fileManager.get(studyFqn, testFile1, QueryOptions.empty(), token).first(); + File file = fileManager.get(studyFqn, testFile1, QueryOptions.empty(), ownerToken).first(); assertEquals(1, file.getRelatedFiles().size()); assertEquals(testFile2, file.getRelatedFiles().get(0).getFile().getPath()); assertEquals(FileRelatedFile.Relation.PRODUCED_FROM, file.getRelatedFiles().get(0).getRelation()); @@ -382,39 +383,39 @@ public void testUpdateRelatedFiles() throws CatalogException { updateParams = new FileUpdateParams() .setRelatedFiles(Collections.singletonList(new SmallRelatedFileParams(testFile2, FileRelatedFile.Relation.PART_OF_PAIR))); - fileManager.update(studyFqn, testFile1, updateParams, new QueryOptions(Constants.ACTIONS, actionMap), token); - file = fileManager.get(studyFqn, testFile1, QueryOptions.empty(), token).first(); + fileManager.update(studyFqn, testFile1, updateParams, new QueryOptions(Constants.ACTIONS, actionMap), ownerToken); + file = fileManager.get(studyFqn, testFile1, QueryOptions.empty(), ownerToken).first(); assertEquals(1, file.getRelatedFiles().size()); assertEquals(testFile2, file.getRelatedFiles().get(0).getFile().getPath()); assertEquals(FileRelatedFile.Relation.PART_OF_PAIR, file.getRelatedFiles().get(0).getRelation()); actionMap.put(FileDBAdaptor.QueryParams.RELATED_FILES.key(), ParamUtils.BasicUpdateAction.REMOVE.name()); - fileManager.update(studyFqn, testFile1, updateParams, new QueryOptions(Constants.ACTIONS, actionMap), token); - file = fileManager.get(studyFqn, testFile1, QueryOptions.empty(), token).first(); + fileManager.update(studyFqn, testFile1, updateParams, new QueryOptions(Constants.ACTIONS, actionMap), ownerToken); + file = fileManager.get(studyFqn, testFile1, QueryOptions.empty(), ownerToken).first(); assertEquals(0, file.getRelatedFiles().size()); // We add it again updateParams = new FileUpdateParams() .setRelatedFiles(Collections.singletonList(new SmallRelatedFileParams(testFile2, FileRelatedFile.Relation.PRODUCED_FROM))); - fileManager.update(studyFqn, testFile1, updateParams, QueryOptions.empty(), token); + fileManager.update(studyFqn, testFile1, updateParams, QueryOptions.empty(), ownerToken); // And now we will update with an empty list updateParams = new FileUpdateParams().setRelatedFiles(Collections.emptyList()); actionMap.put(FileDBAdaptor.QueryParams.RELATED_FILES.key(), ParamUtils.BasicUpdateAction.SET.name()); - fileManager.update(studyFqn, testFile1, updateParams, new QueryOptions(Constants.ACTIONS, actionMap), token); + fileManager.update(studyFqn, testFile1, updateParams, new QueryOptions(Constants.ACTIONS, actionMap), ownerToken); assertEquals(0, file.getRelatedFiles().size()); } @Test public void testLinkVCFandBAMPair() throws CatalogException { String vcfFile = getClass().getResource("/biofiles/variant-test-file.vcf.gz").getFile(); - fileManager.link(studyFqn, new FileLinkParams(vcfFile, "", "", "", null, null, null, null, null), false, token); + fileManager.link(studyFqn, new FileLinkParams(vcfFile, "", "", "", null, null, null, null, null), false, ownerToken); String bamFile = getClass().getResource("/biofiles/NA19600.chrom20.small.bam").getFile(); - fileManager.link(studyFqn, new FileLinkParams(bamFile, "", "", "", null, null, null, null, null), false, token); + fileManager.link(studyFqn, new FileLinkParams(bamFile, "", "", "", null, null, null, null, null), false, ownerToken); Sample sample = catalogManager.getSampleManager().get(studyFqn, "NA19600", - new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.FILE_IDS.key()), token).first(); + new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.FILE_IDS.key()), ownerToken).first(); assertEquals(2, sample.getFileIds().size()); assertTrue(Arrays.asList("variant-test-file.vcf.gz", "NA19600.chrom20.small.bam").containsAll(sample.getFileIds())); } @@ -422,29 +423,29 @@ public void testLinkVCFandBAMPair() throws CatalogException { @Test public void testLinkVirtualWithDifferentSamples() throws CatalogException { String vcfFile = getClass().getResource("/biofiles/variant-test-file.vcf.gz").getFile(); - fileManager.link(studyFqn, new FileLinkParams(vcfFile, "", "", "", null, "biofiles/virtual_file.vcf", null, null, null), false, token); + fileManager.link(studyFqn, new FileLinkParams(vcfFile, "", "", "", null, "biofiles/virtual_file.vcf", null, null, null), false, ownerToken); String bamFile = getClass().getResource("/biofiles/NA19600.chrom20.small.bam").getFile(); thrown.expect(CatalogException.class); thrown.expectMessage("samples differ"); - fileManager.link(studyFqn, new FileLinkParams(bamFile, "", "", "", null, "biofiles/virtual_file.vcf", null, null, null), false, token); + fileManager.link(studyFqn, new FileLinkParams(bamFile, "", "", "", null, "biofiles/virtual_file.vcf", null, null, null), false, ownerToken); } @Test public void testLinkVirtualExcludeType() throws CatalogException { String vcfFile = getClass().getResource("/biofiles/variant-test-file.vcf.gz").getFile(); - fileManager.link(studyFqn, new FileLinkParams(vcfFile, "", "", "", null, "biofiles/virtual_file.vcf", null, null, null), false, token); + fileManager.link(studyFqn, new FileLinkParams(vcfFile, "", "", "", null, "biofiles/virtual_file.vcf", null, null, null), false, ownerToken); OpenCGAResult result; result = fileManager.get(studyFqn, Arrays.asList("variant-test-file.vcf.gz"), - new QueryOptions(QueryOptions.INCLUDE, FileDBAdaptor.QueryParams.RELATED_FILES.key()), token); + new QueryOptions(QueryOptions.INCLUDE, FileDBAdaptor.QueryParams.RELATED_FILES.key()), ownerToken); assertEquals(1, result.getNumResults()); result = fileManager.get(studyFqn, Arrays.asList("variant-test-file.vcf.gz"), - new QueryOptions(QueryOptions.EXCLUDE, "type"), token); + new QueryOptions(QueryOptions.EXCLUDE, "type"), ownerToken); assertEquals(1, result.getNumResults()); } @@ -452,10 +453,10 @@ public void testLinkVirtualExcludeType() throws CatalogException { @Test public void testLinkVirtual() throws CatalogException { String vcfFile = getClass().getResource("/biofiles/variant-test-file.vcf.gz").getFile(); - fileManager.link(studyFqn, new FileLinkParams(vcfFile, "", "", "", null, "biofiles/virtual_file.vcf", null, null, null), false, token); + fileManager.link(studyFqn, new FileLinkParams(vcfFile, "", "", "", null, "biofiles/virtual_file.vcf", null, null, null), false, ownerToken); String vcfFileCopy = getClass().getResource("/biofiles/variant-test-file-copy.vcf.gz").getFile(); - fileManager.link(studyFqn, new FileLinkParams(vcfFileCopy, "", "", "", null, "biofiles/virtual_file.vcf", null, null, null), false, token); + fileManager.link(studyFqn, new FileLinkParams(vcfFileCopy, "", "", "", null, "biofiles/virtual_file.vcf", null, null, null), false, ownerToken); checkTestLinkVirtualFile(false); } @@ -463,19 +464,19 @@ public void testLinkVirtual() throws CatalogException { @Test public void testLinkVirtualOverSampleLinkThreshold() throws CatalogException { String vcfFile = getClass().getResource("/biofiles/variant-test-file.vcf.gz").getFile(); - fileManager.setFileSampleLinkThreshold(2); + fileManager.setFileSampleLinkThreshold(organizationId, 2); - fileManager.link(studyFqn, new FileLinkParams(vcfFile, "", "", "", null, "biofiles/virtual_file.vcf", null, null, null), false, token); + fileManager.link(studyFqn, new FileLinkParams(vcfFile, "", "", "", null, "biofiles/virtual_file.vcf", null, null, null), false, ownerToken); String vcfFileCopy = getClass().getResource("/biofiles/variant-test-file-copy.vcf.gz").getFile(); - fileManager.link(studyFqn, new FileLinkParams(vcfFileCopy, "", "", "", null, "biofiles/virtual_file.vcf", null, null, null), false, token); + fileManager.link(studyFqn, new FileLinkParams(vcfFileCopy, "", "", "", null, "biofiles/virtual_file.vcf", null, null, null), false, ownerToken); checkTestLinkVirtualFile(true); } private void checkTestLinkVirtualFile(boolean missingSamples) throws CatalogException { OpenCGAResult result = fileManager.get(studyFqn, - Arrays.asList("variant-test-file.vcf.gz", "variant-test-file-copy.vcf.gz", "virtual_file.vcf"), QueryOptions.empty(), token); + Arrays.asList("variant-test-file.vcf.gz", "variant-test-file-copy.vcf.gz", "virtual_file.vcf"), QueryOptions.empty(), ownerToken); assertEquals(3, result.getNumResults()); @@ -515,15 +516,15 @@ private void checkTestLinkVirtualFile(boolean missingSamples) throws CatalogExce @Test public void testGetBase64Image() throws CatalogException { String qualityImageFile = getClass().getResource("/fastqc-per_base_sequence_quality.png").getFile(); - fileManager.link(studyFqn, new FileLinkParams(qualityImageFile, "", "", "", null, null, null, null, null), false, token); + fileManager.link(studyFqn, new FileLinkParams(qualityImageFile, "", "", "", null, null, null, null, null), false, ownerToken); - OpenCGAResult result = fileManager.image(studyFqn, "fastqc-per_base_sequence_quality.png", token); + OpenCGAResult result = fileManager.image(studyFqn, "fastqc-per_base_sequence_quality.png", ownerToken); assertEquals(1, result.getNumResults()); assertEquals("iVBORw0KGgoAAAANSUhEUgAAAyAAAAJYCAIAAAAVFBUnAAAgAklEQVR42u3dS3bbRgKGUex/M16EJ72DTHrscXbA+DSPGTRQqHcV8bj/0cCRLV2JgKwvIEUvLzMzMzPrusVNYGZmZiawzMzMzASWmZmZmcAyMzMzM4FlZmZmJrDMzMzMBJaZmZmZCSwzMzMzgWVmZmYmsMzMzMxMYJmZmZkJLDMzMzOBZWZmZiawzMzMzExgmZmZmQksMzMzM4FlZmZmZgLLzMzMTGCZmZmZCSyzZ3xhLMvmF/atW/vrhyD+UTV+eM85wdpvMV+MJrDMvvz3+Gcdv+35y11gBT/UM3x4kbO9y5fDuE4VWCawzC72rbr7/yv7y/2ZgXV0Rp3tGmcksM75tSmwTGCZXbWuIt8I168M/i/+5q32f2bzh4/o4HvOf+XmnR9djYhfpTj6HCOfVzVa95HkBFbcavnYkh9zZmAlz5/g2RL5k/HPsWNgZZ5+Lbdt5BbLfCuNZQLL7GKBFeyk5IWK4HvL+WCC18aC332D38MiH2rOzRJ5/+1oy0eSeaPlNG71O2m8ghUp+MwTLPk+Ix9t9V2EOadf/jlT/d5yvgQElgkss8sEVvIPdw+sim/hRX8s/1Yq+mAq3rbiI6n4kBo/rwmBFT9n6s7PurqtvmW63LYVt5grWCawzC4cWMF7YXK+9wfvwghePOgVWJG72JKXN+L3cpYG1hHa+JGUfkj5N0j1rff1wMq8lzDzrsDkERFYZgLLrENg1X37zAysQd+fqm+BjlewuryyMbCKPrZxt97QwMo/B3p9gpkfrcAyE1j26MbKeVjM5n/lRwdW0UNYjv5Y9XfQnMdg5Xxs7YE17jFY+bf8oMBKXgeq7rPSwMr/UYz8wOr7GKzkl6fAMoFldqLGOro/JfijSTk/6PQ6vkes9MMo/SnCo7dN/pRf8A1fx3c/5aAVr8z5SOp+ijDy3pK3fOmtlx9GR5/aq/kuwlfefXzBG63xHuTqQ59/uHM+ZXVlAsvsjFezLkrYE04kK6pbM4FldudvjWd4dmyze3/9+hIzgWXmr34z6/bF5UvMBJaZmZmZCSwzMzMzgWVmZmYmsMzMzMwEVu0/NW9mZmZmh4G1iarNL4L/aWZmZmYCy8zMzGxuYCX/sY79f/749eM/y1L08vtNfv76+ful9A1/v8nvtyUSiUQikUgsFZfC/X6Tzlew8v99sXdgBV8iv7X+VP/+O/dlWd24pW/YLlZ/jsGX93s+ermceKHjSCQSiURivjj2LsL4P2wusASWL0sikUgkCqyej8ESWALLlyWRSCQSBdZhYO3vExRYAktgEYlEIlFgtQZWxfNgCSyB5cuSSCQSiQIr9y7C3DcWWALLlyWRSCQSBZbAElgCi0gkEolEgSWwBJa/CIhEIpEosASWwPJlSSQSiUSBJbAElsAiEolEIlFgXSCwCp/Kv8OLwCISiUQiUWD1uXGL4qNCvFDuzE+65Iu/CIhEIpEosC4QWDnfwoM3bnUiuIuw5Tjuj5S/CErF0nPVrUokEon3Dqys60mpHX4LKVrLv6QtsHICK2/fP46RU6v7S8nnGH9XuZ/jCS9hnuel35kz/1wlPlSUO93F0r84LnkFq/peNrlzDzFyjbCXmH9hcsLnWHSV1JnzkDvQb/MoTKLHtrqCdbq7CH1ZEjff+Tp+m3QciSLSAyEElsASWL4siVnfIdyqROIXr7a6SiewBJbAIhKJRKKrdFe9Sid3BJbAIhKJRKKrdJ2TTu4ILIFFJBKJRGK3pJM7AktgEYlEIpF4rrtBBZbA8mVJJBKJROLAJ+J2zUxg+bIkEolEIvE7j8Gq+NfPBJbAIhKJRCJRYPWJj9J/fFZg/VtLn+1DKvhbAotIJBKJRE/TUPQPp17ocWb9r2CtQ2rfWwKLSCQSiUSBNVnMvxPzAoEVqSuBRSQSiUSiwDqPGHy42CkCa38/4PvXkbsOH/JF8oR/9d1fdkQikXiewLrK946ziu/SahU7X8Fat9TRr13BSr5E3mf85VSiv+yIRCKReFGx/V+xnHcXocASWP4iIBKJROJFn75VYAksgUUkEolEYmdxn1nDA2vzuKvgTxG6i1Bg+YuASCQSiVcX1xe0ZgRW5MmuPMhdYPmyJBKJROLNxOQjtDyTu8ASWEQikUgkdhYFlsASWEQikUgkCiyBJbCIRCKRSBRYAktg+bIkEolEosASWAJLYBGJRCKRKLAElsAiEolEIlFgCSyB5cuSSCQSiQJLYAksX5ZEIpFIJF4hsJ7wb3c/IbD8G+xEIpE44XuH3BFYTVewnnB1h0gkEolEuSOwBBaRSCQSiQKLKLB8WRKJRCJRYMkdgeXLkkgkEolEuSOwBBaRSCQSiQKLKLCIRCKRSBRYRIHly5JIJBKJRLkjsAQWkUgkEokCS2AJLCKRSCQSBRbxhIG1fhbaYEvtXy+wiEQikUiUOwIr9wpWsKUEFpFIJBKJAktgdQus938KLCKRSCQSBZbAKg6s4F2En9cILCKRSCQS80X/oLXAit0b+Pm1wCISiUQikegKVoe7CIOlJbCIRCKRSCQKrKbA2kxgEYlEIpFIFFhZgbW5UnX0TA2uYBGJRCKRSBRYBYEVfx4sgUUkEolEIlFgeSZ3IpFIJBKJAktgOWWJRCKRSBRYAktgEYlEIpFIFFgCi0gkEolEosASWE5ZIpFIJBIFlsByyhKJRCKRKLAElsAiEolEIpEosAQWkUgkEolEgSWwnLJEIpFIJN5PXAr3EUvfsF0UWE5ZIpFIJBKJM0SB5QQiEolEIpEosBxOIpFIJBKJAktgEYlEIpFIFFgCi0gkEolEIlFgOZxEIpFIJBIFlsAiEolEIpEosAQWkUgkEolEosByAhGJRCKRSBRYAotIJBKJRKLAygqs9fPKx18psIhEIpFIJAqs4itYn5zaxJbAIhKJRCKRKLBaAyv+SoFFJBKJRCJRYCUCK3JvoMAiEolEIpF4XXEpXP8rWMHG8hgsIpFIJBKJjxKH30UYuablcBKJRCKRSBRYxYEVv9PQ4SQSiUQikSiwwoF1VFSRuhJYRCKRSCQSBVYisJLPg+VB7kQikUgkEgWWZ3InEolEIpFIFFgOJ5FIJBKJRIHlcBKJRCKRSBRYAotIJBKJRCJRYDmBiEQikUgkCiyHk0gkEolEosASWEQikUgkEokCi0gkEolEIlFgOZxEIpFIJBIFlsAiEolEIpEosAQWkUgkEolEosByOIlEIpFIJAosgUUkEolEIlFgCSwikUgkEolEgeUEIhKJRCKRKLAcTiKRSCQSiQJLYBGJRCKRSCQKLCcQkUgkEonE6wXWstqmooKvF1hEIpFIJBIFVsEVrHVLfX4tsIhEIpFIJAqsDoG1v5olsIhEIpFIJAqsgsDa3xUosIhEIpFIJAqsDlew1o0lsIhEIpFIJAosdxESiUQikUgkCiyHk0gkEolE4i0Da/O4Kz9FSCQSiUQiUWB1CCzPg0UkEolEIpHomdyJRCKRSCQSBZbDSSQSiUQiUWAJLCKRSCQSiQJLYBGJRCKRSCQKLIeTSCQSiUSiwBJYRCKRSCQSBZbAIhKJRCKRSBRYTiAikUgkEokCy+EkEolEIpEosAQWkUgkEolEosByAhGJRCKRSBRYDieRSCQSiUSBJbCIRCKRSCQSBRaRSCQSiUSiwHI4iUQikUgkCiyBRSQSiUQiUWAJLCKRSCQSiUSB5XASiUQikUi8YWAtq20qKvh6gUUkEolEIlFgpQNr/+t9bAksIpFIJBKJAqvmLkKBRSQSiUQiUWB9LbB+/fWj5uW/Pytf6jgikUgkEonEwpfOgVX0GCyHk0gkEolEosBKBFawrva/FlhEIpFIJBIFVlZgBRPKXYREIpFIJBIFVtPTNCQvaAksIpFIJBKJAqvmebA2T9ngMVhEIpFIJBIF1rxncnc4iUQikUgkCiyBRSQSiUQikSiwnEBEIpFIJBIFlsNJJBKJRCJRYAksIpFIJBKJRIFFJBKJRCKRKLAcTiKRSCQSiQJLYBGJRCKRSBRYAotIJBKJRCJRYDmcRCKRSCQSBZbAIhKJRCKRKLAEFpFIJBKJRKLAcgIRiUQikUgUWA4nkUgkEolEgSWwiEQikUgkEgWWE4hIJBKJRKLAcjiJRCKRSCQKLIFFJBKJRCKRKLCIRCKRSCQSTx1Yy2r7kAr+lsAiEolEIpEosBKBlfy1K1hEIpFIJBIFVuVdhJ+oitSVwCISiUQikSiw6gMrctehw0kkEolEIlFgpQNrc//gOrY8BotIJBKJRKLAKg6sfULF/9PhJBKJRCKRKLBigRW8RiWwiEQikUgkCqymp2mIPx5LYBGJRCKRSBRYlc+DtX8Ylge5E4lEIpFIFFieyZ1IJBKJRCJRYDmcRCKRSCQSBZbAIhKJRCKRKLAEFpFIJBKJRKLAcgIRiUQikUgUWA4nkUgkEolEgSWwiEQikUgkEgUWkUgkEolEosByOIlEIpFIJAosgUUkEolEIlFgCSwikUgkEolEgeVwEolEIpFIFFgCi0gkEolEosASWEQikUgkEokCy+EkEolEIpEosBxOIpFIJBKJAktgEYlEIpFIJAosJxCRSCQSicRLBtayWrCl9q8XWEQikUgkEgVWIrCCvxZYRCKRSCQSBVaHuwg3LfX+T4FFJBKJRCJRYPUJrM+1K4FFJBKJRCJRYFUGVvDylcAiEolEIpEosCoD66iuBBaRSCQSiUSBVRNY+0eyL7sJLCKRSCQSiQKr7Gka4s965QoWkUgkEolEgVX5PFhHT4UlsIhEIpFIJAosz+ROJBKJRCKRKLCcQEQikUgkEgWWw0kkEolEIlFgCSwikUgkEolEgeUEIhKJRCKRKLAcTiKRSCQSiQJLYBGJRCKRSCQKLCKRSCQSiUSB5XASiUQikUgUWAKLSCQSiUSiwBJYRCKRSCQSiQLL4SQSiUQikSiwBBaRSCQSiUSBJbCIRCKRSCQSBZYTiEgkEolEosByOIlEIpFIJAosgUUkEolEIpEosJxARCKRSCQSLxZYy2rxVwosIpFIJBKJAis3sPa/Dr5SYBGJRCKRSBRYxXcRHl2vElhEIpFIJBIFlsAiEolEIpFIPEFgZdaVwCISiUQikSiwsgIrv64EFpFIJBKJRIGVDqzgTwse/QihwCISiUQikSiwsp6mIb+uBBaRSCQSiUSBVfA8WOunaYg8FZbAIhKJRCKRKLA8kzuRSCQSiUSiwHICEYlEIpFIFFgOJ5FIJBKJRIElsIhEIpFIJBIFFpFIJBKJRKLAcjiJRCKRSCQKLIFFJBKJRCJRYAksIpFIJBKJRIHlcBKJRCKRSBRYAotIJBKJRKLAElhEIpFIJBKJAsvhJBKJRCKRKLAcTiKRSCQSiQJLYBGJRCKRSCQKLCcQkUgkEolEgeVwEolEIpFIFFgCi0gkEolEIlFgEYlEIpFIJAosh5NIJBKJROLDAmtZbVNRwdcLLCKRSCQSiQIrHVjxXwssIpFIJBKJAqv+LsKjqNr/p8NJJBKJRCJRYPUOrB8/al5+/qx8qeOIRCKRSCQSC186B9bRfYUCi0gkEolEosCqCaxkUQksIpFIJBKJAqsgsII/QiiwiEQikUgkCqymp2nIfzyWwCISiUQikSiwCp4Ha/MwrNjzYDmcRCKRSCQSBVbnZ3J3OIlEIpFIJAosgUUkEolEIpEosJxARCKRSCQSBZbDSSQSiUQiUWAJLCKRSCQSiUSBRSQSiUQikSiwHE4ikUgkEokCS2ARiUQikUgUWAKLSCQSiUQiUWA5nEQikUgkEgWWwCISiUQikSiwBBaRSCQSiUSiwHICEYlEIpFIFFgOJ5FIJBKJRIElsIhEIpFIJBIFlhOISCQSiUSiwHI4iUQikUgkCiyBRSQSiUQikdg1sJY/21dU5LccTiKRSCQSiQIrcQVrU1HJ/3Q4iUQikUgkCiyBRSQSiUQikSiwnEBEIpFIJBLvFFgeg0UkEolEIlFg9b+C9XnNvrEEFpFIJBKJRIHlLkIikUgkEolEgeUEIhKJRCKReIPnwdrcFegxWEQikUgkEgWWZ3InEolEIpFIFFgOJ5FIJBKJRIElsIhEIpFIJAosgUUkEolEIpEosJxARCKRSCQSBZbDSSQSiUQiUWAJLCKRSCQSiUSB5QQiEolEIpEosBxOIpFIJBKJAktgEYlEIpFIJAosIpFIJBKJRIHlcBKJRCKRSBRYAotIJBKJRKLAElhEIpFIJBKJAsvhJBKJRCKRKLAEFpFIJBKJRIElsIhEIpFIJBIFlhOISCQSiUSiwHI4iUQikUgkCqxVLb0XDKng7wosIpFIJBKJAit9BWsfWMHkElhEIpFIJBIFVmVgRepKYBGJRCKRSBRY9YF1dO+hwCISiUQikSiwagLr8xqPwSISiUQikSiw+t9FKLCIRCKRSCQKLIFFJBKJRCKReOKfInQXIZFIJBKJRIFV+TxYm5A6ehIsgUUkEolEIlFgeSZ3IpFIJBKJRIHlBCISiUQikSiwHE4ikUgkEokCS2ARiUQikUgkCiwikUgkEolEgeVwEolEIpFIFFgCi0gkEolEosASWEQikUgkEokCy+EkEolEIpEosAQWkUgkEolEgSWwiEQikUgkEgWWw0kkEolEIlFgCSwikUgkEokCS2ARiUQikUgkCiwnEJFIJBKJRIHlcBKJRCKRSBRYAotIJBKJRCJRYBGJRCKRSCReILCWPztqqf1vCSwikUgkEokCK30FS2ARiUQikUgkzgis9ysFFpFIJBKJRIHVJ7A+164EFpFIJBKJRIHVLbAiv+VwEolEIpFIFFhlgbX+T4FFJBKJRCJRYPUJrM0EFpFIJBKJRIHV56cIXcEiEolEIpEosOqfB+vo2bAEFpFIJBKJRIHlmdyJRCKRSCQSBZbDSSQSiUQiUWA5nEQikUgkEgWWwCISiUQikUgUWE4gIpFIJBKJAsvhJBKJRCKRKLAEFpFIJBKJRKLAIhKJRCKRSBRYDieRSCQSiUSBJbCIRCKRSCQKLIFFJBKJRCKRKLAcTiKRSCQSiQJLYBGJRCKRSBRYAotIJBKJRCJRYDmBiEQikUgkCiyHk0gkEolEosASWEQikUgkEokCywlEJBKJRCLxeoG1/NkmoYKvF1hEIpFIJBIFVu4VrH1gHf2WwCISiUQikSiwagIr/lsCi0gkEolEosASWEQikUgkEomnCSyPwSISiUQikSiwegZWpLocTiKRSCQSiQKrOLCOfoRQYBGJRCKRSBRYlT9FGH9UlsNJJBKJRCJRYKWfB2sdVcFXCiwikUgkEokCyzO5E4lEIpFIJAosJxCRSCQSiUSB5XASiUQikUgUWAKLSCQSiUQiUWA5gYhEIpFIJAosh5NIJBKJRKLAElhEIpFIJBKJAotIJBKJRCJRYDmcRCKRSCQSBZbAIhKJRCKRKLAEFpFIJBKJRKLAcjiJRCKRSCQKLIFFJBKJRCJRYAksIpFIJBKJRIHlBCISiUQikSiwHE4ikUgkEokCS2ARiUQikUgkCiwikUgkEolEgeVwEolEIpFIfGpgLX+2r6jIbzmcRCKRSCQSBVbiClawoiK/5XASiUQikUgUWGWBlfxPh5NIJBKJRKLAElhEIpFIJBKJAssJRCQSiUQiUWA5nEQikUgkEgWWwCISiUQikUj0U4REIpFIJBKJF3oerM1TXnkeLCKRSCQSiQLLM7kTiUQikUgkCiyHk0gkEolEosASWEQikUgkEgWWwCISiUQikUgUWE4gIpFIJBKJAsvhJBKJRCKRKLAEFpFIJBKJRKLAcgIRiUQikUgUWA4nkUgkEolEgSWwiEQikUgkEgUWkUgkEolEosByOIlEIpFIJAosgUUkEolEIlFgCSwikUgkEolEgeVwEolEIpFIFFgCi0gkEolEosASWEQikUgkEokCywlEJBKJRCLx2oG1rCawiEQikUgkCqzWwNpEVU5jCSwikUgkEokCS2ARiUQikUgkCiwnEJFIJBKJRI/BcjiJRCKRSCQKrP+rq/2vBRaRSCQSiUSB5S5CIpFIJBKJRIHlcBKJRCKRSPQYLIFFJBKJRCJRYHkmdyKRSCQSiUSB5QQiEolEIpEosBxOIpFIJBKJAktgEYlEIpFIJAosJxCRSCQSiUSB5XASiUQikUgUWAKLSCQSiUQiUWARiUQikUgkCiyHk0gkEolEosASWEQikUgkEgWWwCISiUQikUgUWA4nkUgkEolEgSWwiEQikUgkCiyBRSQSiUQikSiwnEBEIpFIJBIFlsNJJBKJRCJRYAksIpFIJBKJRIFFJBKJRCKReKXAWlYTWEQikUgkEgVWa2DlRJXAIhKJRCKRKLByA6u0rgQWkUgkEolEgZUVWPn3DwosIpFIJBKJAisdWJ+u8hgsIpFIJBKJAqv/XYQCi0gkEolEosASWEQikUgkEokn/ilCdxESiUQikUgUWN0Cy4PciUQikUgkEj2TO5FIJBKJRKLAcjiJRCKRSCQKLIFFJBKJRCJRYAksIpFIJBKJRIHlcBKJRCKRSBRYAotIJBKJRKLAElhEIpFIJBKJAssJRCQSiUQiUWA5nEQikUgkEgWWwCISiUQikUgUWEQikUgkEokCy+EkEolEIpEosAQWkUgkEolEgSWwiEQikUgkEgWWw0kkEolEIlFgCSwikUgkEokCS2ARiUQikUgkCiyHk0gkEolEosASWEQikUgkEgVWRmAt/5vAIhKJRCKRKLAEFpFIJBKJROIpA+udVgKLSCQSiUSiwOoTWJ9rVwKLSCQSiUSiwOoWWJtfCCwikUgkEokCqz6w1lElsIhEIpFIJAqsPoG1mcAiEolEIpEosLo9D5YrWEQikUgkEgWWwCISiUQikUj0TO4OJ5FIJBKJRIElsIhEIpFIJAosgUUkEolEIpEosBxOIpFIJBKJAsvhJBKJRCKRKLAEFpFIJBKJRKLAcgIRiUQikUgUWA4nkUgkEolEgSWwiEQikUgkEgUWkUgkEolEosByOIlEIpFIJAosgUUkEolEIlFgCSwikUgkEolEgeVwEolEIpFIFFgCi0gkEolEosASWEQikUgkEokCywlEJBKJRCJRYDmcRCKRSCQSBVYosJbVBBaRSCQSiUSB1Sewgr8WWEQikUgkEgVWh7sIBRaRSCQSiUSBJbCIRCKRSCQSTxxYHoNFJBKJRCKR2DOw8utKYBGJRCKRSBRY6cAq+hFCgUUkEolEIlFgZT1Ng+fBIhKJRCKRSBzyPFge5E4kEolEIlFgeSZ3IpFIJBKJRIHlcBKJRCKRSBRYAotIJBKJRKLAElhEIpFIJBKJAsvhJBKJRCKRKLAcTiKRSCQSiQJLYBGJRCKRSCQKLCcQkUgkEolEgeVwEolEIpFIFFgCi0gkEolEIlFgEYlEIpFIJAosh5NIJBKJRKLAElhEIpFIJBIFlsAiEolEIpFIFFgOJ5FIJBKJRIElsIhEIpFIJAosgUUkEolEIpEosJxARCKRSCQS7xNYy2oCi0gkEolEosDqE1ibXwgsIpFIJBKJAqs+sDZRldNYAotIJBKJRKLA6h9YZmZmZs/cqMAyMzMzM4FlZmZmJrDMzMzMrhtYr/KfIjQzMzOzdGAVPQ+WmZmZmSUCq897rE206rxr7MLJYsXbFv3Ywldu1aM/OS7Zg+926P8hxN/ztM9x6Gf6xeM47VAG3/OcMydyKKeJo8+c7x5HZ44zp/r7/ghx1JWquvJoefMWtyI+pn14LR9t9ePq1ko19xp/p3PkE5wjDv3LLnmrzvniHfrozKNDNu7MSSpzxNHfJmd+UWQeu+/qg8Q5N++0M2fyX6eZ3y/Gxevo71YnCqyvFMz7z88JrC6HrfTyVXVgVbzhzG/Myfc5M7DqzqLRB/ESgfWtv16TR3PmbTs/BW7zbdKZ48yphiIF2Uu8SWC13AvWwlXkzsx7MwXWzP+PnBlYoy+eC6w5VwXmf5uc8Cjbmd8m93fYTRbnXGgJ3pk1TRx95kw+jsFb70GB1XL5qq4/5oib+90a70Ud3YIVH+p5Amvmt+T5d4M2nkXVETn0G/PMb5Onugjxmnjf2egz5+h0HX3mTE7zo4coDT1zJt8NenS2THtQ1NDH0k37O+d0gTXt0levO5gnPCKq+k1avjDWJ/e1Ausr3yNfX7qLcMKtOj/pnDnOnPZvmTP/R2vmHffzH/X1lWvYg86czIcn3jOw2m/KOT+a95XAan8w/ujP8QzfJmc+8LPjWXTyb5NDxcijTaeJEwpg8plzhlv1IXcuz3lio01wzLyofKfjeHTr3T+w2u81a3ls0xyxpcrbr3iNvnFOdffZ/BN7/k8R3iDpIj+CfuO6Gn3m5NyqE47j0G+TR/SgM+e7d7YmD+jMW3XCcfzKgwSu8VOEXZ6xqfpC1IQcbBEb37DxcIx7q6PbZM5z0hz9v93kE3uaOP9zvL3ozLmWGLlJJ4vjzpyc93zXW/U1/XGfr2s9D5aZmZnZYyewzMzMzASWmZmZmcAyMzMzE1hmZmZmJrDMzMzMBJaZmZmZwDIzMzMzgWVmZmYmsMzMzMwElpk99e+OYf98x4R/Orf6M2r/J0HNTGCZ2W3bqOM/vHWbgBBYZiawzKxbSTQWwL0Dq/22ElhmAsvMnh5Y+f+4/fsXR6+PvLf1G8Y/vMw/udGDYvLzKgqs+Ce1fz8ay0xgmdlzA2tfKpE+OPrdnPeW/56Tf3IfUvG3zfmo4rdV0ScosMwElpk9KLAi15aSCZUfWMnX57/nivvvMq+QdSHqPhIzE1hmdqvAynl98F62kwdWpB3j92YKLDMTWGY2O7AiSXHaK1iNVSSwzExgmVmfwMp/TFLLY7DGBVbyj3kMlpkJLDObGlivjJ/COwqp0p8iHBFYr+jDy4I/LZj8KcLgh1H0U4TqykxgmZlZQY9mNqvb0ExgmZnZkAgzM4FlZmYCy8wElpmZmZnAMjMzM7vr/gHlNE9yyIVrkAAAAABJRU5ErkJg", result.first().getContent()); thrown.expect(CatalogException.class); thrown.expectMessage("not an image"); - fileManager.image(studyFqn, "test_1K.txt.gz", token); + fileManager.image(studyFqn, "test_1K.txt.gz", ownerToken); } @Test @@ -539,30 +540,30 @@ public void testLinkFolder() throws CatalogException, IOException { // assertEquals("Linking the same folders should not change the number of files in catalog", numFiles, numFilesAfterLink); // Now we try to create it into a folder that does not exist with parents = true - link(uri, "myDirectory", studyFqn, new ObjectMap("parents", true), token); + link(uri, "myDirectory", studyFqn, new ObjectMap("parents", true), ownerToken); DataResult folderDataResult = fileManager.search(studyFqn, new Query() - .append(FileDBAdaptor.QueryParams.PATH.key(), "myDirectory/"), null, token); + .append(FileDBAdaptor.QueryParams.PATH.key(), "myDirectory/"), null, ownerToken); assertEquals(1, folderDataResult.getNumResults()); assertTrue(!folderDataResult.first().isExternal()); folderDataResult = fileManager.search(studyFqn, new Query(FileDBAdaptor.QueryParams.PATH.key(), - "myDirectory/A/"), null, token); + "myDirectory/A/"), null, ownerToken); assertEquals(1, folderDataResult.getNumResults()); assertTrue(folderDataResult.first().isExternal()); folderDataResult = fileManager.search(studyFqn, new Query(FileDBAdaptor.QueryParams.PATH.key(), - "myDirectory/A/C/D/"), null, token); + "myDirectory/A/C/D/"), null, ownerToken); assertEquals(1, folderDataResult.getNumResults()); assertTrue(folderDataResult.first().isExternal()); folderDataResult = fileManager.search(studyFqn, new Query(FileDBAdaptor.QueryParams.PATH.key(), - "myDirectory/A/B/"), null, token); + "myDirectory/A/B/"), null, ownerToken); assertEquals(1, folderDataResult.getNumResults()); assertTrue(folderDataResult.first().isExternal()); // Now we try to create it into a folder that does not exist with parents = false thrown.expect(CatalogException.class); thrown.expectMessage("already linked"); - link(uri, "myDirectory2", studyFqn, new ObjectMap(), token); + link(uri, "myDirectory2", studyFqn, new ObjectMap(), ownerToken); } @Test @@ -573,7 +574,7 @@ public void testLinkFolder2() throws CatalogException, IOException { // Now we try to create it into a folder that does not exist with parents = false thrown.expect(CatalogException.class); thrown.expectMessage("not exist"); - link(uri, "myDirectory2", studyFqn, new ObjectMap(), token); + link(uri, "myDirectory2", studyFqn, new ObjectMap(), ownerToken); } @@ -582,7 +583,7 @@ public void testLinkFolder3() throws CatalogException, IOException { URI uri = Paths.get(getStudyURI()).resolve("data").toUri(); thrown.expect(CatalogException.class); thrown.expectMessage("already existed and is not external"); - link(uri, null, studyFqn, new ObjectMap(), token); + link(uri, null, studyFqn, new ObjectMap(), ownerToken); // // Make sure that the path of the files linked do not start with / // Query query = new Query(FileDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) @@ -601,10 +602,10 @@ public void testLinkFolder3() throws CatalogException, IOException { public void testLinkFolder4() throws CatalogException, IOException { URI uri = Paths.get(getStudyURI()).resolve("data").toUri(); ObjectMap params = new ObjectMap("parents", true); - DataResult allFiles = link(uri, "test/myLinkedFolder/", studyFqn, params, token); - assertEquals(6, allFiles.getNumResults()); + DataResult allFiles = link(uri, "test/myLinkedFolder/", studyFqn, params, ownerToken); + assertEquals(11, allFiles.getNumResults()); - DataResult sameAllFiles = link(uri, "test/myLinkedFolder/", studyFqn, params, token); + DataResult sameAllFiles = link(uri, "test/myLinkedFolder/", studyFqn, params, ownerToken); assertEquals(allFiles.getNumResults(), sameAllFiles.getNumResults()); List result = allFiles.getResults(); @@ -616,7 +617,7 @@ public void testLinkFolder4() throws CatalogException, IOException { thrown.expect(CatalogException.class); thrown.expectMessage("already linked"); - link(uri, "data", studyFqn, new ObjectMap(), token); + link(uri, "data", studyFqn, new ObjectMap(), ownerToken); } @Test @@ -624,7 +625,7 @@ public void testLinkNormalizedUris() throws CatalogException, IOException, URISy Path path = createExternalDummyData(); URI uri = UriUtils.createUri(path.toString() + "/../A"); ObjectMap params = new ObjectMap("parents", true); - DataResult allFiles = link(uri, "test/myLinkedFolder/", studyFqn, params, token); + DataResult allFiles = link(uri, "test/myLinkedFolder/", studyFqn, params, ownerToken); assertEquals(6, allFiles.getNumResults()); for (File file : allFiles.getResults()) { assertTrue(file.getUri().isAbsolute()); @@ -651,19 +652,19 @@ public void testLinkNonExistentFile() throws CatalogException, IOException { ObjectMap params = new ObjectMap("parents", true); thrown.expect(CatalogException.class); thrown.expectMessage("does not exist"); - link(uri, "test/myLinkedFolder/", studyFqn, params, token); + link(uri, "test/myLinkedFolder/", studyFqn, params, ownerToken); } // The VCF file that is going to be linked contains names with "." Issue: #570 @Test public void testLinkFile() throws CatalogException, IOException, URISyntaxException { URI uri = getClass().getResource("/biofiles/variant-test-file-dot-names.vcf.gz").toURI(); - DataResult link = fileManager.link(studyFqn, uri, ".", new ObjectMap(), token); + DataResult link = fileManager.link(studyFqn, uri, ".", new ObjectMap(), ownerToken); assertEquals(4, link.first().getSampleIds().size()); Query query = new Query(SampleDBAdaptor.QueryParams.ID.key(), link.first().getSampleIds()); - DataResult sampleDataResult = catalogManager.getSampleManager().search(studyFqn, query, QueryOptions.empty(), token); + DataResult sampleDataResult = catalogManager.getSampleManager().search(studyFqn, query, QueryOptions.empty(), ownerToken); assertEquals(4, sampleDataResult.getNumResults()); List sampleNames = sampleDataResult.getResults().stream().map(Sample::getId).collect(Collectors.toList()); @@ -679,7 +680,7 @@ public void testLinkFilePassingNoDirectoryPath() throws CatalogException, URISyn String path = "A/B/C/variant-test-file-dot-names.vcf.gz"; // Instead of choosing a directory path, we will set a file equivalent to the file name (not how OpenCGA asks for the path) - OpenCGAResult link = fileManager.link(studyFqn, new FileLinkParams().setUri(uri.toString()).setPath(path), true, token); + OpenCGAResult link = fileManager.link(studyFqn, new FileLinkParams().setUri(uri.toString()).setPath(path), true, ownerToken); assertEquals(path, link.first().getPath()); assertEquals(File.Type.FILE, link.first().getType()); } @@ -687,27 +688,28 @@ public void testLinkFilePassingNoDirectoryPath() throws CatalogException, URISyn @Test public void testMoveFiles() throws CatalogException { // Generate data set - catalogManager.getFileManager().create(studyFqn, new FileCreateParams().setType(File.Type.DIRECTORY).setPath("A/B/C/D/"), true, token); - catalogManager.getFileManager().create(studyFqn, new FileCreateParams().setType(File.Type.FILE).setPath("A/B/C/hello.txt").setContent("test"), false, token); - catalogManager.getFileManager().create(studyFqn, new FileCreateParams().setType(File.Type.FILE).setPath("A/B/C/hello2.txt").setContent("test"), false, token); - catalogManager.getFileManager().create(studyFqn, new FileCreateParams().setType(File.Type.FILE).setPath("A/B/C/D/hello.txt").setContent("test"), false, token); - catalogManager.getFileManager().create(studyFqn, new FileCreateParams().setType(File.Type.FILE).setPath("A/B/C/D/hello2.txt").setContent("test"), false, token); - catalogManager.getFileManager().create(studyFqn, new FileCreateParams().setType(File.Type.FILE).setPath("A/B/C/D/hello3.txt").setContent("test"), false, token); + catalogManager.getFileManager().create(studyFqn, new FileCreateParams().setType(File.Type.DIRECTORY).setPath("A/B/C/D/"), true, ownerToken); + catalogManager.getFileManager().create(studyFqn, new FileCreateParams().setType(File.Type.FILE).setPath("A/B/C/hello.txt").setContent("test"), false, ownerToken); + catalogManager.getFileManager().create(studyFqn, new FileCreateParams().setType(File.Type.FILE).setPath("A/B/C/hello2.txt").setContent("test"), false, ownerToken); + catalogManager.getFileManager().create(studyFqn, new FileCreateParams().setType(File.Type.FILE).setPath("A/B/C/D/hello.txt").setContent("test"), false, ownerToken); + catalogManager.getFileManager().create(studyFqn, new FileCreateParams().setType(File.Type.FILE).setPath("A/B/C/D/hello2.txt").setContent("test"), false, ownerToken); + catalogManager.getFileManager().create(studyFqn, new FileCreateParams().setType(File.Type.FILE).setPath("A/B/C/D/hello3.txt").setContent("test"), false, ownerToken); - catalogManager.getSampleManager().create(studyFqn, new Sample().setId("sam1"), QueryOptions.empty(), token); - catalogManager.getSampleManager().create(studyFqn, new Sample().setId("sam2"), QueryOptions.empty(), token); - catalogManager.getSampleManager().create(studyFqn, new Sample().setId("sam3"), QueryOptions.empty(), token); + catalogManager.getSampleManager().create(studyFqn, new Sample().setId("sam1"), QueryOptions.empty(), ownerToken); + catalogManager.getSampleManager().create(studyFqn, new Sample().setId("sam2"), QueryOptions.empty(), ownerToken); + catalogManager.getSampleManager().create(studyFqn, new Sample().setId("sam3"), QueryOptions.empty(), ownerToken); Map fileMap = new HashMap<>(); - File file = catalogManager.getFileManager().update(studyFqn, "A/B/C/hello.txt", new FileUpdateParams().setSampleIds(Arrays.asList("sam1", "sam3")), INCLUDE_RESULT, token).first(); + File file = catalogManager.getFileManager().update(studyFqn, "A/B/C/hello.txt", new FileUpdateParams().setSampleIds(Arrays.asList("sam1", "sam3")), INCLUDE_RESULT, ownerToken).first(); fileMap.put(file.getUid(), file); - file = catalogManager.getFileManager().update(studyFqn, "A/B/C/hello2.txt", new FileUpdateParams().setSampleIds(Collections.singletonList("sam1")), INCLUDE_RESULT, token).first(); + file = catalogManager.getFileManager().update(studyFqn, "A/B/C/hello2.txt", new FileUpdateParams().setSampleIds(Collections.singletonList("sam1")), INCLUDE_RESULT, ownerToken).first(); fileMap.put(file.getUid(), file); - file = catalogManager.getFileManager().update(studyFqn, "A/B/C/D/hello.txt", new FileUpdateParams().setSampleIds(Arrays.asList("sam2", "sam3")), INCLUDE_RESULT, token).first(); + file = catalogManager.getFileManager().update(studyFqn, "A/B/C/D/hello.txt", new FileUpdateParams().setSampleIds(Arrays.asList("sam2", "sam3")), INCLUDE_RESULT, ownerToken).first(); fileMap.put(file.getUid(), file); - file = catalogManager.getFileManager().update(studyFqn, "A/B/C/D/hello2.txt", new FileUpdateParams().setSampleIds(Collections.singletonList("sam2")), INCLUDE_RESULT, token).first(); + file = catalogManager.getFileManager().update(studyFqn, "A/B/C/D/hello2.txt", new FileUpdateParams().setSampleIds(Collections.singletonList("sam2")), INCLUDE_RESULT, ownerToken).first(); fileMap.put(file.getUid(), file); - file = catalogManager.getFileManager().update(studyFqn, "A/B/C/D/hello3.txt", new FileUpdateParams().setSampleIds(Collections.singletonList("sam3")), INCLUDE_RESULT, token).first(); + file = catalogManager.getFileManager().update(studyFqn, "A/B/C/D/hello3.txt", new FileUpdateParams().setSampleIds(Collections.singletonList("sam3")), INCLUDE_RESULT, ownerToken).first(); + fileMap.put(file.getUid(), file); Map sampleVersionMap = new HashMap<>(); @@ -715,7 +717,7 @@ public void testMoveFiles() throws CatalogException { sampleVersionMap.put("sam2", 3); sampleVersionMap.put("sam3", 4); - OpenCGAResult sampleResults = catalogManager.getSampleManager().get(studyFqn, Arrays.asList("sam1", "sam2", "sam3"), QueryOptions.empty(), token); + OpenCGAResult sampleResults = catalogManager.getSampleManager().get(studyFqn, Arrays.asList("sam1", "sam2", "sam3"), QueryOptions.empty(), ownerToken); assertEquals(3, sampleResults.getNumResults()); for (Sample sample : sampleResults.getResults()) { assertEquals(sampleVersionMap.get(sample.getId()).intValue(), sample.getVersion()); @@ -729,11 +731,11 @@ public void testMoveFiles() throws CatalogException { } // Expected path after moving (not yet moved) - CatalogException catalogException = assertThrows(CatalogException.class, () -> catalogManager.getFileManager().get(studyFqn, Arrays.asList("A/C/D", "A/C/hello.txt", "A/C/hello2.txt", "A/C/D/hello.txt", "A/C/D/hello2.txt", "A/C/D/hello3.txt"), FileManager.INCLUDE_FILE_URI_PATH, token)); + CatalogException catalogException = assertThrows(CatalogException.class, () -> catalogManager.getFileManager().get(studyFqn, Arrays.asList("A/C/D", "A/C/hello.txt", "A/C/hello2.txt", "A/C/D/hello.txt", "A/C/D/hello2.txt", "A/C/D/hello3.txt"), FileManager.INCLUDE_FILE_URI_PATH, ownerToken)); assertTrue(catalogException.getMessage().contains("not found")); // Path before moving - OpenCGAResult beforeResult = catalogManager.getFileManager().get(studyFqn, Arrays.asList("A/B/C/D/", "A/B/C/hello.txt", "A/B/C/hello2.txt", "A/B/C/D/hello.txt", "A/B/C/D/hello2.txt", "A/B/C/D/hello3.txt"), QueryOptions.empty(), token); + OpenCGAResult beforeResult = catalogManager.getFileManager().get(studyFqn, Arrays.asList("A/B/C/D/", "A/B/C/hello.txt", "A/B/C/hello2.txt", "A/B/C/D/hello.txt", "A/B/C/D/hello2.txt", "A/B/C/D/hello3.txt"), QueryOptions.empty(), ownerToken); assertEquals(6, beforeResult.getNumResults()); for (File tmpFile : beforeResult.getResults()) { assertNotNull(tmpFile.getUri()); @@ -747,16 +749,16 @@ public void testMoveFiles() throws CatalogException { } // Move folder - catalogManager.getFileManager().move(studyFqn, "A/B/C/", "A/C/", QueryOptions.empty(), token); + catalogManager.getFileManager().move(studyFqn, "A/B/C/", "A/C/", QueryOptions.empty(), ownerToken); // Path before moving - catalogException = assertThrows(CatalogException.class, () -> catalogManager.getFileManager().get(studyFqn, Arrays.asList("A/B/C/D/", "A/B/C/hello.txt", "A/B/C/hello2.txt", "A/B/C/D/hello.txt", "A/B/C/D/hello2.txt", "A/B/C/D/hello3.txt"), FileManager.INCLUDE_FILE_URI_PATH, token)); + catalogException = assertThrows(CatalogException.class, () -> catalogManager.getFileManager().get(studyFqn, Arrays.asList("A/B/C/D/", "A/B/C/hello.txt", "A/B/C/hello2.txt", "A/B/C/D/hello.txt", "A/B/C/D/hello2.txt", "A/B/C/D/hello3.txt"), FileManager.INCLUDE_FILE_URI_PATH, ownerToken)); assertTrue(catalogException.getMessage().contains("not found")); Map afterMoveFileMap = new HashMap<>(); // Path after moving - OpenCGAResult afterResult = catalogManager.getFileManager().get(studyFqn, Arrays.asList("A/C/D/", "A/C/hello.txt", "A/C/hello2.txt", "A/C/D/hello.txt", "A/C/D/hello2.txt", "A/C/D/hello3.txt"), QueryOptions.empty(), token); + OpenCGAResult afterResult = catalogManager.getFileManager().get(studyFqn, Arrays.asList("A/C/D/", "A/C/hello.txt", "A/C/hello2.txt", "A/C/D/hello.txt", "A/C/D/hello2.txt", "A/C/D/hello3.txt"), QueryOptions.empty(), ownerToken); assertEquals(6, afterResult.getNumResults()); for (int i = 0; i < afterResult.getResults().size(); i++) { File beforeMoving = beforeResult.getResults().get(i); @@ -776,7 +778,7 @@ public void testMoveFiles() throws CatalogException { } // Check modifications in sample - sampleResults = catalogManager.getSampleManager().get(studyFqn, Arrays.asList("sam1", "sam2", "sam3"), QueryOptions.empty(), token); + sampleResults = catalogManager.getSampleManager().get(studyFqn, Arrays.asList("sam1", "sam2", "sam3"), QueryOptions.empty(), ownerToken); assertEquals(3, sampleResults.getNumResults()); for (Sample sample : sampleResults.getResults()) { assertEquals(sampleVersionMap.get(sample.getId()).intValue() + 1, sample.getVersion()); @@ -790,13 +792,14 @@ public void testMoveFiles() throws CatalogException { } // Attempt to move folder to folder in use - CatalogDBException catalogDBException = assertThrows(CatalogDBException.class, () -> catalogManager.getFileManager().move(studyFqn, "A/C/", "A/B/", QueryOptions.empty(), token)); + CatalogDBException catalogDBException = assertThrows(CatalogDBException.class, () -> catalogManager.getFileManager().move(studyFqn, "A/C/", "A/B/", QueryOptions.empty(), ownerToken)); assertTrue(catalogDBException.getMessage().contains("exists")); // Rename file - catalogManager.getFileManager().move(studyFqn, "A/C/D/hello3.txt", "A/C/D/otherName.txt", QueryOptions.empty(), token); - assertThrows(CatalogException.class, () -> catalogManager.getFileManager().get(studyFqn, "A/C/D/hello3.txt", QueryOptions.empty(), token)); - file = catalogManager.getFileManager().get(studyFqn, "A/C/D/otherName.txt", QueryOptions.empty(), token).first(); + catalogManager.getFileManager().move(studyFqn, "A/C/D/hello3.txt", "A/C/D/otherName.txt", QueryOptions.empty(), ownerToken); + assertThrows(CatalogException.class, () -> catalogManager.getFileManager().get(studyFqn, "A/C/D/hello3.txt", QueryOptions.empty(), ownerToken)); + file = catalogManager.getFileManager().get(studyFqn, "A/C/D/otherName.txt", QueryOptions.empty(), ownerToken).first(); + assertTrue(file.getId().endsWith(":otherName.txt")); assertTrue(file.getPath().endsWith("/otherName.txt")); assertTrue(file.getUri().toString().endsWith("/otherName.txt")); @@ -806,7 +809,7 @@ public void testMoveFiles() throws CatalogException { assertEquals(1, file.getSampleIds().size()); assertTrue(file.getSampleIds().contains("sam3")); - Sample sample3 = catalogManager.getSampleManager().get(studyFqn, "sam3", QueryOptions.empty(), token).first(); + Sample sample3 = catalogManager.getSampleManager().get(studyFqn, "sam3", QueryOptions.empty(), ownerToken).first(); assertEquals(6, sample3.getVersion()); assertEquals(3, sample3.getFileIds().size()); assertTrue(sample3.getFileIds().contains(file.getId())); @@ -815,75 +818,75 @@ public void testMoveFiles() throws CatalogException { @Test public void testAssociateSamples() throws CatalogException, URISyntaxException { URI uri = getClass().getResource("/biofiles/variant-test-file-dot-names.vcf.gz").toURI(); - DataResult link = fileManager.link(studyFqn, uri, ".", new ObjectMap(), token); + DataResult link = fileManager.link(studyFqn, uri, ".", new ObjectMap(), ownerToken); assertEquals(4, link.first().getSampleIds().size()); - assertThat(catalogManager.getSampleManager().get(studyFqn, "test-name.bam", QueryOptions.empty(), token).first().getFileIds(), + assertThat(catalogManager.getSampleManager().get(studyFqn, "test-name.bam", QueryOptions.empty(), ownerToken).first().getFileIds(), hasItem(link.first().getId())); - assertThat(catalogManager.getSampleManager().get(studyFqn, "NA19660", QueryOptions.empty(), token).first().getFileIds(), + assertThat(catalogManager.getSampleManager().get(studyFqn, "NA19660", QueryOptions.empty(), ownerToken).first().getFileIds(), hasItem(link.first().getId())); - assertThat(catalogManager.getSampleManager().get(studyFqn, "NA19661", QueryOptions.empty(), token).first().getFileIds(), + assertThat(catalogManager.getSampleManager().get(studyFqn, "NA19661", QueryOptions.empty(), ownerToken).first().getFileIds(), hasItem(link.first().getId())); - assertThat(catalogManager.getSampleManager().get(studyFqn, "NA19685", QueryOptions.empty(), token).first().getFileIds(), + assertThat(catalogManager.getSampleManager().get(studyFqn, "NA19685", QueryOptions.empty(), ownerToken).first().getFileIds(), hasItem(link.first().getId())); Map actionMap = new HashMap<>(); actionMap.put(FileDBAdaptor.QueryParams.SAMPLE_IDS.key(), ParamUtils.BasicUpdateAction.SET.name()); fileManager.update(studyFqn, link.first().getId(), new FileUpdateParams().setSampleIds(Collections.emptyList()), - new QueryOptions(Constants.ACTIONS, actionMap), token); - assertThat(catalogManager.getSampleManager().get(studyFqn, "test-name.bam", QueryOptions.empty(), token).first().getFileIds(), + new QueryOptions(Constants.ACTIONS, actionMap), ownerToken); + assertThat(catalogManager.getSampleManager().get(studyFqn, "test-name.bam", QueryOptions.empty(), ownerToken).first().getFileIds(), not(hasItem(link.first().getId()))); - assertThat(catalogManager.getSampleManager().get(studyFqn, "NA19660", QueryOptions.empty(), token).first().getFileIds(), + assertThat(catalogManager.getSampleManager().get(studyFqn, "NA19660", QueryOptions.empty(), ownerToken).first().getFileIds(), not(hasItem(link.first().getId()))); - assertThat(catalogManager.getSampleManager().get(studyFqn, "NA19661", QueryOptions.empty(), token).first().getFileIds(), + assertThat(catalogManager.getSampleManager().get(studyFqn, "NA19661", QueryOptions.empty(), ownerToken).first().getFileIds(), not(hasItem(link.first().getId()))); - assertThat(catalogManager.getSampleManager().get(studyFqn, "NA19685", QueryOptions.empty(), token).first().getFileIds(), + assertThat(catalogManager.getSampleManager().get(studyFqn, "NA19685", QueryOptions.empty(), ownerToken).first().getFileIds(), not(hasItem(link.first().getId()))); - File file = fileManager.get(studyFqn, link.first().getId(), QueryOptions.empty(), token).first(); + File file = fileManager.get(studyFqn, link.first().getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(0, file.getSampleIds().size()); actionMap.put(FileDBAdaptor.QueryParams.SAMPLE_IDS.key(), ParamUtils.BasicUpdateAction.ADD.name()); fileManager.update(studyFqn, link.first().getId(), new FileUpdateParams().setSampleIds(Arrays.asList("NA19660", "NA19661")), - new QueryOptions(Constants.ACTIONS, actionMap), token); + new QueryOptions(Constants.ACTIONS, actionMap), ownerToken); - file = fileManager.get(studyFqn, link.first().getId(), QueryOptions.empty(), token).first(); + file = fileManager.get(studyFqn, link.first().getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(2, file.getSampleIds().size()); - assertThat(catalogManager.getSampleManager().get(studyFqn, "NA19661", QueryOptions.empty(), token).first().getFileIds(), + assertThat(catalogManager.getSampleManager().get(studyFqn, "NA19661", QueryOptions.empty(), ownerToken).first().getFileIds(), hasItem(file.getId())); - assertThat(catalogManager.getSampleManager().get(studyFqn, "NA19660", QueryOptions.empty(), token).first().getFileIds(), + assertThat(catalogManager.getSampleManager().get(studyFqn, "NA19660", QueryOptions.empty(), ownerToken).first().getFileIds(), hasItem(file.getId())); actionMap.put(FileDBAdaptor.QueryParams.SAMPLE_IDS.key(), ParamUtils.BasicUpdateAction.REMOVE.name()); fileManager.update(studyFqn, link.first().getId(), new FileUpdateParams().setSampleIds(Arrays.asList("NA19661")), - new QueryOptions(Constants.ACTIONS, actionMap), token); + new QueryOptions(Constants.ACTIONS, actionMap), ownerToken); - file = fileManager.get(studyFqn, link.first().getId(), QueryOptions.empty(), token).first(); + file = fileManager.get(studyFqn, link.first().getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(1, file.getSampleIds().size()); assertEquals("NA19660", file.getSampleIds().get(0)); - assertThat(catalogManager.getSampleManager().get(studyFqn, "NA19660", QueryOptions.empty(), token).first().getFileIds(), + assertThat(catalogManager.getSampleManager().get(studyFqn, "NA19660", QueryOptions.empty(), ownerToken).first().getFileIds(), hasItem(file.getId())); - assertThat(catalogManager.getSampleManager().get(studyFqn, "NA19661", QueryOptions.empty(), token).first().getFileIds(), + assertThat(catalogManager.getSampleManager().get(studyFqn, "NA19661", QueryOptions.empty(), ownerToken).first().getFileIds(), not(hasItem(file.getId()))); actionMap.put(FileDBAdaptor.QueryParams.SAMPLE_IDS.key(), ParamUtils.BasicUpdateAction.SET); fileManager.update(studyFqn, link.first().getId(), new FileUpdateParams().setSampleIds(Arrays.asList("NA19661")), - new QueryOptions(Constants.ACTIONS, actionMap), token); - file = fileManager.get(studyFqn, link.first().getId(), QueryOptions.empty(), token).first(); + new QueryOptions(Constants.ACTIONS, actionMap), ownerToken); + file = fileManager.get(studyFqn, link.first().getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(1, file.getSampleIds().size()); assertEquals("NA19661", file.getSampleIds().get(0)); - assertThat(catalogManager.getSampleManager().get(studyFqn, "NA19660", QueryOptions.empty(), token).first().getFileIds(), + assertThat(catalogManager.getSampleManager().get(studyFqn, "NA19660", QueryOptions.empty(), ownerToken).first().getFileIds(), not(hasItem(file.getId()))); - assertThat(catalogManager.getSampleManager().get(studyFqn, "NA19661", QueryOptions.empty(), token).first().getFileIds(), + assertThat(catalogManager.getSampleManager().get(studyFqn, "NA19661", QueryOptions.empty(), ownerToken).first().getFileIds(), hasItem(file.getId())); file = fileManager.get(studyFqn, link.first().getId(), - new QueryOptions(QueryOptions.INCLUDE, FileDBAdaptor.QueryParams.SAMPLE_IDS.key()), token).first(); + new QueryOptions(QueryOptions.INCLUDE, FileDBAdaptor.QueryParams.SAMPLE_IDS.key()), ownerToken).first(); assertEquals(1, file.getSampleIds().size()); assertEquals("NA19661", file.getSampleIds().get(0)); assertNull(file.getCreationDate()); file = fileManager.get(studyFqn, link.first().getId(), - new QueryOptions(QueryOptions.EXCLUDE, FileDBAdaptor.QueryParams.SAMPLE_IDS.key()), token).first(); + new QueryOptions(QueryOptions.EXCLUDE, FileDBAdaptor.QueryParams.SAMPLE_IDS.key()), ownerToken).first(); assertTrue(file.getSampleIds().isEmpty()); assertNotNull(file.getCreationDate()); } @@ -901,12 +904,12 @@ public void testLinkFileWithDifferentSampleNames() throws CatalogException, URIS FileLinkParams params = new FileLinkParams() .setUri(uri.toString()) .setInternal(new FileLinkInternalParams(sampleIdNames)); - DataResult link = fileManager.link(studyFqn, params, false, token); + DataResult link = fileManager.link(studyFqn, params, false, ownerToken); assertEquals(4, link.first().getSampleIds().size()); Query query = new Query(SampleDBAdaptor.QueryParams.ID.key(), link.first().getSampleIds()); - DataResult sampleDataResult = catalogManager.getSampleManager().search(studyFqn, query, QueryOptions.empty(), token); + DataResult sampleDataResult = catalogManager.getSampleManager().search(studyFqn, query, QueryOptions.empty(), ownerToken); assertEquals(4, sampleDataResult.getNumResults()); List sampleNames = sampleDataResult.getResults().stream().map(Sample::getId).collect(Collectors.toList()); @@ -925,14 +928,14 @@ public void testLinkFileWithDifferentSampleNamesFromVCFHeader() throws CatalogEx FileLinkParams params = new FileLinkParams() .setUri(uri.toString()); - DataResult link = fileManager.link(studyFqn, params, false, token); + DataResult link = fileManager.link(studyFqn, params, false, ownerToken); assertEquals(3, link.first().getSampleIds().size()); assertEquals(Arrays.asList("sample_tumor", "sample_normal", "sample_other"), link.first().getSampleIds()); assertEquals(Arrays.asList("TUMOR", "NORMAL", "OTHER"), new ObjectMap(link.first().getAttributes()).getAsStringList("variantFileMetadata.attributes.originalSamples")); Query query = new Query(SampleDBAdaptor.QueryParams.ID.key(), link.first().getSampleIds()); - DataResult sampleDataResult = catalogManager.getSampleManager().search(studyFqn, query, QueryOptions.empty(), token); + DataResult sampleDataResult = catalogManager.getSampleManager().search(studyFqn, query, QueryOptions.empty(), ownerToken); assertEquals(3, sampleDataResult.getNumResults()); List sampleNames = sampleDataResult.getResults().stream().map(Sample::getId).collect(Collectors.toList()); @@ -943,9 +946,9 @@ public void testLinkFileWithDifferentSampleNamesFromVCFHeader() throws CatalogEx } @Test - public void testFileHooks() throws CatalogException, IOException, URISyntaxException { + public void testFileHooks() throws CatalogException, URISyntaxException { URI uri = getClass().getResource("/biofiles/variant-test-file-dot-names.vcf.gz").toURI(); - DataResult link = fileManager.link(studyFqn, uri, ".", new ObjectMap(), token); + DataResult link = fileManager.link(studyFqn, uri, ".", new ObjectMap(), ownerToken); assertEquals(2, link.first().getTags().size()); assertTrue(link.first().getTags().containsAll(Arrays.asList("VCF", "FILE"))); @@ -964,7 +967,7 @@ public void stressTestLinkFile() throws Exception { for (int i = 0; i < numOperations; i++) { executorService.submit(() -> { try { - fileManager.link(studyFqn, new FileLinkParams().setUri(uri.getPath()).setPath("."), false, token); + fileManager.link(studyFqn, new FileLinkParams().setUri(uri.getPath()).setPath("."), false, ownerToken); numOk.incrementAndGet(); } catch (Exception ignore) { ignore.printStackTrace(); @@ -987,7 +990,7 @@ public void stressTestLinkFile() throws Exception { @Test public void testUnlinkFolder() throws CatalogException, IOException { URI uri = createExternalDummyData().toUri(); - link(uri, "myDirectory", studyFqn, new ObjectMap("parents", true), token); + link(uri, "myDirectory", studyFqn, new ObjectMap("parents", true), ownerToken); IOManager ioManager = catalogManager.getIoManagerFactory().get(uri); @@ -995,7 +998,7 @@ public void testUnlinkFolder() throws CatalogException, IOException { .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(FileDBAdaptor.QueryParams.PATH.key(), "~myDirectory/*") .append(FileDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), FileStatus.READY); - DataResult fileDataResultLinked = fileManager.search(studyFqn, query, null, token); + DataResult fileDataResultLinked = fileManager.search(studyFqn, query, null, ownerToken); System.out.println("Number of files/folders linked = " + fileDataResultLinked.getNumResults()); @@ -1007,15 +1010,15 @@ public void testUnlinkFolder() throws CatalogException, IOException { setToPendingDelete(studyFqn, updateQuery); // Now we try to unlink them - fileManager.unlink(studyFqn, "myDirectory/A/", token); - fileDataResultLinked = fileManager.search(studyFqn, query, null, token); + fileManager.unlink(studyFqn, "myDirectory/A/", ownerToken); + fileDataResultLinked = fileManager.search(studyFqn, query, null, ownerToken); assertEquals(1, fileDataResultLinked.getNumResults()); query = new Query() .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(FileDBAdaptor.QueryParams.PATH.key(), "~myDirectory/*") .append(FileDBAdaptor.QueryParams.DELETED.key(), true); - DataResult fileDataResultUnlinked = fileManager.search(studyFqn, query, null, token); + DataResult fileDataResultUnlinked = fileManager.search(studyFqn, query, null, ownerToken); assertEquals(6, fileDataResultUnlinked.getNumResults()); for (File file : fileDataResultUnlinked.getResults()) { @@ -1028,12 +1031,12 @@ public void testUnlinkFolder() throws CatalogException, IOException { @Test public void testUnlinkFile() throws CatalogException, IOException { URI uri = Paths.get(getStudyURI()).resolve("data").toUri(); - link(uri, "myDirectory", studyFqn, new ObjectMap("parents", true), token); + link(uri, "myDirectory", studyFqn, new ObjectMap("parents", true), ownerToken); Query query = new Query() .append(FileDBAdaptor.QueryParams.PATH.key(), "~myDirectory/*") .append(FileDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), FileStatus.READY); - DataResult fileDataResultLinked = fileManager.search(studyFqn, query, null, token); + DataResult fileDataResultLinked = fileManager.search(studyFqn, query, null, ownerToken); int numberLinkedFiles = fileDataResultLinked.getNumResults(); System.out.println("Number of files/folders linked = " + numberLinkedFiles); @@ -1043,23 +1046,23 @@ public void testUnlinkFile() throws CatalogException, IOException { setToPendingDelete(studyFqn, unlinkQuery); // Now we try to unlink the file - fileManager.unlink(studyFqn, "myDirectory/data/test/folder/test_0.5K.txt", token); - fileDataResultLinked = fileManager.search(studyFqn, unlinkQuery, QueryOptions.empty(), token); + fileManager.unlink(studyFqn, "myDirectory/data/test/folder/test_0.5K.txt", ownerToken); + fileDataResultLinked = fileManager.search(studyFqn, unlinkQuery, QueryOptions.empty(), ownerToken); assertEquals(0, fileDataResultLinked.getNumResults()); unlinkQuery.put(FileDBAdaptor.QueryParams.DELETED.key(), true); - fileDataResultLinked = fileManager.search(studyFqn, unlinkQuery, QueryOptions.empty(), token); + fileDataResultLinked = fileManager.search(studyFqn, unlinkQuery, QueryOptions.empty(), ownerToken); assertEquals(1, fileDataResultLinked.getNumResults()); assertEquals(FileStatus.REMOVED, fileDataResultLinked.first().getInternal().getStatus().getId()); // Check the other root linked files/folders have not been touched - fileDataResultLinked = fileManager.search(studyFqn, query, QueryOptions.empty(), token); + fileDataResultLinked = fileManager.search(studyFqn, query, QueryOptions.empty(), ownerToken); assertEquals(numberLinkedFiles - 1, fileDataResultLinked.getNumResults()); // We send the unlink command again thrown.expect(CatalogException.class); thrown.expectMessage("not unlink"); - fileManager.unlink(studyFqn, "myDirectory/data/test/folder/test_0.5K.txt", token); + fileManager.unlink(studyFqn, "myDirectory/data/test/folder/test_0.5K.txt", ownerToken); } @Test @@ -1071,7 +1074,7 @@ public void testCreateFile() throws CatalogException, IOException { .setType(File.Type.FILE) .setPath("data/test/myTest/myFile.txt") .setContent("This is the content\tof the file"), - false, sessionIdUser2); + false, orgAdminToken1); fail("An error should be raised because parents is false"); } catch (CatalogException e) { System.out.println("Correct"); @@ -1082,7 +1085,7 @@ public void testCreateFile() throws CatalogException, IOException { .setType(File.Type.FILE) .setPath("data/test/myTest/myFile.txt") .setContent(content), - true, sessionIdUser2); + true, orgAdminToken1); IOManager ioManager = catalogManager.getIoManagerFactory().get(fileDataResult.first().getUri()); assertTrue(ioManager.exists(fileDataResult.first().getUri())); @@ -1092,68 +1095,59 @@ public void testCreateFile() throws CatalogException, IOException { @Test public void testCreateFolder() throws Exception { - Query query = new Query(StudyDBAdaptor.QueryParams.OWNER.key(), "user2"); - Study study = catalogManager.getStudyManager().search(query, QueryOptions.empty(), sessionIdUser2).first(); - Set paths = fileManager.search(study.getFqn(), new Query("type", File.Type.DIRECTORY), new - QueryOptions(), sessionIdUser2) + Set paths = fileManager.search(studyFqn, new Query("type", File.Type.DIRECTORY), new QueryOptions(), orgAdminToken1) .getResults().stream().map(File::getPath).collect(Collectors.toSet()); - assertEquals(2, paths.size()); - assertTrue(paths.contains("")); //root - assertTrue(paths.contains("JOBS/")); //JOBS -// assertTrue(paths.contains("analysis/")); //analysis + assertEquals(9, paths.size()); + assertTrue(paths.containsAll(Arrays.asList("", "JOBS/", "data/", "data/test/", "data/test/folder/", "data/d1/", "data/d1/d2/", + "data/d1/d2/d3/", "data/d1/d2/d3/d4/"))); Path folderPath = Paths.get("data", "new", "folder"); - File folder = fileManager.createFolder(study.getFqn(), folderPath.toString(), true, null, - QueryOptions.empty(), sessionIdUser2).first(); + File folder = fileManager.createFolder(studyFqn, folderPath.toString(), true, null, QueryOptions.empty(), orgAdminToken1).first(); System.out.println(folder); IOManager ioManager = catalogManager.getIoManagerFactory().get(folder.getUri()); assertTrue(!ioManager.exists(folder.getUri())); - paths = fileManager.search(study.getFqn(), new Query(FileDBAdaptor.QueryParams.TYPE.key(), File.Type - .DIRECTORY), new QueryOptions(), sessionIdUser2).getResults().stream().map(File::getPath).collect(Collectors.toSet()); - assertEquals(5, paths.size()); - assertTrue(paths.contains("data/new/")); - assertTrue(paths.contains("data/new/folder/")); + paths = fileManager.search(studyFqn, new Query(FileDBAdaptor.QueryParams.TYPE.key(), File.Type.DIRECTORY), new QueryOptions(), + orgAdminToken1).getResults().stream().map(File::getPath).collect(Collectors.toSet()); + assertEquals(11, paths.size()); + assertTrue(paths.containsAll(Arrays.asList("", "JOBS/", "data/", "data/test/", "data/test/folder/", "data/d1/", "data/d1/d2/", + "data/d1/d2/d3/", "data/d1/d2/d3/d4/", "data/new/", "data/new/folder/"))); - URI uri = fileManager.getUri(folder); + URI uri = fileManager.getUri(organizationId, folder); assertTrue(!catalogManager.getIoManagerFactory().get(uri).exists(uri)); - fileManager.createFolder(study.getFqn(), Paths.get("WOLOLO").toString(), true, null, QueryOptions.empty(), - sessionIdUser2); + fileManager.createFolder(studyFqn, Paths.get("WOLOLO").toString(), true, null, QueryOptions.empty(), orgAdminToken1); - Path myStudy = Files.createDirectory(catalogManagerResource.getOpencgaHome().resolve("myStudy")); - String newStudy = catalogManager.getStudyManager().create(project2, "alias", null, "name", "", null, null, null, null, null, sessionIdUser2).first().getFqn(); + String newStudy = catalogManager.getStudyManager().create(project2, "alias", null, "name", "", null, null, null, null, null, orgAdminToken1).first().getFqn(); folder = fileManager.createFolder(newStudy, Paths.get("WOLOLO").toString(), true, null, - QueryOptions.empty(), sessionIdUser2).first(); + QueryOptions.empty(), orgAdminToken1).first(); assertTrue(!ioManager.exists(folder.getUri())); } @Test public void testCreateFolderAlreadyExists() throws Exception { - Set paths = fileManager.search(studyFqn3, new Query("type", File.Type.DIRECTORY), new QueryOptions(), sessionIdUser2).getResults().stream().map(File::getPath).collect(Collectors.toSet()); + Set paths = fileManager.search(studyFqn3, new Query("type", File.Type.DIRECTORY), new QueryOptions(), orgAdminToken1).getResults().stream().map(File::getPath).collect(Collectors.toSet()); assertEquals(2, paths.size()); assertTrue(paths.contains("")); //root // assertTrue(paths.contains("data/")); //data // assertTrue(paths.contains("analysis/")); //analysis Path folderPath = Paths.get("data", "new", "folder"); - File folder = fileManager.createFolder(studyFqn3, folderPath.toString(), true, null, null, - sessionIdUser2).first(); + File folder = fileManager.createFolder(studyFqn3, folderPath.toString(), true, null, null, orgAdminToken1).first(); assertNotNull(folder); assertTrue(folder.getPath().contains(folderPath.toString())); // When creating the same folder, we should not complain and return it directly - File sameFolder = fileManager.createFolder(studyFqn3, folderPath.toString(), true, null, null, sessionIdUser2).first(); + File sameFolder = fileManager.createFolder(studyFqn3, folderPath.toString(), true, null, null, orgAdminToken1).first(); assertNotNull(sameFolder); assertEquals(folder.getPath(), sameFolder.getPath()); assertEquals(folder.getUid(), sameFolder.getUid()); // However, a user without create permissions will receive an exception thrown.expect(CatalogAuthorizationException.class); - fileManager.createFolder(studyFqn3, folderPath.toString(), true, null, null, - sessionIdUser3); + fileManager.createFolder(studyFqn3, folderPath.toString(), true, null, null, normalToken1); } @Test @@ -1166,7 +1160,7 @@ public void testAnnotationWrongEntity() throws CatalogException, JsonProcessingE variables.add(new Variable("HEIGHT", "", "", Variable.VariableType.DOUBLE, "", false, false, Collections.emptyList(), null, 0, "", "", null, Collections.emptyMap())); VariableSet vs1 = catalogManager.getStudyManager().createVariableSet(studyFqn, "vs1", "vs1", false, false, "", null, variables, - Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), token).first(); + Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), ownerToken).first(); ObjectMap annotations = new ObjectMap() .append("var_name", "Joe") @@ -1178,7 +1172,7 @@ public void testAnnotationWrongEntity() throws CatalogException, JsonProcessingE thrown.expect(CatalogException.class); thrown.expectMessage("intended only for"); - fileManager.update(studyFqn, "data/", updateParams, QueryOptions.empty(), token); + fileManager.update(studyFqn, "data/", updateParams, QueryOptions.empty(), ownerToken); } @Test @@ -1191,7 +1185,7 @@ public void testAnnotationForAnyEntity() throws CatalogException, JsonProcessing variables.add(new Variable("HEIGHT", "", "", Variable.VariableType.DOUBLE, "", false, false, Collections.emptyList(), null, 0, "", "", null, Collections.emptyMap())); VariableSet vs1 = catalogManager.getStudyManager().createVariableSet(studyFqn, "vs1", "vs1", false, false, "", null, variables, - null, token).first(); + null, ownerToken).first(); ObjectMap annotations = new ObjectMap() .append("var_name", "Joe") @@ -1201,10 +1195,10 @@ public void testAnnotationForAnyEntity() throws CatalogException, JsonProcessing FileUpdateParams updateParams = new FileUpdateParams().setAnnotationSets(Collections.singletonList(annotationSet)); - DataResult updateResult = fileManager.update(studyFqn, "data/", updateParams, QueryOptions.empty(), token); + DataResult updateResult = fileManager.update(studyFqn, "data/", updateParams, QueryOptions.empty(), ownerToken); assertEquals(1, updateResult.getNumUpdated()); - File file = fileManager.get(studyFqn, "data/", QueryOptions.empty(), token).first(); + File file = fileManager.get(studyFqn, "data/", QueryOptions.empty(), ownerToken).first(); assertEquals(1, file.getAnnotationSets().size()); } @@ -1218,7 +1212,7 @@ public void testAnnotations() throws CatalogException, JsonProcessingException { variables.add(new Variable("HEIGHT", "", "", Variable.VariableType.DOUBLE, "", false, false, Collections.emptyList(), null, 0, "", "", null, Collections.emptyMap())); VariableSet vs1 = catalogManager.getStudyManager().createVariableSet(studyFqn, "vs1", "vs1", false, false, "", null, variables, - Collections.singletonList(VariableSet.AnnotableDataModels.FILE), token).first(); + Collections.singletonList(VariableSet.AnnotableDataModels.FILE), ownerToken).first(); ObjectMap annotations = new ObjectMap() .append("var_name", "Joe") @@ -1228,38 +1222,38 @@ public void testAnnotations() throws CatalogException, JsonProcessingException { AnnotationSet annotationSet1 = new AnnotationSet("annotation2", vs1.getId(), annotations); FileUpdateParams updateParams = new FileUpdateParams().setAnnotationSets(Arrays.asList(annotationSet, annotationSet1)); - DataResult updateResult = fileManager.update(studyFqn, "data/", updateParams, QueryOptions.empty(), token); + DataResult updateResult = fileManager.update(studyFqn, "data/", updateParams, QueryOptions.empty(), ownerToken); assertEquals(1, updateResult.getNumUpdated()); - File file = fileManager.get(studyFqn, "data/", QueryOptions.empty(), token).first(); + File file = fileManager.get(studyFqn, "data/", QueryOptions.empty(), ownerToken).first(); assertEquals(2, file.getAnnotationSets().size()); } @Test public void testUpdateSamples() throws CatalogException { // Update the same sample twice to the file - Sample sample1 = catalogManager.getSampleManager().get(studyFqn, "s_1", QueryOptions.empty(), token).first(); - Sample sample2 = catalogManager.getSampleManager().get(studyFqn, "s_2", QueryOptions.empty(), token).first(); + Sample sample1 = catalogManager.getSampleManager().get(studyFqn, "s_1", QueryOptions.empty(), ownerToken).first(); + Sample sample2 = catalogManager.getSampleManager().get(studyFqn, "s_2", QueryOptions.empty(), ownerToken).first(); assertFalse(sample1.getFileIds().contains("data:test:folder:test_1K.txt.gz")); assertFalse(sample2.getFileIds().contains("data:test:folder:test_1K.txt.gz")); FileUpdateParams updateParams = new FileUpdateParams().setSampleIds(Arrays.asList("s_1", "s_1", "s_2", "s_1")); - DataResult updateResult = fileManager.update(studyFqn, "test_1K.txt.gz", updateParams, null, token); + DataResult updateResult = fileManager.update(studyFqn, "test_1K.txt.gz", updateParams, null, ownerToken); assertEquals(1, updateResult.getNumUpdated()); - File file = fileManager.get(studyFqn, "test_1K.txt.gz", QueryOptions.empty(), token).first(); + File file = fileManager.get(studyFqn, "test_1K.txt.gz", QueryOptions.empty(), ownerToken).first(); assertEquals(2, file.getSampleIds().size()); assertTrue(file.getSampleIds().containsAll(Arrays.asList("s_1", "s_2"))); - OpenCGAResult sampleResult = catalogManager.getSampleManager().get(studyFqn, Arrays.asList("s_1", "s_2"), QueryOptions.empty(), token); + OpenCGAResult sampleResult = catalogManager.getSampleManager().get(studyFqn, Arrays.asList("s_1", "s_2"), QueryOptions.empty(), ownerToken); assertEquals(2, sampleResult.getNumResults()); for (Sample sample : sampleResult.getResults()) { assertTrue(sample.getFileIds().contains(file.getId())); } System.out.println(file.getId()); - sample1 = catalogManager.getSampleManager().get(studyFqn, "s_1", QueryOptions.empty(), token).first(); - sample2 = catalogManager.getSampleManager().get(studyFqn, "s_2", QueryOptions.empty(), token).first(); + sample1 = catalogManager.getSampleManager().get(studyFqn, "s_1", QueryOptions.empty(), ownerToken).first(); + sample2 = catalogManager.getSampleManager().get(studyFqn, "s_2", QueryOptions.empty(), ownerToken).first(); assertTrue(sample1.getFileIds().contains(file.getId())); assertTrue(sample2.getFileIds().contains(file.getId())); } @@ -1274,8 +1268,8 @@ public void testCreate() throws Exception { .setBioformat(File.Bioformat.VARIANT) .setPath("data/" + fileName) .setDescription("description") - .setContent(getDummyVCFContent()), - true, token); + .setContent(MongoBackupUtils.getDummyVCFContent()), + true, ownerToken); assertEquals(3, fileResult.first().getSampleIds().size()); fileName = "item." + TimeUtils.getTimeMillis() + ".vcf"; @@ -1286,8 +1280,8 @@ public void testCreate() throws Exception { .setBioformat(File.Bioformat.VARIANT) .setPath("data/" + fileName) .setDescription("description") - .setContent(getDummyVCFContent()), - true, token); + .setContent(MongoBackupUtils.getDummyVCFContent()), + true, ownerToken); fileName = "item." + TimeUtils.getTimeMillis() + ".txt"; DataResult queryResult = fileManager.create(studyFqn, @@ -1295,7 +1289,7 @@ public void testCreate() throws Exception { .setContent(RandomStringUtils.randomAlphanumeric(200)) .setType(File.Type.FILE) .setPath("data/" + fileName), - false, token); + false, ownerToken); assertEquals(FileStatus.READY, queryResult.first().getInternal().getStatus().getId()); assertEquals(200, queryResult.first().getSize()); @@ -1306,8 +1300,8 @@ public void testCreate() throws Exception { .setBioformat(File.Bioformat.NONE) .setPath("data/deletable/folder/item." + TimeUtils.getTimeMillis() + ".txt") .setDescription("description") - .setContent(createRandomString(200)), - true, token); + .setContent(MongoBackupUtils.createRandomString(200)), + true, ownerToken); fileManager.create(studyFqn2, new FileCreateParams() @@ -1316,8 +1310,8 @@ public void testCreate() throws Exception { .setBioformat(File.Bioformat.NONE) .setPath("data/deletable/item." + TimeUtils.getTimeMillis() + ".txt") .setDescription("description") - .setContent(createRandomString(200)), - true, token); + .setContent(MongoBackupUtils.createRandomString(200)), + true, ownerToken); fileManager.create(studyFqn2, new FileCreateParams() @@ -1326,8 +1320,8 @@ public void testCreate() throws Exception { .setBioformat(File.Bioformat.NONE) .setPath("item." + TimeUtils.getTimeMillis() + ".txt") .setDescription("file at root") - .setContent(createRandomString(200)), - true, token); + .setContent(MongoBackupUtils.createRandomString(200)), + true, ownerToken); fileName = "item." + TimeUtils.getTimeMillis() + ".txt"; fileManager.create(studyFqn2, @@ -1337,10 +1331,10 @@ public void testCreate() throws Exception { .setBioformat(File.Bioformat.NONE) .setPath(fileName) .setDescription("file at root") - .setContent(createRandomString(200)), - true, token); + .setContent(MongoBackupUtils.createRandomString(200)), + true, ownerToken); - DataResult fileDataResult = fileManager.get(studyFqn2, fileName, null, token); + DataResult fileDataResult = fileManager.get(studyFqn2, fileName, null, ownerToken); assertTrue(fileDataResult.first().getSize() > 0); } @@ -1352,7 +1346,7 @@ public void testCreateFileInLinkedFolder() throws Exception { URI uri = dir.toUri(); // Link the folder in the root - link(uri, "", studyFqn, new ObjectMap(), token); + link(uri, "", studyFqn, new ObjectMap(), ownerToken); File file = fileManager.create(studyFqn, new FileCreateParams() @@ -1360,7 +1354,7 @@ public void testCreateFileInLinkedFolder() throws Exception { .setType(File.Type.FILE) .setFormat(File.Format.PLAIN) .setPath("folder_to_link/file.txt"), - false, token).first(); + false, ownerToken).first(); assertEquals(uri.resolve("file.txt"), file.getUri()); } @@ -1376,13 +1370,13 @@ public void testDownloadAndHeadFile() throws CatalogException, IOException, Inte .setBioformat(File.Bioformat.VARIANT) .setPath("data/" + fileName) .setDescription("description") - .setContent(getDummyVCFContent()), - true, token).first(); + .setContent(MongoBackupUtils.getDummyVCFContent()), + true, ownerToken).first(); byte[] bytes = new byte[100]; byte[] bytesOrig = new byte[100]; DataInputStream fis = new DataInputStream(new FileInputStream(file.getUri().getPath())); - DataInputStream dis = fileManager.download(studyFqn, file.getPath(), -1, -1, token); + DataInputStream dis = fileManager.download(studyFqn, file.getPath(), -1, -1, ownerToken); fis.read(bytesOrig, 0, 100); dis.read(bytes, 0, 100); fis.close(); @@ -1391,7 +1385,7 @@ public void testDownloadAndHeadFile() throws CatalogException, IOException, Inte int offset = 1; int limit = 10; - dis = fileManager.download(studyFqn, file.getPath(), offset, limit, token); + dis = fileManager.download(studyFqn, file.getPath(), offset, limit, ownerToken); fis = new DataInputStream(new FileInputStream(file.getUri().getPath())); for (int i = 0; i < offset; i++) { fis.readLine(); @@ -1415,9 +1409,9 @@ public void testDownloadAndHeadFile() throws CatalogException, IOException, Inte @Test public void testDownloadFile() throws CatalogException, IOException, URISyntaxException { URI sourceUri = getClass().getResource("/biofiles/variant-test-file.vcf.gz").toURI(); - OpenCGAResult fileResult = fileManager.link(studyFqn, sourceUri, "data/", new ObjectMap("parents", true), token); + OpenCGAResult fileResult = fileManager.link(studyFqn, sourceUri, "data/", new ObjectMap("parents", true), ownerToken); - DataInputStream dis = fileManager.download(studyFqn, fileResult.first().getPath(), -1, -1, token); + DataInputStream dis = fileManager.download(studyFqn, fileResult.first().getPath(), -1, -1, ownerToken); byte[] bytes = new byte[(int) fileResult.first().getSize()]; dis.read(bytes, 0, (int) fileResult.first().getSize()); @@ -1437,64 +1431,64 @@ public void testGetTreeView() throws CatalogException { fileManager.create(studyFqn, new FileCreateParams() .setPath("myFile_a.txt") .setType(File.Type.FILE) - .setContent("content"), false, token); + .setContent("content"), false, ownerToken); fileManager.create(studyFqn, new FileCreateParams() .setPath("myFile_b.txt") .setType(File.Type.FILE) - .setContent("content"), false, token); + .setContent("content"), false, ownerToken); fileManager.create(studyFqn, new FileCreateParams() .setPath("myFile_c.txt") .setType(File.Type.FILE) - .setContent("content"), false, token); + .setContent("content"), false, ownerToken); fileManager.create(studyFqn, new FileCreateParams() .setPath("data/myFile_a.txt") .setType(File.Type.FILE) - .setContent("content"), false, token); + .setContent("content"), false, ownerToken); fileManager.create(studyFqn, new FileCreateParams() .setPath("data/myFile_b.txt") .setType(File.Type.FILE) - .setContent("content"), false, token); + .setContent("content"), false, ownerToken); fileManager.create(studyFqn, new FileCreateParams() .setPath("data/myFile_c.txt") .setType(File.Type.FILE) - .setContent("content"), false, token); + .setContent("content"), false, ownerToken); fileManager.create(studyFqn, new FileCreateParams() .setPath("JOBS/myFile_a.txt") .setType(File.Type.FILE) - .setContent("content"), false, token); + .setContent("content"), false, ownerToken); fileManager.create(studyFqn, new FileCreateParams() .setPath("JOBS/myFile_b.txt") .setType(File.Type.FILE) - .setContent("content"), false, token); + .setContent("content"), false, ownerToken); fileManager.create(studyFqn, new FileCreateParams() .setPath("JOBS/myFile_c.txt") .setType(File.Type.FILE) - .setContent("content"), false, token); + .setContent("content"), false, ownerToken); fileManager.create(studyFqn, new FileCreateParams() .setPath("JOBS/AAAAAA/myFile_a.txt") .setType(File.Type.FILE) - .setContent("content"), true, token); + .setContent("content"), true, ownerToken); fileManager.create(studyFqn, new FileCreateParams() .setPath("JOBS/BBBBBB/myFile_b.txt") .setType(File.Type.FILE) - .setContent("content"), true, token); + .setContent("content"), true, ownerToken); fileManager.create(studyFqn, new FileCreateParams() .setPath("JOBS/CCCCCC/myFile_c.txt") .setType(File.Type.FILE) - .setContent("content"), true, token); + .setContent("content"), true, ownerToken); - DataResult fileTree = fileManager.getTree(studyFqn, "/", 5, new QueryOptions(QueryOptions.INCLUDE, FileDBAdaptor.QueryParams.ID.key()), token); - assertEquals(23, fileTree.getNumResults()); - assertEquals(23, countElementsInTree(fileTree.first())); + DataResult fileTree = fileManager.getTree(studyFqn, "/", 5, new QueryOptions(QueryOptions.INCLUDE, FileDBAdaptor.QueryParams.ID.key()), ownerToken); + assertEquals(27, fileTree.getNumResults()); + assertEquals(27, countElementsInTree(fileTree.first())); - fileTree = fileManager.getTree(studyFqn, "/", 2, new QueryOptions(), token); - assertEquals(16, fileTree.getNumResults()); + fileTree = fileManager.getTree(studyFqn, "/", 2, new QueryOptions(), ownerToken); + assertEquals(17, fileTree.getNumResults()); QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, FileDBAdaptor.QueryParams.ID.key()); - fileTree = fileManager.getTree(studyFqn, "/", 2, options, token); + fileTree = fileManager.getTree(studyFqn, "/", 2, options, ownerToken); assertNotNull(fileTree.first().getFile().getId()); assertNull(fileTree.first().getFile().getName()); @@ -1509,20 +1503,19 @@ public void testGetTreeViewMoreThanOneFile() throws CatalogException { // Create a new study so more than one file will be found under the root /. However, it should be able to consider the study given // properly - catalogManager.getStudyManager().create(project1, "phase2", null, "Phase 2", "Done", null, null, null, null, null, token).first().getUid(); + catalogManager.getStudyManager().create(project1, "phase2", null, "Phase 2", "Done", null, null, null, null, null, ownerToken); - DataResult fileTree = fileManager.getTree(studyFqn, "/", 5, new QueryOptions(), - token); - assertEquals(8, fileTree.getNumResults()); + DataResult fileTree = fileManager.getTree(studyFqn, "/", 5, new QueryOptions(), ownerToken); + assertEquals(12, fileTree.getNumResults()); - fileTree = fileManager.getTree("user@1000G:phase2", ".", 5, new QueryOptions(), token); + fileTree = fileManager.getTree("phase2", ".", 5, new QueryOptions(), ownerToken); assertEquals(2, fileTree.getNumResults()); } @Test public void getFileIdByString() throws CatalogException { StudyAclParams aclParams = new StudyAclParams("", "analyst"); - catalogManager.getStudyManager().updateAcl(Arrays.asList(studyFqn), "user2", aclParams, ParamUtils.AclAction.ADD, token); + catalogManager.getStudyManager().updateAcl(studyFqn, normalUserId2, aclParams, ParamUtils.AclAction.ADD, ownerToken); File file = fileManager.create(studyFqn, new FileCreateParams() .setType(File.Type.FILE) @@ -1531,14 +1524,14 @@ public void getFileIdByString() throws CatalogException { .setPath("data/test/folder/file.txt") .setDescription("My description") .setContent("blabla"), - true, sessionIdUser2).first(); - long fileId = fileManager.get(studyFqn, file.getPath(), FileManager.INCLUDE_FILE_IDS, token).first().getUid(); + true, normalToken2).first(); + long fileId = fileManager.get(studyFqn, file.getPath(), FileManager.INCLUDE_FILE_IDS, ownerToken).first().getUid(); assertEquals(file.getUid(), fileId); - fileId = fileManager.get(studyFqn, file.getPath(), FileManager.INCLUDE_FILE_IDS, token).first().getUid(); + fileId = fileManager.get(studyFqn, file.getPath(), FileManager.INCLUDE_FILE_IDS, ownerToken).first().getUid(); assertEquals(file.getUid(), fileId); - fileId = fileManager.get(studyFqn, "/", FileManager.INCLUDE_FILE_IDS, token).first().getUid(); + fileId = fileManager.get(studyFqn, "/", FileManager.INCLUDE_FILE_IDS, ownerToken).first().getUid(); System.out.println(fileId); } @@ -1549,40 +1542,40 @@ public void searchFileTest() throws CatalogException { // Look for a file and folder DataResult queryResults = fileManager.get(studyFqn, Arrays.asList("data/", "data/test/folder/test_1K.txt.gz"), - new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(FileDBAdaptor.QueryParams.NAME.key())), token); + new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(FileDBAdaptor.QueryParams.NAME.key())), ownerToken); assertEquals(2, queryResults.getNumResults()); assertTrue("Name not included", queryResults.getResults().stream().map(File::getName) .filter(org.apache.commons.lang3.StringUtils::isNotEmpty) .collect(Collectors.toList()).size() == 2); query = new Query(FileDBAdaptor.QueryParams.NAME.key(), "~data"); - result = fileManager.search(studyFqn, query, null, token); + result = fileManager.search(studyFqn, query, null, ownerToken); assertEquals(1, result.getNumResults()); query = new Query(FileDBAdaptor.QueryParams.NAME.key(), "~txt.gz$"); - result = fileManager.search(studyFqn, query, null, token); + result = fileManager.search(studyFqn, query, null, ownerToken); assertEquals(1, result.getNumResults()); //Get all files in data query = new Query(FileDBAdaptor.QueryParams.PATH.key(), "~data/[^/]+/?") .append(FileDBAdaptor.QueryParams.TYPE.key(), "FILE"); - result = fileManager.search(studyFqn, query, null, token); - assertEquals(3, result.getNumResults()); + result = fileManager.search(studyFqn, query, null, ownerToken); + assertEquals(4, result.getNumResults()); //Folder "jobs" does not exist query = new Query(FileDBAdaptor.QueryParams.DIRECTORY.key(), "jobs"); - result = fileManager.search(studyFqn, query, null, token); + result = fileManager.search(studyFqn, query, null, ownerToken); assertEquals(0, result.getNumResults()); //Get all files in data query = new Query(FileDBAdaptor.QueryParams.DIRECTORY.key(), "data/"); - result = fileManager.search(studyFqn, query, null, token); - assertEquals(1, result.getNumResults()); + result = fileManager.search(studyFqn, query, null, ownerToken); + assertEquals(2, result.getNumResults()); //Get all files in data query = new Query(FileDBAdaptor.QueryParams.DIRECTORY.key(), "/data/"); - result = fileManager.search(studyFqn, query, null, token); - assertEquals(1, result.getNumResults()); + result = fileManager.search(studyFqn, query, null, ownerToken); + assertEquals(2, result.getNumResults()); //Get all files in data recursively // query = new Query(FileDBAdaptor.QueryParams.DIRECTORY.key(), "~data/.*"); @@ -1590,183 +1583,54 @@ public void searchFileTest() throws CatalogException { // assertEquals(5, result.getNumResults()); query = new Query(FileDBAdaptor.QueryParams.TYPE.key(), "FILE"); - result = fileManager.search(studyFqn, query, null, token); + result = fileManager.search(studyFqn, query, null, ownerToken); result.getResults().forEach(f -> assertEquals(File.Type.FILE, f.getType())); int numFiles = result.getNumResults(); - assertEquals(3, numFiles); + assertEquals(4, numFiles); query = new Query(FileDBAdaptor.QueryParams.TYPE.key(), "DIRECTORY"); - result = fileManager.search(studyFqn, query, null, token); + result = fileManager.search(studyFqn, query, null, ownerToken); result.getResults().forEach(f -> assertEquals(File.Type.DIRECTORY, f.getType())); int numFolders = result.getNumResults(); - assertEquals(5, numFolders); + assertEquals(9, numFolders); query = new Query(FileDBAdaptor.QueryParams.PATH.key(), ""); - result = fileManager.search(studyFqn, query, null, token); + result = fileManager.search(studyFqn, query, null, ownerToken); assertEquals(1, result.getNumResults()); assertEquals(".", result.first().getName()); query = new Query(FileDBAdaptor.QueryParams.TYPE.key(), "FILE,DIRECTORY"); - result = fileManager.search(studyFqn, query, null, token); - assertEquals(8, result.getNumResults()); + result = fileManager.search(studyFqn, query, null, ownerToken); + assertEquals(13, result.getNumResults()); assertEquals(numFiles + numFolders, result.getNumResults()); query = new Query("type", "FILE"); query.put("size", ">500"); - result = fileManager.search(studyFqn, query, null, token); + result = fileManager.search(studyFqn, query, null, ownerToken); assertEquals(2, result.getNumResults()); query = new Query("type", "FILE"); query.put("size", "<=500"); - result = fileManager.search(studyFqn, query, null, token); - assertEquals(1, result.getNumResults()); + result = fileManager.search(studyFqn, query, null, ownerToken); + assertEquals(2, result.getNumResults()); - List sampleIds = catalogManager.getSampleManager().search(studyFqn, new Query(SampleDBAdaptor.QueryParams.ID.key(), "s_1,s_3,s_4"), null, token).getResults() + List sampleIds = catalogManager.getSampleManager().search(studyFqn, new Query(SampleDBAdaptor.QueryParams.ID.key(), "s_1,s_3,s_4"), null, ownerToken).getResults() .stream() .map(Sample::getId) .collect(Collectors.toList()); - result = fileManager.search(studyFqn, new Query(FileDBAdaptor.QueryParams.SAMPLE_IDS.key(), sampleIds), null, token); + result = fileManager.search(studyFqn, new Query(FileDBAdaptor.QueryParams.SAMPLE_IDS.key(), sampleIds), null, ownerToken); assertEquals(1, result.getNumResults()); query = new Query(FileDBAdaptor.QueryParams.TYPE.key(), "FILE"); query.put(FileDBAdaptor.QueryParams.FORMAT.key(), "PLAIN"); - result = fileManager.search(studyFqn, query, null, token); - assertEquals(2, result.getNumResults()); - -// String attributes = FileDBAdaptor.QueryParams.ATTRIBUTES.key(); -// String nattributes = FileDBAdaptor.QueryParams.NATTRIBUTES.key(); -// String battributes = FileDBAdaptor.QueryParams.BATTRIBUTES.key(); - /* - - interface Searcher { - DataResult search(Integer id, Query query); - } - - BiFunction searcher = (s, q) -> catalogManager.searchFile(s, q, sessionIdUser); - - result = searcher.apply(studyUid, new Query(attributes + ".nested.text", "~H")); - */ -// result = fileManager.search(studyFqn, new Query(attributes + ".nested.text", "~H"), null, token); -// assertEquals(1, result.getNumResults()); -// result = fileManager.search(studyFqn, new Query(nattributes + ".nested.num1", ">0"), null, token); -// assertEquals(1, result.getNumResults()); -// result = fileManager.search(studyFqn, new Query(attributes + ".nested.num1", ">0"), null, token); -// assertEquals(0, result.getNumResults()); -// -// result = fileManager.search(studyFqn, new Query(attributes + ".nested.num1", "notANumber"), null, token); -// assertEquals(0, result.getNumResults()); -// -// result = fileManager.search(studyFqn, new Query(attributes + ".field", "~val"), null, token); -// assertEquals(3, result.getNumResults()); -// -// result = fileManager.search(studyFqn, new Query("attributes.field", "~val"), null, token); -// assertEquals(3, result.getNumResults()); -// -// result = fileManager.search(studyFqn, new Query(attributes + ".field", "=~val"), null, token); -// assertEquals(3, result.getNumResults()); -// -// result = fileManager.search(studyFqn, new Query(attributes + ".field", "~val"), null, token); -// assertEquals(3, result.getNumResults()); -// -// result = fileManager.search(studyFqn, new Query(attributes + ".field", "value"), null, token); -// assertEquals(2, result.getNumResults()); -// -// result = fileManager.search(studyFqn, new Query(attributes + ".field", "other"), null, token); -// assertEquals(1, result.getNumResults()); -// -// result = fileManager.search(studyFqn, new Query("nattributes.numValue", ">=5"), null, token); -// assertEquals(3, result.getNumResults()); -// -// result = fileManager.search(studyFqn, new Query("nattributes.numValue", ">4,<6"), null, token); -// assertEquals(3, result.getNumResults()); -// -// result = fileManager.search(studyFqn, new Query(nattributes + ".numValue", "==5"), null, token); -// assertEquals(2, result.getNumResults()); -// -// result = fileManager.search(studyFqn, new Query(nattributes + ".numValue", "==5.0"), null, token); -// assertEquals(2, result.getNumResults()); -// -// result = fileManager.search(studyFqn, new Query(nattributes + ".numValue", "=5.0"), null, token); -// assertEquals(2, result.getNumResults()); -// -// result = fileManager.search(studyFqn, new Query(nattributes + ".numValue", "5.0"), null, token); -// assertEquals(2, result.getNumResults()); -// -// result = fileManager.search(studyFqn, new Query(nattributes + ".numValue", ">5"), null, token); -// assertEquals(1, result.getNumResults()); -// -// result = fileManager.search(studyFqn, new Query(nattributes + ".numValue", ">4"), null, token); -// assertEquals(3, result.getNumResults()); -// -// result = fileManager.search(studyFqn, new Query(nattributes + ".numValue", "<6"), null, token); -// assertEquals(2, result.getNumResults()); -// -// result = fileManager.search(studyFqn, new Query(nattributes + ".numValue", "<=5"), null, token); -// assertEquals(2, result.getNumResults()); -// -// result = fileManager.search(studyFqn, new Query(nattributes + ".numValue", "<5"), null, token); -// assertEquals(0, result.getNumResults()); -// -// result = fileManager.search(studyFqn, new Query(nattributes + ".numValue", "<2"), null, token); -// assertEquals(0, result.getNumResults()); -// -// result = fileManager.search(studyFqn, new Query(nattributes + ".numValue", "==23"), null, token); -// assertEquals(0, result.getNumResults()); -// -// result = fileManager.search(studyFqn, new Query(attributes + ".numValue", "=~10"), null, token); -// assertEquals(1, result.getNumResults()); -// -// result = fileManager.search(studyFqn, new Query(nattributes + ".numValue", "=10"), null, token); -// assertEquals(0, result.getNumResults()); -// -// result = fileManager.search(studyFqn, new Query(attributes + ".boolean", "true"), null, token); -// assertEquals(0, result.getNumResults()); -// -// result = fileManager.search(studyFqn, new Query(attributes + ".boolean", "=true"), null, token); -// assertEquals(0, result.getNumResults()); -// -// result = fileManager.search(studyFqn, new Query(attributes + ".boolean", "=1"), null, token); -// assertEquals(0, result.getNumResults()); -// -// result = fileManager.search(studyFqn, new Query(battributes + ".boolean", "true"), null, token); -// assertEquals(1, result.getNumResults()); -// -// result = fileManager.search(studyFqn, new Query(battributes + ".boolean", "=true"), null, token); -// assertEquals(1, result.getNumResults()); -// -// // This has to return not only the ones with the attribute boolean = false, but also all the files that does not contain -// // that attribute at all. -// result = fileManager.search(studyFqn, new Query(battributes + ".boolean", "!=true"), null, token); -// assertEquals(7, result.getNumResults()); -// -// result = fileManager.search(studyFqn, new Query(battributes + ".boolean", "=false"), null, token); -// assertEquals(1, result.getNumResults()); -// -// query = new Query(); -// query.append(attributes + ".name", "fileTest1k"); -// query.append(attributes + ".field", "value"); -// result = fileManager.search(studyFqn, query, null, token); -// assertEquals(1, result.getNumResults()); -// -// query = new Query(); -// query.append(attributes + ".name", "fileTest1k"); -// query.append(attributes + ".field", "value"); -// query.append(attributes + ".numValue", Arrays.asList(8, 9, 10)); //Searching as String. numValue = "10" -// result = fileManager.search(studyFqn, query, null, token); -// assertEquals(1, result.getNumResults()); + result = fileManager.search(studyFqn, query, null, ownerToken); + assertEquals(3, result.getNumResults()); QueryOptions options = new QueryOptions(QueryOptions.LIMIT, 2).append(QueryOptions.COUNT, true); - result = fileManager.search(studyFqn, new Query(), options, token); - assertEquals(2, result.getNumResults()); - assertEquals(8, result.getNumMatches()); - - options = new QueryOptions(QueryOptions.LIMIT, 2) - .append(QueryOptions.COUNT, true); - result = fileManager.search(studyFqn, new Query(), options, token); + result = fileManager.search(studyFqn, new Query(), options, ownerToken); assertEquals(2, result.getNumResults()); - assertEquals(8, result.getNumMatches()); - + assertEquals(13, result.getNumMatches()); } // // @Test @@ -1794,21 +1658,9 @@ interface Searcher { // assertEquals(2, result.getNumResults()); // } - @Test - public void testSearchFileFail1() throws CatalogException { - thrown.expect(CatalogDBException.class); - fileManager.search(studyFqn, new Query(FileDBAdaptor.QueryParams.NATTRIBUTES.key() + ".numValue", "==NotANumber"), null, - token); - } - @Test public void testGetFileParents1() throws CatalogException { - long fileId; - DataResult fileParents; - - fileId = fileManager.get(studyFqn, "data/test/folder/", FileManager.INCLUDE_FILE_IDS, token).first().getUid(); - fileParents = fileManager.getParents(fileId, null, token); - + DataResult fileParents = fileManager.getParents(studyFqn, "data/test/folder/", true, QueryOptions.empty(), ownerToken); assertEquals(4, fileParents.getNumResults()); assertEquals("", fileParents.getResults().get(0).getPath()); assertEquals("data/", fileParents.getResults().get(1).getPath()); @@ -1818,13 +1670,7 @@ public void testGetFileParents1() throws CatalogException { @Test public void testGetFileParents2() throws CatalogException { - long fileId; - DataResult fileParents; - - fileId = fileManager.get(studyFqn, "data/test/folder/test_1K.txt.gz", FileManager.INCLUDE_FILE_IDS, token) - .first().getUid(); - fileParents = fileManager.getParents(fileId, null, token); - + DataResult fileParents = fileManager.getParents(studyFqn, "data/test/folder/test_1K.txt.gz", true, QueryOptions.empty(), ownerToken); assertEquals(5, fileParents.getNumResults()); assertEquals("", fileParents.getResults().get(0).getPath()); assertEquals("data/", fileParents.getResults().get(1).getPath()); @@ -1835,13 +1681,8 @@ public void testGetFileParents2() throws CatalogException { @Test public void testGetFileParents3() throws CatalogException { - long fileId; - DataResult fileParents; - - fileId = fileManager.get(studyFqn, "data/test/", FileManager.INCLUDE_FILE_IDS, token).first().getUid(); - fileParents = fileManager.getParents(fileId, new QueryOptions("include", "projects.studies.files.path," + - "projects.studies.files.id"), token); - + DataResult fileParents = fileManager.getParents(studyFqn, "data/test/", true, + new QueryOptions("include", "projects.studies.files.path,projects.studies.files.id"), ownerToken); assertEquals(3, fileParents.getNumResults()); assertEquals("", fileParents.getResults().get(0).getPath()); assertEquals("data/", fileParents.getResults().get(1).getPath()); @@ -1858,22 +1699,22 @@ public void testGetFileParents3() throws CatalogException { @Test public void testGetFileWithSamples() throws CatalogException { DataResult fileDataResult = fileManager.get(studyFqn, "data/test/", QueryOptions.empty(), - token); + ownerToken); assertEquals(1, fileDataResult.getNumResults()); assertEquals(0, fileDataResult.first().getSampleIds().size()); // Create two samples - Sample sample1 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("sample1"), INCLUDE_RESULT, token).first(); - Sample sample2 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("sample2"), INCLUDE_RESULT, token).first(); + Sample sample1 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("sample1"), INCLUDE_RESULT, ownerToken).first(); + Sample sample2 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("sample2"), INCLUDE_RESULT, ownerToken).first(); // Associate the two samples to the file fileManager.update(studyFqn, "data/test/", new FileUpdateParams().setSampleIds(Arrays.asList(sample1.getId(), sample2.getId())), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); // Fetch the file fileDataResult = fileManager.get(studyFqn, "data/test/", new QueryOptions( QueryOptions.INCLUDE, Arrays.asList(FileDBAdaptor.QueryParams.ID.key(), FileDBAdaptor.QueryParams.SAMPLE_IDS.key())), - token); + ownerToken); assertEquals(1, fileDataResult.getNumResults()); assertEquals(2, fileDataResult.first().getSampleIds().size()); for (String sampleId : fileDataResult.first().getSampleIds()) { @@ -1888,15 +1729,15 @@ public void testDelete1() throws CatalogException { Query query = new Query() .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(FileDBAdaptor.QueryParams.PATH.key(), filePath); - DataResult fileDataResult = fileManager.search(studyFqn, query, null, token); + DataResult fileDataResult = fileManager.search(studyFqn, query, null, ownerToken); // Change the status to MISSING FileUpdateParams updateParams = new FileUpdateParams() .setInternal(new SmallFileInternal(new FileStatus(FileStatus.MISSING))); - catalogManager.getFileManager().update(studyFqn, filePath, updateParams, null, token); + catalogManager.getFileManager().update(studyFqn, filePath, updateParams, null, ownerToken); try { - fileManager.delete(studyFqn, new Query(FileDBAdaptor.QueryParams.UID.key(), fileDataResult.first().getUid()), null, token); + fileManager.delete(studyFqn, new Query(FileDBAdaptor.QueryParams.UID.key(), fileDataResult.first().getUid()), null, ownerToken); fail("Expected fail. It should not be able to delete"); } catch (CatalogException e) { assertTrue(e.getMessage().contains("Cannot delete")); @@ -1904,10 +1745,10 @@ public void testDelete1() throws CatalogException { // Change the status to STAGED updateParams = new FileUpdateParams() .setInternal(new SmallFileInternal(new FileStatus(FileStatus.STAGE))); - catalogManager.getFileManager().update(studyFqn, filePath, updateParams, null, token); + catalogManager.getFileManager().update(studyFqn, filePath, updateParams, null, ownerToken); try { - fileManager.delete(studyFqn, new Query(FileDBAdaptor.QueryParams.UID.key(), fileDataResult.first().getUid()), null, token); + fileManager.delete(studyFqn, new Query(FileDBAdaptor.QueryParams.UID.key(), fileDataResult.first().getUid()), null, ownerToken); fail("Expected fail. It should not be able to delete"); } catch (CatalogException e) { assertTrue(e.getMessage().contains("Cannot delete")); @@ -1918,9 +1759,9 @@ public void testDelete1() throws CatalogException { setToPendingDelete(studyFqn, query); DataResult deleteResult = fileManager.delete(studyFqn, new Query(FileDBAdaptor.QueryParams.UID.key(), - fileDataResult.first().getUid()), null, token); - assertEquals(6, deleteResult.getNumMatches()); - assertEquals(6, deleteResult.getNumUpdated()); + fileDataResult.first().getUid()), null, ownerToken); + assertEquals(11, deleteResult.getNumMatches()); + assertEquals(11, deleteResult.getNumUpdated()); } // It will try to delete a folder in status ready @@ -1930,25 +1771,25 @@ public void testDelete2() throws CatalogException, IOException { Query query = new Query() .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(FileDBAdaptor.QueryParams.PATH.key(), filePath); - File file = fileManager.search(studyFqn, query, null, token).first(); + File file = fileManager.search(studyFqn, query, null, ownerToken).first(); // We look for all the files and folders that fall within that folder query = new Query() .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(FileDBAdaptor.QueryParams.PATH.key(), "~^" + filePath + "*") .append(FileDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), FileStatus.READY); - int numResults = fileManager.search(studyFqn, query, null, token).getNumResults(); - assertEquals(6, numResults); + int numResults = fileManager.search(studyFqn, query, null, ownerToken).getNumResults(); + assertEquals(11, numResults); query = new Query(FileDBAdaptor.QueryParams.PATH.key(), "~^" + file.getPath() + "*"); setToPendingDelete(studyFqn, query); // We delete it - fileManager.delete(studyFqn, new Query(FileDBAdaptor.QueryParams.UID.key(), file.getUid()), null, token); + fileManager.delete(studyFqn, new Query(FileDBAdaptor.QueryParams.UID.key(), file.getUid()), null, ownerToken); // The files should have been moved to trashed status - OpenCGAResult search = fileManager.search(studyFqn, query, null, token); - assertEquals(6, search.getNumResults()); + OpenCGAResult search = fileManager.search(studyFqn, query, null, ownerToken); + assertEquals(11, search.getNumResults()); for (File trashedFile : search.getResults()) { assertEquals(FileStatus.TRASHED, trashedFile.getInternal().getStatus().getId()); } @@ -1961,56 +1802,56 @@ public void testDelete3() throws CatalogException, IOException { Query query = new Query() .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(FileDBAdaptor.QueryParams.PATH.key(), filePath); - File file = fileManager.search(studyFqn, query, null, token).first(); + File file = fileManager.search(studyFqn, query, null, ownerToken).first(); // We look for all the files and folders that fall within that folder query = new Query() .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(FileDBAdaptor.QueryParams.PATH.key(), "~^" + filePath + "*") .append(FileDBAdaptor.QueryParams.INTERNAL_STATUS.key(), FileStatus.READY); - int numResults = fileManager.search(studyFqn, query, null, token).getNumResults(); - assertEquals(6, numResults); + int numResults = fileManager.search(studyFqn, query, null, ownerToken).getNumResults(); + assertEquals(11, numResults); setToPendingDelete(studyFqn, query); // We delete it QueryOptions queryOptions = new QueryOptions(Constants.SKIP_TRASH, true); fileManager.delete(studyFqn, new Query(FileDBAdaptor.QueryParams.UID.key(), file.getUid()), - queryOptions, token); + queryOptions, ownerToken); // The files should have been moved to trashed status - numResults = fileManager.search(studyFqn, query, null, token).getNumResults(); + numResults = fileManager.search(studyFqn, query, null, ownerToken).getNumResults(); assertEquals(0, numResults); query.put(FileDBAdaptor.QueryParams.DELETED.key(), true); query.put(FileDBAdaptor.QueryParams.INTERNAL_STATUS.key(), FileStatus.DELETED); - numResults = fileManager.search(studyFqn, query, null, token).getNumResults(); - assertEquals(6, numResults); + numResults = fileManager.search(studyFqn, query, null, ownerToken).getNumResults(); + assertEquals(11, numResults); } @Test public void testDeleteFile() throws CatalogException, IOException { List result = fileManager.search(studyFqn, new Query(FileDBAdaptor.QueryParams.TYPE.key(), - "FILE"), new QueryOptions(), token).getResults(); + "FILE"), new QueryOptions(), ownerToken).getResults(); // 1st we set the status to PENDING DELETE. setToPendingDelete(studyFqn, new Query(FileDBAdaptor.QueryParams.TYPE.key(), "FILE")); for (File file : result) { - fileManager.delete(studyFqn, new Query(FileDBAdaptor.QueryParams.UID.key(), file.getUid()), null, token); + fileManager.delete(studyFqn, new Query(FileDBAdaptor.QueryParams.UID.key(), file.getUid()), null, ownerToken); } // CatalogFileUtils catalogFileUtils = new CatalogFileUtils(catalogManager); - fileManager.search(studyFqn, new Query(FileDBAdaptor.QueryParams.TYPE.key(), "FILE"), new QueryOptions(), token).getResults().forEach(f -> { + fileManager.search(studyFqn, new Query(FileDBAdaptor.QueryParams.TYPE.key(), "FILE"), new QueryOptions(), ownerToken).getResults().forEach(f -> { assertEquals(f.getInternal().getStatus().getId(), FileStatus.TRASHED); }); - result = fileManager.search(studyFqn2, new Query(FileDBAdaptor.QueryParams.TYPE.key(), "FILE"), new QueryOptions(), token).getResults(); + result = fileManager.search(studyFqn2, new Query(FileDBAdaptor.QueryParams.TYPE.key(), "FILE"), new QueryOptions(), ownerToken).getResults(); // 1st we set the status to PENDING DELETE. setToPendingDelete(studyFqn2, new Query(FileDBAdaptor.QueryParams.TYPE.key(), "FILE")); for (File file : result) { - fileManager.delete(studyFqn2, new Query(FileDBAdaptor.QueryParams.UID.key(), file.getUid()), null, token); + fileManager.delete(studyFqn2, new Query(FileDBAdaptor.QueryParams.UID.key(), file.getUid()), null, ownerToken); } - fileManager.search(studyFqn2, new Query(FileDBAdaptor.QueryParams.TYPE.key(), "FILE"), new QueryOptions(), token).getResults().forEach(f -> { + fileManager.search(studyFqn2, new Query(FileDBAdaptor.QueryParams.TYPE.key(), "FILE"), new QueryOptions(), ownerToken).getResults().forEach(f -> { assertEquals(f.getInternal().getStatus().getId(), FileStatus.TRASHED); }); } @@ -2018,12 +1859,12 @@ public void testDeleteFile() throws CatalogException, IOException { @Test public void removeFileReferencesFromSamplesOnFileDeleteTest() throws CatalogException { String fileId = "data:test:folder:test_0.1K.png"; - File file = fileManager.get(studyFqn, fileId, QueryOptions.empty(), token).first(); + File file = fileManager.get(studyFqn, fileId, QueryOptions.empty(), ownerToken).first(); assertFalse(file.getSampleIds().isEmpty()); assertEquals(5, file.getSampleIds().size()); List samples = catalogManager.getSampleManager().search(studyFqn, - new Query(SampleDBAdaptor.QueryParams.FILE_IDS.key(), fileId), QueryOptions.empty(), token).getResults(); + new Query(SampleDBAdaptor.QueryParams.FILE_IDS.key(), fileId), QueryOptions.empty(), ownerToken).getResults(); assertEquals(5, samples.size()); for (Sample sample : samples) { @@ -2033,15 +1874,15 @@ public void removeFileReferencesFromSamplesOnFileDeleteTest() throws CatalogExce // Send to TRASH BIN setToPendingDelete(studyFqn, new Query(FileDBAdaptor.QueryParams.ID.key(), fileId)); - fileManager.delete(studyFqn, Collections.singletonList(fileId), QueryOptions.empty(), token); + fileManager.delete(studyFqn, Collections.singletonList(fileId), QueryOptions.empty(), ownerToken); - file = fileManager.get(studyFqn, fileId, QueryOptions.empty(), token).first(); + file = fileManager.get(studyFqn, fileId, QueryOptions.empty(), ownerToken).first(); assertFalse(file.getSampleIds().isEmpty()); assertEquals(5, file.getSampleIds().size()); assertEquals(FileStatus.TRASHED, file.getInternal().getStatus().getId()); samples = catalogManager.getSampleManager().search(studyFqn, - new Query(SampleDBAdaptor.QueryParams.FILE_IDS.key(), fileId), QueryOptions.empty(), token).getResults(); + new Query(SampleDBAdaptor.QueryParams.FILE_IDS.key(), fileId), QueryOptions.empty(), ownerToken).getResults(); assertEquals(5, samples.size()); for (Sample sample : samples) { @@ -2051,14 +1892,14 @@ public void removeFileReferencesFromSamplesOnFileDeleteTest() throws CatalogExce // Delete permanently setToPendingDelete(studyFqn, new Query(FileDBAdaptor.QueryParams.ID.key(), fileId)); - fileManager.delete(studyFqn, Collections.singletonList(fileId), new QueryOptions(Constants.SKIP_TRASH, true), token); + fileManager.delete(studyFqn, Collections.singletonList(fileId), new QueryOptions(Constants.SKIP_TRASH, true), ownerToken); List noResults = catalogManager.getSampleManager().search(studyFqn, - new Query(SampleDBAdaptor.QueryParams.FILE_IDS.key(), fileId), QueryOptions.empty(), token).getResults(); + new Query(SampleDBAdaptor.QueryParams.FILE_IDS.key(), fileId), QueryOptions.empty(), ownerToken).getResults(); assertEquals(0, noResults.size()); samples = catalogManager.getSampleManager().get(studyFqn, samples.stream().map(Sample::getId).collect(Collectors.toList()), - QueryOptions.empty(), token).getResults(); + QueryOptions.empty(), ownerToken).getResults(); assertEquals(5, samples.size()); for (Sample sample : samples) { @@ -2067,29 +1908,29 @@ public void removeFileReferencesFromSamplesOnFileDeleteTest() throws CatalogExce thrown.expect(CatalogException.class); thrown.expectMessage("Missing"); - fileManager.get(studyFqn, fileId, QueryOptions.empty(), token).first(); + fileManager.get(studyFqn, fileId, QueryOptions.empty(), ownerToken).first(); } @Test public void testDeleteLeafFolder() throws CatalogException, IOException { - File deletable = fileManager.get(studyFqn, "/data/test/folder/", QueryOptions.empty(), token).first(); + File deletable = fileManager.get(studyFqn, "/data/test/folder/", QueryOptions.empty(), ownerToken).first(); deleteFolderAndCheck(deletable); } @Test public void testDeleteMiddleFolder() throws CatalogException, IOException { - File deletable = fileManager.get(studyFqn, "/data/", QueryOptions.empty(), token).first(); + File deletable = fileManager.get(studyFqn, "/data/", QueryOptions.empty(), ownerToken).first(); deleteFolderAndCheck(deletable); } @Test public void testDeleteRootFolder() throws CatalogException { - File deletable = fileManager.get(studyFqn, "/", QueryOptions.empty(), token).first(); + File deletable = fileManager.get(studyFqn, "/", QueryOptions.empty(), ownerToken).first(); thrown.expect(CatalogException.class); thrown.expectMessage("Root directories cannot be deleted"); - fileManager.delete(studyFqn, new Query(FileDBAdaptor.QueryParams.PATH.key(), deletable.getPath()), null, token); + fileManager.delete(studyFqn, new Query(FileDBAdaptor.QueryParams.PATH.key(), deletable.getPath()), null, ownerToken); } // Cannot delete staged files @@ -2099,9 +1940,9 @@ public void deleteFolderTest() throws CatalogException, IOException { File folder = createBasicDirectoryFileTestEnvironment(folderFiles); - IOManager ioManager = catalogManager.getIoManagerFactory().get(fileManager.getUri(folder)); + IOManager ioManager = catalogManager.getIoManagerFactory().get(fileManager.getUri(organizationId, folder)); for (File file : folderFiles) { - assertTrue(ioManager.exists(fileManager.getUri(file))); + assertTrue(ioManager.exists(fileManager.getUri(organizationId, file))); } fileManager.create(studyFqn, new FileCreateParams() @@ -2109,19 +1950,19 @@ public void deleteFolderTest() throws CatalogException, IOException { .setFormat(File.Format.PLAIN) .setPath("folder/subfolder/subsubfolder/my_staged.txt") .setContent("bla bla"), - true, token).first(); + true, ownerToken).first(); Query query = new Query() .append(FileDBAdaptor.QueryParams.PATH.key(), "~^" + folder.getPath() + "*") .append(FileDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), FileStatus.READY); setToPendingDelete(studyFqn, query); - File fileTmp = fileManager.get(studyFqn, folder.getPath(), null, token).first(); + File fileTmp = fileManager.get(studyFqn, folder.getPath(), null, ownerToken).first(); assertEquals("Folder name should not be modified", folder.getPath(), fileTmp.getPath()); assertTrue(ioManager.exists(fileTmp.getUri())); for (File file : folderFiles) { - fileTmp = fileManager.get(studyFqn, file.getPath(), null, token).first(); + fileTmp = fileManager.get(studyFqn, file.getPath(), null, ownerToken).first(); assertEquals("File name should not be modified", file.getPath(), fileTmp.getPath()); assertTrue("File uri: " + fileTmp.getUri() + " should exist", ioManager.exists(fileTmp.getUri())); } @@ -2135,20 +1976,20 @@ public void deleteFolderTest2() throws CatalogException, IOException { File folder = createBasicDirectoryFileTestEnvironment(folderFiles); - IOManager ioManager = catalogManager.getIoManagerFactory().get(fileManager.getUri(folder)); + IOManager ioManager = catalogManager.getIoManagerFactory().get(fileManager.getUri(organizationId, folder)); for (File file : folderFiles) { - assertTrue(ioManager.exists(fileManager.getUri(file))); + assertTrue(ioManager.exists(fileManager.getUri(organizationId, file))); } // 1st we set the status to PENDING DELETE. setToPendingDelete(studyFqn, new Query(FileDBAdaptor.QueryParams.PATH.key(), "~^" + folder.getPath() + "*")); // Now we delete the files - fileManager.delete(studyFqn, new Query(FileDBAdaptor.QueryParams.UID.key(), folder.getUid()), null, token); + fileManager.delete(studyFqn, new Query(FileDBAdaptor.QueryParams.UID.key(), folder.getUid()), null, ownerToken); Query query = new Query() .append(FileDBAdaptor.QueryParams.UID.key(), folder.getUid()); - File fileTmp = fileManager.search(studyFqn, query, QueryOptions.empty(), token).first(); + File fileTmp = fileManager.search(studyFqn, query, QueryOptions.empty(), ownerToken).first(); assertEquals("Folder name should not be modified", folder.getPath(), fileTmp.getPath()); assertEquals("Status should be to TRASHED", FileStatus.TRASHED, fileTmp.getInternal().getStatus().getId()); @@ -2157,7 +1998,7 @@ public void deleteFolderTest2() throws CatalogException, IOException { for (File file : folderFiles) { query.put(FileDBAdaptor.QueryParams.UID.key(), file.getUid()); - fileTmp = fileManager.search(studyFqn, query, QueryOptions.empty(), token).first(); + fileTmp = fileManager.search(studyFqn, query, QueryOptions.empty(), ownerToken).first(); assertEquals("Folder name should not be modified", file.getPath(), fileTmp.getPath()); assertEquals("Status should be to TRASHED", FileStatus.TRASHED, fileTmp.getInternal().getStatus().getId()); assertEquals("Name should not have changed", file.getName(), fileTmp.getName()); @@ -2172,9 +2013,9 @@ public void deleteFolderTest3() throws CatalogException, IOException { File folder = createBasicDirectoryFileTestEnvironment(folderFiles); - IOManager ioManager = catalogManager.getIoManagerFactory().get(fileManager.getUri(folder)); + IOManager ioManager = catalogManager.getIoManagerFactory().get(fileManager.getUri(organizationId, folder)); for (File file : folderFiles) { - assertTrue(ioManager.exists(fileManager.getUri(file))); + assertTrue(ioManager.exists(fileManager.getUri(organizationId, file))); } // 1st we set the status to PENDING DELETE. @@ -2182,7 +2023,7 @@ public void deleteFolderTest3() throws CatalogException, IOException { Query query = new Query(FileDBAdaptor.QueryParams.PATH.key(), "~^" + folder.getPath() + "*"); OpenCGAResult results = fileManager.search(studyFqn, query, - new QueryOptions(QueryOptions.INCLUDE, FileDBAdaptor.QueryParams.INTERNAL_STATUS.key()), token); + new QueryOptions(QueryOptions.INCLUDE, FileDBAdaptor.QueryParams.INTERNAL_STATUS.key()), ownerToken); assertEquals(9, results.getNumResults()); for (File result : results.getResults()) { assertEquals(FileStatus.PENDING_DELETE, result.getInternal().getStatus().getId()); @@ -2196,9 +2037,9 @@ public void deleteFolderTest4() throws CatalogException, IOException { File folder = createBasicDirectoryFileTestEnvironment(folderFiles); - IOManager ioManager = catalogManager.getIoManagerFactory().get(fileManager.getUri(folder)); + IOManager ioManager = catalogManager.getIoManagerFactory().get(fileManager.getUri(organizationId, folder)); for (File file : folderFiles) { - assertTrue(ioManager.exists(fileManager.getUri(file))); + assertTrue(ioManager.exists(fileManager.getUri(organizationId, file))); } // 1st we set the status to PENDING DELETE. @@ -2206,12 +2047,12 @@ public void deleteFolderTest4() throws CatalogException, IOException { // Now we delete the files QueryOptions params = new QueryOptions(Constants.SKIP_TRASH, true); - fileManager.delete(studyFqn, new Query(FileDBAdaptor.QueryParams.UID.key(), folder.getUid()), params, token); + fileManager.delete(studyFqn, new Query(FileDBAdaptor.QueryParams.UID.key(), folder.getUid()), params, ownerToken); Query query = new Query() .append(FileDBAdaptor.QueryParams.UID.key(), folder.getUid()) .append(FileDBAdaptor.QueryParams.DELETED.key(), true); - File fileTmp = fileManager.search(studyFqn, query, QueryOptions.empty(), token).first(); + File fileTmp = fileManager.search(studyFqn, query, QueryOptions.empty(), ownerToken).first(); assertEquals("Folder name should not be modified", folder.getPath(), fileTmp.getPath()); assertEquals("Status should be to DELETED", FileStatus.DELETED, fileTmp.getInternal().getStatus().getId()); @@ -2220,7 +2061,7 @@ public void deleteFolderTest4() throws CatalogException, IOException { for (File file : folderFiles) { query.put(FileDBAdaptor.QueryParams.UID.key(), file.getUid()); - fileTmp = fileManager.search(studyFqn, query, QueryOptions.empty(), token).first(); + fileTmp = fileManager.search(studyFqn, query, QueryOptions.empty(), ownerToken).first(); assertEquals("Folder name should not be modified", file.getPath(), fileTmp.getPath()); assertEquals("Status should be to DELETED", FileStatus.DELETED, fileTmp.getInternal().getStatus().getId()); assertEquals("Name should not have changed", file.getName(), fileTmp.getName()); @@ -2232,18 +2073,18 @@ public void deleteFolderTest4() throws CatalogException, IOException { public void deleteFileInClinicalAnalysis() throws CatalogException, IOException { // START DATA PREPARATION FOR TEST !!! String bamFile = getClass().getResource("/biofiles/NA19600.chrom20.small.bam").getFile(); - File file = fileManager.link(studyFqn, new FileLinkParams(bamFile, "", "", "", null, null, null, null, null), false, token).first(); + File file = fileManager.link(studyFqn, new FileLinkParams(bamFile, "", "", "", null, null, null, null, null), false, ownerToken).first(); Family family1 = DummyModelUtils.getDummyFamily("familyId1"); - catalogManager.getFamilyManager().create(studyFqn, family1, QueryOptions.empty(), token); + catalogManager.getFamilyManager().create(studyFqn, family1, QueryOptions.empty(), ownerToken); // Associate BAM file to sample String sampleId = family1.getMembers().get(0).getSamples().get(0).getId(); catalogManager.getFileManager().update(studyFqn, file.getId(), new FileUpdateParams().setSampleIds(Collections.singletonList(sampleId)), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); Panel myPanel = DummyModelUtils.getDummyPanel("myPanel"); - catalogManager.getPanelManager().create(studyFqn, myPanel, QueryOptions.empty(), token); + catalogManager.getPanelManager().create(studyFqn, myPanel, QueryOptions.empty(), ownerToken); Family copy = JacksonUtils.copy(family1, Family.class); for (Individual member : copy.getMembers()) { @@ -2252,41 +2093,41 @@ public void deleteFileInClinicalAnalysis() throws CatalogException, IOException } ClinicalAnalysis clinicalAnalysis1 = DummyModelUtils.getDummyClinicalAnalysis(copy.getMembers().get(0), copy, Collections.singletonList(myPanel)); - clinicalAnalysis1 = catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis1, INCLUDE_RESULT, token).first(); + clinicalAnalysis1 = catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis1, INCLUDE_RESULT, ownerToken).first(); assertEquals(1, clinicalAnalysis1.getFiles().size()); assertEquals(file.getPath(), clinicalAnalysis1.getFiles().get(0).getPath()); assertFalse(clinicalAnalysis1.isLocked()); ClinicalAnalysis clinicalAnalysis2 = DummyModelUtils.getDummyClinicalAnalysis(copy.getMembers().get(0), copy, Collections.singletonList(myPanel)); - clinicalAnalysis2 = catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis2, INCLUDE_RESULT, token).first(); + clinicalAnalysis2 = catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis2, INCLUDE_RESULT, ownerToken).first(); assertEquals(1, clinicalAnalysis2.getFiles().size()); assertEquals(file.getPath(), clinicalAnalysis2.getFiles().get(0).getPath()); assertFalse(clinicalAnalysis2.isLocked()); // Lock clinicalAnalysis2 clinicalAnalysis2 = catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis2.getId(), - new ClinicalAnalysisUpdateParams().setLocked(true), INCLUDE_RESULT, token).first(); + new ClinicalAnalysisUpdateParams().setLocked(true), INCLUDE_RESULT, ownerToken).first(); assertTrue(clinicalAnalysis2.isLocked()); // END DATA PREPARATION FOR TEST !!! // Mark as pending delete - catalogManager.getFileManager().fileDBAdaptor.update(file.getUid(), new ObjectMap(FileDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), FileStatus.PENDING_DELETE), QueryOptions.empty()); - CatalogException catalogException = assertThrows(CatalogException.class, () -> catalogManager.getFileManager().unlink(studyFqn, file.getId(), token)); + catalogManager.getFileManager().getFileDBAdaptor(organizationId).update(file.getUid(), new ObjectMap(FileDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), FileStatus.PENDING_DELETE), QueryOptions.empty()); + CatalogException catalogException = assertThrows(CatalogException.class, () -> catalogManager.getFileManager().unlink(studyFqn, file.getId(), ownerToken)); assertTrue(catalogException.getMessage().contains("Could not unlink")); assertTrue(catalogException.getCause().getMessage().contains("clinical analyses")); // Unlock clinicalAnalysis2 clinicalAnalysis2 = catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis2.getId(), - new ClinicalAnalysisUpdateParams().setLocked(false), INCLUDE_RESULT, token).first(); + new ClinicalAnalysisUpdateParams().setLocked(false), INCLUDE_RESULT, ownerToken).first(); assertFalse(clinicalAnalysis2.isLocked()); // Unlink file - catalogManager.getFileManager().unlink(studyFqn, file.getId(), token); + catalogManager.getFileManager().unlink(studyFqn, file.getId(), ownerToken); - Sample sample = catalogManager.getSampleManager().get(studyFqn, sampleId, QueryOptions.empty(), token).first(); + Sample sample = catalogManager.getSampleManager().get(studyFqn, sampleId, QueryOptions.empty(), ownerToken).first(); assertEquals(0, sample.getFileIds().size()); - OpenCGAResult search = catalogManager.getClinicalAnalysisManager().search(studyFqn, new Query(), QueryOptions.empty(), token); + OpenCGAResult search = catalogManager.getClinicalAnalysisManager().search(studyFqn, new Query(), QueryOptions.empty(), ownerToken); assertEquals(2, search.getNumResults()); for (ClinicalAnalysis clinicalAnalysis : search.getResults()) { assertEquals(0, clinicalAnalysis.getFiles().size()); @@ -2300,35 +2141,35 @@ public void deleteFileUserInRelatedFilesTest() throws CatalogException { fileManager.update(studyFqn, "data/test/folder/test_1K.txt.gz", new FileUpdateParams().setRelatedFiles(Collections.singletonList( new SmallRelatedFileParams("data/test/folder/test_0.5K.txt", FileRelatedFile.Relation.PART_OF_PAIR))), - null, token); - File file = fileManager.get(studyFqn, "data/test/folder/test_1K.txt.gz", QueryOptions.empty(), token).first(); + null, ownerToken); + File file = fileManager.get(studyFqn, "data/test/folder/test_1K.txt.gz", QueryOptions.empty(), ownerToken).first(); assertFalse(file.getRelatedFiles().isEmpty()); assertEquals(1, file.getRelatedFiles().size()); assertEquals("data/test/folder/test_0.5K.txt", file.getRelatedFiles().get(0).getFile().getPath()); - file = fileManager.get(studyFqn, "data/test/folder/test_0.5K.txt", FileManager.INCLUDE_FILE_IDS, token).first(); + file = fileManager.get(studyFqn, "data/test/folder/test_0.5K.txt", FileManager.INCLUDE_FILE_IDS, ownerToken).first(); // Mark as pending delete - catalogManager.getFileManager().fileDBAdaptor.update(file.getUid(), new ObjectMap(FileDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), FileStatus.PENDING_DELETE), QueryOptions.empty()); + catalogManager.getFileManager().getFileDBAdaptor(organizationId).update(file.getUid(), new ObjectMap(FileDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), FileStatus.PENDING_DELETE), QueryOptions.empty()); // Delete test_0.5K file QueryOptions options = new QueryOptions(Constants.SKIP_TRASH, true); - fileManager.delete(studyFqn, Collections.singletonList("data/test/folder/test_0.5K.txt"), options, token); + fileManager.delete(studyFqn, Collections.singletonList("data/test/folder/test_0.5K.txt"), options, ownerToken); // Ensure there are no more references to test_0.5K file - file = fileManager.get(studyFqn, "data/test/folder/test_1K.txt.gz", QueryOptions.empty(), token).first(); + file = fileManager.get(studyFqn, "data/test/folder/test_1K.txt.gz", QueryOptions.empty(), ownerToken).first(); assertTrue(file.getRelatedFiles().isEmpty()); } private File createBasicDirectoryFileTestEnvironment(List folderFiles) throws CatalogException { File folder = fileManager.createFolder(studyFqn, Paths.get("folder").toString(), false, - null, QueryOptions.empty(), token).first(); + null, QueryOptions.empty(), ownerToken).first(); folderFiles.add( fileManager.create(studyFqn, new FileCreateParams() .setPath("folder/my.txt") .setType(File.Type.FILE) .setContent(RandomStringUtils.randomAlphanumeric(200)), - false, token).first() + false, ownerToken).first() ); folderFiles.add( fileManager.create(studyFqn, @@ -2336,7 +2177,7 @@ private File createBasicDirectoryFileTestEnvironment(List folderFiles) thr .setPath("folder/my2.txt") .setType(File.Type.FILE) .setContent(RandomStringUtils.randomAlphanumeric(200)), - false, token).first() + false, ownerToken).first() ); folderFiles.add( fileManager.create(studyFqn, @@ -2344,7 +2185,7 @@ private File createBasicDirectoryFileTestEnvironment(List folderFiles) thr .setPath("folder/my3.txt") .setType(File.Type.FILE) .setContent(RandomStringUtils.randomAlphanumeric(200)), - false, token).first() + false, ownerToken).first() ); folderFiles.add( fileManager.create(studyFqn, @@ -2352,7 +2193,7 @@ private File createBasicDirectoryFileTestEnvironment(List folderFiles) thr .setPath("folder/subfolder/my4.txt") .setType(File.Type.FILE) .setContent(RandomStringUtils.randomAlphanumeric(200)), - true, token).first() + true, ownerToken).first() ); folderFiles.add( fileManager.create(studyFqn, @@ -2360,7 +2201,7 @@ private File createBasicDirectoryFileTestEnvironment(List folderFiles) thr .setPath("folder/subfolder/my5.txt") .setType(File.Type.FILE) .setContent(RandomStringUtils.randomAlphanumeric(200)), - false, token).first() + false, ownerToken).first() ); folderFiles.add( fileManager.create(studyFqn, @@ -2368,7 +2209,7 @@ private File createBasicDirectoryFileTestEnvironment(List folderFiles) thr .setPath("folder/subfolder/subsubfolder/my6.txt") .setType(File.Type.FILE) .setContent(RandomStringUtils.randomAlphanumeric(200)), - true, token).first() + true, ownerToken).first() ); return folder; } @@ -2380,32 +2221,32 @@ public void sendFolderToTrash() { @Test public void getAllFilesInFolder() throws CatalogException { - List allFilesInFolder = fileManager.getFilesFromFolder("/data/test/folder/", studyFqn, null, - token).getResults(); + List allFilesInFolder = fileManager.getFilesFromFolder(studyFqn, "/data/test/folder/", null, + ownerToken).getResults(); assertEquals(3, allFilesInFolder.size()); } private void deleteFolderAndCheck(File deletable) throws CatalogException { List allFilesInFolder; - Study study = fileManager.getStudy(deletable, token); + Study study = fileManager.getStudy(organizationId, deletable, ownerToken); // 1st, we set the status to PENDING_DELETE Query query = new Query(FileDBAdaptor.QueryParams.PATH.key(), "~^" + deletable.getPath() + "*"); setToPendingDelete(study.getFqn(), query); - fileManager.delete(study.getFqn(), query, null, token); + fileManager.delete(study.getFqn(), query, null, ownerToken); query = new Query() .append(FileDBAdaptor.QueryParams.PATH.key(), deletable.getPath()) .append(FileDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), FileStatus.TRASHED); QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, FileDBAdaptor.QueryParams.PATH.key()); - DataResult fileDataResult = fileManager.search(study.getFqn(), query, options, token); + DataResult fileDataResult = fileManager.search(study.getFqn(), query, options, ownerToken); assertEquals(1, fileDataResult.getNumResults()); query = new Query() .append(FileDBAdaptor.QueryParams.DIRECTORY.key(), fileDataResult.first().getPath() + ".*") .append(FileDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), FileStatus.TRASHED); - allFilesInFolder = fileManager.search(study.getFqn(), query, null, token).getResults(); + allFilesInFolder = fileManager.search(study.getFqn(), query, null, ownerToken).getResults(); for (File subFile : allFilesInFolder) { assertTrue(subFile.getInternal().getStatus().getId().equals(FileStatus.TRASHED)); @@ -2415,14 +2256,14 @@ private void deleteFolderAndCheck(File deletable) throws CatalogException { private void setToPendingDelete(String study, Query query) throws CatalogException { FileUpdateParams updateParams = new FileUpdateParams() .setInternal(new SmallFileInternal(new FileStatus(FileStatus.PENDING_DELETE))); - fileManager.update(study, query, updateParams, QueryOptions.empty(), token); + fileManager.update(study, query, updateParams, QueryOptions.empty(), ownerToken); } @Test public void assignPermissionsRecursively() throws Exception { Path folderPath = Paths.get("data", "new", "folder"); fileManager.createFolder(studyFqn, folderPath.toString(), true, null, - QueryOptions.empty(), token).first(); + QueryOptions.empty(), ownerToken).first(); Path filePath = Paths.get("data", "file1.txt"); fileManager.create(studyFqn, @@ -2433,15 +2274,15 @@ public void assignPermissionsRecursively() throws Exception { .setPath(filePath.toString()) .setDescription("") .setContent("My content"), - true, token); + true, ownerToken); OpenCGAResult> dataResult = fileManager.updateAcl(studyFqn, Arrays.asList("data/new/", - filePath.toString()), "user2", new FileAclParams(null, "VIEW"), ParamUtils.AclAction.SET, token); + filePath.toString()), normalUserId2, new FileAclParams(null, "VIEW"), ParamUtils.AclAction.SET, ownerToken); assertEquals(3, dataResult.getNumResults()); for (AclEntryList result : dataResult.getResults()) { assertEquals(1, result.getAcl().size()); - assertEquals("user2", result.getAcl().get(0).getMember()); + assertEquals(normalUserId2, result.getAcl().get(0).getMember()); assertEquals(1, result.getAcl().get(0).getPermissions().size()); assertTrue(result.getAcl().get(0).getPermissions().contains(FilePermissions.VIEW)); } @@ -2454,10 +2295,11 @@ public void testUpdateIndexStatus() throws CatalogException, URISyntaxException, DataResult fileResult = fileManager.link(studyFqn, new FileLinkParams() .setUri(sourcePath.toString()) .setPath("data/"), - true, token); + true, ownerToken); - fileManager.updateFileInternalVariantIndex(fileResult.first(), FileInternalVariantIndex.init().setStatus(new VariantIndexStatus(VariantIndexStatus.TRANSFORMED, null)), token); - DataResult read = fileManager.get(studyFqn, fileResult.first().getPath(), new QueryOptions(), token); + fileManager.updateFileInternalVariantIndex(studyFqn, fileResult.first(), FileInternalVariantIndex.init() + .setStatus(new VariantIndexStatus(VariantIndexStatus.TRANSFORMED, null)), ownerToken); + DataResult read = fileManager.get(studyFqn, fileResult.first().getPath(), new QueryOptions(), ownerToken); assertEquals(VariantIndexStatus.TRANSFORMED, read.first().getInternal().getVariant().getIndex().getStatus().getId()); } @@ -2472,11 +2314,11 @@ public void testMoveAndRegister() throws URISyntaxException, CatalogException, I catalogManager.getIoManagerFactory().getDefault().deleteDirectory(Paths.get("/tmp/other").toUri()); } - Study study = catalogManager.getStudyManager().resolveId(studyFqn, "user"); + Study study = catalogManager.getStudyManager().resolveId(studyFqn, "user", organizationId); Path studyPath = Paths.get(study.getUri()); // Register in workspace folder - OpenCGAResult result = fileManager.moveAndRegister(studyFqn, copy, studyPath.resolve("myFolder"), "myFolder", token); + OpenCGAResult result = fileManager.moveAndRegister(studyFqn, copy, studyPath.resolve("myFolder"), "myFolder", ownerToken); assertEquals("myFolder/variant-test-file.vcf.gz", result.first().getPath()); assertEquals(studyPath.resolve("myFolder").resolve("variant-test-file.vcf.gz").toString(), Paths.get(result.first().getUri()).toString()); @@ -2485,12 +2327,12 @@ public void testMoveAndRegister() throws URISyntaxException, CatalogException, I // We remove the file to start again Query query = new Query(FileDBAdaptor.QueryParams.UID.key(), result.first().getUid()); setToPendingDelete(studyFqn, query); - fileManager.delete(studyFqn, query, new QueryOptions(Constants.SKIP_TRASH, true), token); - assertEquals(0, fileManager.search(studyFqn, query, QueryOptions.empty(), token).getNumResults()); + fileManager.delete(studyFqn, query, new QueryOptions(Constants.SKIP_TRASH, true), ownerToken); + assertEquals(0, fileManager.search(studyFqn, query, QueryOptions.empty(), ownerToken).getNumResults()); Files.copy(sourcePath, copy); // Register without passing the path - result = fileManager.moveAndRegister(studyFqn, copy, studyPath.resolve("myFolder"), null, token); + result = fileManager.moveAndRegister(studyFqn, copy, studyPath.resolve("myFolder"), null, ownerToken); assertEquals("myFolder/variant-test-file.vcf.gz", result.first().getPath()); assertEquals(studyPath.resolve("myFolder").resolve("variant-test-file.vcf.gz").toString(), Paths.get(result.first().getUri()).toString()); assertTrue(Files.exists(studyPath.resolve("myFolder").resolve("variant-test-file.vcf.gz"))); @@ -2498,12 +2340,12 @@ public void testMoveAndRegister() throws URISyntaxException, CatalogException, I // We remove the file to start again query = new Query(FileDBAdaptor.QueryParams.UID.key(), result.first().getUid()); setToPendingDelete(studyFqn, query); - fileManager.delete(studyFqn, query, new QueryOptions(Constants.SKIP_TRASH, true), token); - assertEquals(0, fileManager.search(studyFqn, query, QueryOptions.empty(), token).getNumResults()); + fileManager.delete(studyFqn, query, new QueryOptions(Constants.SKIP_TRASH, true), ownerToken); + assertEquals(0, fileManager.search(studyFqn, query, QueryOptions.empty(), ownerToken).getNumResults()); Files.copy(sourcePath, copy); // Register without passing the destiny path - result = fileManager.moveAndRegister(studyFqn, copy, null, "myFolder", token); + result = fileManager.moveAndRegister(studyFqn, copy, null, "myFolder", ownerToken); assertEquals("myFolder/variant-test-file.vcf.gz", result.first().getPath()); assertEquals(studyPath.resolve("myFolder").resolve("variant-test-file.vcf.gz").toString(), Paths.get(result.first().getUri()).toString()); assertTrue(Files.exists(studyPath.resolve("myFolder").resolve("variant-test-file.vcf.gz"))); @@ -2511,37 +2353,37 @@ public void testMoveAndRegister() throws URISyntaxException, CatalogException, I // We remove the file to start again query = new Query(FileDBAdaptor.QueryParams.UID.key(), result.first().getUid()); setToPendingDelete(studyFqn, query); - fileManager.delete(studyFqn, query, new QueryOptions(Constants.SKIP_TRASH, true), token); - assertEquals(0, fileManager.search(studyFqn, query, QueryOptions.empty(), token).getNumResults()); + fileManager.delete(studyFqn, query, new QueryOptions(Constants.SKIP_TRASH, true), ownerToken); + assertEquals(0, fileManager.search(studyFqn, query, QueryOptions.empty(), ownerToken).getNumResults()); Files.copy(sourcePath, copy); // Register to an incorrect path try { - fileManager.moveAndRegister(studyFqn, copy, studyPath.resolve("myFolder"), "otherFolder", token); + fileManager.moveAndRegister(studyFqn, copy, studyPath.resolve("myFolder"), "otherFolder", ownerToken); fail("The method should have raised an error saying the path does not match the one corresponding to the uri. It should both " + "point to myFolder or to otherFolder, but not to different paths."); } catch (CatalogException e) { assertTrue("Destination uri within the workspace and path do not match".equals(e.getMessage())); } - // We grant permissions to user2 to the study - catalogManager.getStudyManager().updateAcl(Collections.singletonList(studyFqn), "user2", - new StudyAclParams("", "admin"), ParamUtils.AclAction.ADD, token); + // We grant permissions to normalUserId2 to the study + catalogManager.getStudyManager().updateAcl(studyFqn, normalUserId2, + new StudyAclParams("", "admin"), ParamUtils.AclAction.ADD, ownerToken); // Now, instead of moving it to the user's workspace, we will move it to an external path try { - fileManager.moveAndRegister(studyFqn, copy, Paths.get("/tmp/other/"), "a/b/c/", sessionIdUser2); + fileManager.moveAndRegister(studyFqn, copy, Paths.get("/tmp/other/"), "a/b/c/", normalToken2); fail("user2 should not have permissions to move to an external folder"); } catch (CatalogAuthorizationException e) { assertTrue(e.getMessage().contains("owners or administrative users")); } - // Now we add user2 to admins group - catalogManager.getStudyManager().updateGroup(studyFqn, "admins", ParamUtils.BasicUpdateAction.ADD, - new GroupUpdateParams(Collections.singletonList("user2")), token); + // Now we add normalUserId2 to admins group + catalogManager.getStudyManager().updateGroup(studyFqn, ParamConstants.ADMINS_GROUP, ParamUtils.BasicUpdateAction.ADD, + new GroupUpdateParams(Collections.singletonList(normalUserId2)), ownerToken); // and try the same action again - result = fileManager.moveAndRegister(studyFqn, copy, Paths.get("/tmp/other/"), "a/b/c/", sessionIdUser2); + result = fileManager.moveAndRegister(studyFqn, copy, Paths.get("/tmp/other/"), "a/b/c/", normalToken2); assertEquals("a/b/c/variant-test-file.vcf.gz", result.first().getPath()); assertEquals("/tmp/other/variant-test-file.vcf.gz", Paths.get(result.first().getUri()).toString()); assertTrue(Files.exists(Paths.get("/tmp/other/variant-test-file.vcf.gz"))); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/IndividualManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/IndividualManagerTest.java index 1f01b9d1acd..8755cc50c4a 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/IndividualManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/IndividualManagerTest.java @@ -44,46 +44,46 @@ public class IndividualManagerTest extends AbstractManagerTest { @Test public void testAnnotateIndividual() throws CatalogException { - VariableSet variableSet = catalogManager.getStudyManager().getVariableSet(studyFqn, "vs", null, token).first(); + VariableSet variableSet = catalogManager.getStudyManager().getVariableSet(studyFqn, "vs", null, ownerToken).first(); String individualId1 = catalogManager.getIndividualManager().create(studyFqn, new Individual().setId("INDIVIDUAL_1") .setKaryotypicSex(IndividualProperty.KaryotypicSex.UNKNOWN).setLifeStatus(IndividualProperty.LifeStatus.UNKNOWN), - INCLUDE_RESULT, token).first().getId(); + INCLUDE_RESULT, ownerToken).first().getId(); String individualId2 = catalogManager.getIndividualManager().create(studyFqn, new Individual().setId("INDIVIDUAL_2") .setKaryotypicSex(IndividualProperty.KaryotypicSex.UNKNOWN).setLifeStatus(IndividualProperty.LifeStatus.UNKNOWN), - INCLUDE_RESULT, token).first().getId(); + INCLUDE_RESULT, ownerToken).first().getId(); String individualId3 = catalogManager.getIndividualManager().create(studyFqn, new Individual().setId("INDIVIDUAL_3") .setKaryotypicSex(IndividualProperty.KaryotypicSex.UNKNOWN).setLifeStatus(IndividualProperty.LifeStatus.UNKNOWN), - INCLUDE_RESULT, token).first().getId(); + INCLUDE_RESULT, ownerToken).first().getId(); catalogManager.getIndividualManager().update(studyFqn, individualId1, new IndividualUpdateParams() .setAnnotationSets(Collections.singletonList(new AnnotationSet("annot1", variableSet.getId(), new ObjectMap("NAME", "INDIVIDUAL_1").append("AGE", 5).append("PHEN", "CASE").append("ALIVE", true)))), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); catalogManager.getIndividualManager().update(studyFqn, individualId2, new IndividualUpdateParams() .setAnnotationSets(Collections.singletonList(new AnnotationSet("annot1", variableSet.getId(), new ObjectMap("NAME", "INDIVIDUAL_2").append("AGE", 15).append("PHEN", "CONTROL").append("ALIVE", true)))), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); catalogManager.getIndividualManager().update(studyFqn, individualId3, new IndividualUpdateParams() .setAnnotationSets(Collections.singletonList(new AnnotationSet("annot1", variableSet.getId(), new ObjectMap("NAME", "INDIVIDUAL_3").append("AGE", 25).append("PHEN", "CASE").append("ALIVE", true)))), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); List individuals; individuals = catalogManager.getIndividualManager().search(studyFqn, new Query(IndividualDBAdaptor.QueryParams.ANNOTATION.key(), - variableSet.getId() + ":NAME=~^INDIVIDUAL_"), null, token) + variableSet.getId() + ":NAME=~^INDIVIDUAL_"), null, ownerToken) .getResults().stream().map(Individual::getName).collect(Collectors.toList()); assertTrue(individuals.containsAll(Arrays.asList("INDIVIDUAL_1", "INDIVIDUAL_2", "INDIVIDUAL_3"))); individuals = catalogManager.getIndividualManager().search(studyFqn, new Query(IndividualDBAdaptor.QueryParams.ANNOTATION.key(), - variableSet.getId() + ":AGE>10"), null, token) + variableSet.getId() + ":AGE>10"), null, ownerToken) .getResults().stream().map(Individual::getName).collect(Collectors.toList()); assertTrue(individuals.containsAll(Arrays.asList("INDIVIDUAL_2", "INDIVIDUAL_3"))); individuals = catalogManager.getIndividualManager().search(studyFqn, new Query(IndividualDBAdaptor.QueryParams.ANNOTATION.key(), - variableSet.getId() + ":AGE>10;" + variableSet.getId() + ":PHEN=CASE"), null, token) + variableSet.getId() + ":AGE>10;" + variableSet.getId() + ":PHEN=CASE"), null, ownerToken) .getResults().stream().map(Individual::getName).collect(Collectors.toList()); assertTrue(individuals.containsAll(Arrays.asList("INDIVIDUAL_3"))); } @@ -93,34 +93,34 @@ public void testDistinctDisorders() throws CatalogException { Individual individual = new Individual() .setId("i1") .setDisorders(Collections.singletonList(new Disorder().setId("disorder1").setName(null))); - catalogManager.getIndividualManager().create(studyFqn, individual, null, token); + catalogManager.getIndividualManager().create(studyFqn, individual, null, ownerToken); individual = new Individual() .setId("i2") .setDisorders(Collections.singletonList(new Disorder().setId("disorder2"))); - catalogManager.getIndividualManager().create(studyFqn, individual, null, token); + catalogManager.getIndividualManager().create(studyFqn, individual, null, ownerToken); individual = new Individual() .setId("i3") .setDisorders(Collections.singletonList(new Disorder().setId("disorder2"))); - catalogManager.getIndividualManager().create(studyFqn, individual, null, token); + catalogManager.getIndividualManager().create(studyFqn, individual, null, ownerToken); individual = new Individual() .setId("i4") .setDisorders(Collections.singletonList(new Disorder().setId("adisorder2"))); - catalogManager.getIndividualManager().create(studyFqn, individual, null, token); + catalogManager.getIndividualManager().create(studyFqn, individual, null, ownerToken); - OpenCGAResult result = catalogManager.getIndividualManager().distinct(studyFqn, - IndividualDBAdaptor.QueryParams.DISORDERS_ID.key(), new Query(), token); + OpenCGAResult result = catalogManager.getIndividualManager().distinct(organizationId, studyFqn, + IndividualDBAdaptor.QueryParams.DISORDERS_ID.key(), new Query(), ownerToken); assertEquals(3, result.getNumResults()); - result = catalogManager.getIndividualManager().distinct(studyFqn, IndividualDBAdaptor.QueryParams.DISORDERS_ID.key(), - new Query(IndividualDBAdaptor.QueryParams.DISORDERS.key(), "~^disor"), token); + result = catalogManager.getIndividualManager().distinct(organizationId, studyFqn, IndividualDBAdaptor.QueryParams.DISORDERS_ID.key(), + new Query(IndividualDBAdaptor.QueryParams.DISORDERS.key(), "~^disor"), ownerToken); assertEquals(2, result.getNumResults()); result = catalogManager.getIndividualManager().distinct(studyFqn, Arrays.asList(IndividualDBAdaptor.QueryParams.DISORDERS_ID.key(), IndividualDBAdaptor.QueryParams.DISORDERS_NAME.key()), - new Query(), token); + new Query(), ownerToken); assertEquals(4, result.getNumResults()); assertFalse(result.getResults().contains(null)); } @@ -130,25 +130,25 @@ public void testSearchDisordersWithCommas() throws CatalogException { Individual individual = new Individual() .setId("i1") .setDisorders(Collections.singletonList(new Disorder().setId("disorder1, description 1"))); - catalogManager.getIndividualManager().create(studyFqn, individual, null, token); + catalogManager.getIndividualManager().create(studyFqn, individual, null, ownerToken); individual = new Individual() .setId("i2") .setDisorders(Collections.singletonList(new Disorder().setId("disorder2, description 2"))); - catalogManager.getIndividualManager().create(studyFqn, individual, null, token); + catalogManager.getIndividualManager().create(studyFqn, individual, null, ownerToken); individual = new Individual() .setId("i3") .setDisorders(Collections.singletonList(new Disorder().setId("disorder2"))); - catalogManager.getIndividualManager().create(studyFqn, individual, null, token); + catalogManager.getIndividualManager().create(studyFqn, individual, null, ownerToken); - OpenCGAResult result = catalogManager.getIndividualManager().distinct(studyFqn, - IndividualDBAdaptor.QueryParams.DISORDERS_ID.key(), new Query(), token); + OpenCGAResult result = catalogManager.getIndividualManager().distinct(organizationId, studyFqn, + IndividualDBAdaptor.QueryParams.DISORDERS_ID.key(), new Query(), ownerToken); assertEquals(3, result.getNumResults()); OpenCGAResult search = catalogManager.getIndividualManager().search(studyFqn, new Query(IndividualDBAdaptor.QueryParams.DISORDERS.key(), "disorder1, description 1,disorder2, description 2"), - IndividualManager.INCLUDE_INDIVIDUAL_IDS, token); + IndividualManager.INCLUDE_INDIVIDUAL_IDS, ownerToken); assertEquals(2, search.getNumResults()); assertTrue(Arrays.asList("i1", "i2").containsAll(search.getResults().stream().map(Individual::getId).collect(Collectors.toList()))); } @@ -156,7 +156,7 @@ public void testSearchDisordersWithCommas() throws CatalogException { @Test public void testUpdatePhenotypes() throws CatalogException { Individual individual = new Individual().setId("i1"); - catalogManager.getIndividualManager().create(studyFqn, individual, null, token); + catalogManager.getIndividualManager().create(studyFqn, individual, null, ownerToken); List phenotypeList = Arrays.asList( new Phenotype("phenotype0", "phenotypeName0", "SOURCE"), @@ -165,8 +165,8 @@ public void testUpdatePhenotypes() throws CatalogException { ); IndividualUpdateParams updateParams = new IndividualUpdateParams().setPhenotypes(phenotypeList); - catalogManager.getIndividualManager().update(studyFqn, individual.getId(), updateParams, QueryOptions.empty(), token); - individual = catalogManager.getIndividualManager().get(studyFqn, individual.getId(), QueryOptions.empty(), token).first(); + catalogManager.getIndividualManager().update(studyFqn, individual.getId(), updateParams, QueryOptions.empty(), ownerToken); + individual = catalogManager.getIndividualManager().get(studyFqn, individual.getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(3, individual.getPhenotypes().size()); for (int i = 0; i < individual.getPhenotypes().size(); i++) { assertEquals("phenotype" + i, individual.getPhenotypes().get(i).getId()); @@ -178,8 +178,8 @@ public void testUpdatePhenotypes() throws CatalogException { QueryOptions options = new QueryOptions(Constants.ACTIONS, actionMap); updateParams = new IndividualUpdateParams().setPhenotypes(Arrays.asList( new Phenotype("phenotype0", "phenotypeName0", "SOURCE"), new Phenotype("phenotype2", "phenotypeName2", "SOURCE"))); - catalogManager.getIndividualManager().update(studyFqn, individual.getId(), updateParams, options, token); - individual = catalogManager.getIndividualManager().get(studyFqn, individual.getId(), QueryOptions.empty(), token).first(); + catalogManager.getIndividualManager().update(studyFqn, individual.getId(), updateParams, options, ownerToken); + individual = catalogManager.getIndividualManager().get(studyFqn, individual.getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(1, individual.getPhenotypes().size()); assertEquals("phenotype1", individual.getPhenotypes().get(0).getId()); @@ -188,8 +188,8 @@ public void testUpdatePhenotypes() throws CatalogException { options = new QueryOptions(Constants.ACTIONS, actionMap); updateParams = new IndividualUpdateParams().setPhenotypes(Arrays.asList( new Phenotype("phenotype1", "phenotypeName1", "SOURCE"), new Phenotype("phenotype2", "phenotypeName2", "SOURCE"))); - catalogManager.getIndividualManager().update(studyFqn, individual.getId(), updateParams, options, token); - individual = catalogManager.getIndividualManager().get(studyFqn, individual.getId(), QueryOptions.empty(), token).first(); + catalogManager.getIndividualManager().update(studyFqn, individual.getId(), updateParams, options, ownerToken); + individual = catalogManager.getIndividualManager().get(studyFqn, individual.getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(2, individual.getPhenotypes().size()); for (int i = 0; i < individual.getPhenotypes().size(); i++) { assertEquals("phenotype" + (i + 1), individual.getPhenotypes().get(i).getId()); @@ -203,8 +203,8 @@ public void testUpdatePhenotypes() throws CatalogException { new Phenotype("phenotype3", "phenotypeName3", "SOURCE") ); updateParams = new IndividualUpdateParams().setPhenotypes(phenotypeList); - catalogManager.getIndividualManager().update(studyFqn, individual.getId(), updateParams, options, token); - individual = catalogManager.getIndividualManager().get(studyFqn, individual.getId(), QueryOptions.empty(), token).first(); + catalogManager.getIndividualManager().update(studyFqn, individual.getId(), updateParams, options, ownerToken); + individual = catalogManager.getIndividualManager().get(studyFqn, individual.getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(2, individual.getPhenotypes().size()); for (int i = 0; i < individual.getPhenotypes().size(); i++) { assertEquals("phenotype" + (i + 2), individual.getPhenotypes().get(i).getId()); @@ -214,7 +214,7 @@ public void testUpdatePhenotypes() throws CatalogException { @Test public void testUpdateDisorders() throws CatalogException { Individual individual = new Individual().setId("i1"); - catalogManager.getIndividualManager().create(studyFqn, individual, null, token); + catalogManager.getIndividualManager().create(studyFqn, individual, null, ownerToken); List disorderList = Arrays.asList( new Disorder("disorder0", "disorderName0", "SOURCE", null, "", null), @@ -223,8 +223,8 @@ public void testUpdateDisorders() throws CatalogException { ); IndividualUpdateParams updateParams = new IndividualUpdateParams().setDisorders(disorderList); - catalogManager.getIndividualManager().update(studyFqn, individual.getId(), updateParams, QueryOptions.empty(), token); - individual = catalogManager.getIndividualManager().get(studyFqn, individual.getId(), QueryOptions.empty(), token).first(); + catalogManager.getIndividualManager().update(studyFqn, individual.getId(), updateParams, QueryOptions.empty(), ownerToken); + individual = catalogManager.getIndividualManager().get(studyFqn, individual.getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(3, individual.getDisorders().size()); for (int i = 0; i < individual.getDisorders().size(); i++) { assertEquals("disorder" + i, individual.getDisorders().get(i).getId()); @@ -237,8 +237,8 @@ public void testUpdateDisorders() throws CatalogException { updateParams = new IndividualUpdateParams().setDisorders(Arrays.asList( new Disorder("disorder0", "disorderName0", "SOURCE", null, "", null), new Disorder("disorder2", "disorder2", "SOURCE", null, "", null))); - catalogManager.getIndividualManager().update(studyFqn, individual.getId(), updateParams, options, token); - individual = catalogManager.getIndividualManager().get(studyFqn, individual.getId(), QueryOptions.empty(), token).first(); + catalogManager.getIndividualManager().update(studyFqn, individual.getId(), updateParams, options, ownerToken); + individual = catalogManager.getIndividualManager().get(studyFqn, individual.getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(1, individual.getDisorders().size()); assertEquals("disorder1", individual.getDisorders().get(0).getId()); @@ -248,8 +248,8 @@ public void testUpdateDisorders() throws CatalogException { updateParams = new IndividualUpdateParams().setDisorders(Arrays.asList( new Disorder("disorder1", "disorderName1", "SOURCE", null, "", null), new Disorder("disorder2", "disorderName2", "SOURCE", null, "", null))); - catalogManager.getIndividualManager().update(studyFqn, individual.getId(), updateParams, options, token); - individual = catalogManager.getIndividualManager().get(studyFqn, individual.getId(), QueryOptions.empty(), token).first(); + catalogManager.getIndividualManager().update(studyFqn, individual.getId(), updateParams, options, ownerToken); + individual = catalogManager.getIndividualManager().get(studyFqn, individual.getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(2, individual.getDisorders().size()); for (int i = 0; i < individual.getDisorders().size(); i++) { assertEquals("disorder" + (i + 1), individual.getDisorders().get(i).getId()); @@ -263,8 +263,8 @@ public void testUpdateDisorders() throws CatalogException { new Disorder("disorder3", "disorderName3", "SOURCE", null, "", null) ); updateParams = new IndividualUpdateParams().setDisorders(disorderList); - catalogManager.getIndividualManager().update(studyFqn, individual.getId(), updateParams, options, token); - individual = catalogManager.getIndividualManager().get(studyFqn, individual.getId(), QueryOptions.empty(), token).first(); + catalogManager.getIndividualManager().update(studyFqn, individual.getId(), updateParams, options, ownerToken); + individual = catalogManager.getIndividualManager().get(studyFqn, individual.getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(2, individual.getDisorders().size()); for (int i = 0; i < individual.getDisorders().size(); i++) { assertEquals("disorder" + (i + 2), individual.getDisorders().get(i).getId()); @@ -276,19 +276,19 @@ public void testUpdateIndividualSamples() throws CatalogException { Sample sample = new Sample().setId("sample1"); Sample sample2 = new Sample().setId("sample2"); Sample sample3 = new Sample().setId("sample3"); - catalogManager.getSampleManager().create(studyFqn, sample, QueryOptions.empty(), token); - catalogManager.getSampleManager().create(studyFqn, sample2, QueryOptions.empty(), token); - catalogManager.getSampleManager().create(studyFqn, sample3, QueryOptions.empty(), token); + catalogManager.getSampleManager().create(studyFqn, sample, QueryOptions.empty(), ownerToken); + catalogManager.getSampleManager().create(studyFqn, sample2, QueryOptions.empty(), ownerToken); + catalogManager.getSampleManager().create(studyFqn, sample3, QueryOptions.empty(), ownerToken); Individual individual = catalogManager.getIndividualManager().create(studyFqn, new Individual().setId("individual"), - Arrays.asList(sample.getId(), sample2.getId()), INCLUDE_RESULT, token).first(); + Arrays.asList(sample.getId(), sample2.getId()), INCLUDE_RESULT, ownerToken).first(); assertEquals(2, individual.getSamples().size()); assertEquals(2, individual.getSamples().stream().map(Sample::getId) .filter(s -> Arrays.asList(sample.getId(), sample2.getId()).contains(s)).count()); // Increase sample2 version catalogManager.getSampleManager().update(studyFqn, sample2.getId(), new SampleUpdateParams().setDescription("new description"), - new QueryOptions(), token); + new QueryOptions(), ownerToken); // Add sample2 (with new version) and sample3 Map actionMap = new HashMap<>(); @@ -298,10 +298,10 @@ public void testUpdateIndividualSamples() throws CatalogException { IndividualUpdateParams updateParams = new IndividualUpdateParams().setSamples(Arrays.asList( new SampleReferenceParam().setId(sample2.getId()), new SampleReferenceParam().setId(sample3.getId()))); OpenCGAResult update = catalogManager.getIndividualManager().update(studyFqn, individual.getId(), updateParams, - options, token); + options, ownerToken); assertEquals(1, update.getNumUpdated()); - individual = catalogManager.getIndividualManager().get(studyFqn, individual.getId(), QueryOptions.empty(), token).first(); + individual = catalogManager.getIndividualManager().get(studyFqn, individual.getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(3, individual.getSamples().size()); assertEquals(3, individual.getSamples().stream().map(Sample::getId) .filter(s -> Arrays.asList(sample.getId(), sample2.getId(), sample3.getId()).contains(s)).count()); @@ -318,10 +318,10 @@ public void testUpdateIndividualSamples() throws CatalogException { updateParams = new IndividualUpdateParams().setSamples(Collections.singletonList(new SampleReferenceParam().setId(sample2.getId()))); - update = catalogManager.getIndividualManager().update(studyFqn, individual.getId(), updateParams, options, token); + update = catalogManager.getIndividualManager().update(studyFqn, individual.getId(), updateParams, options, ownerToken); assertEquals(1, update.getNumUpdated()); - individual = catalogManager.getIndividualManager().get(studyFqn, individual.getId(), QueryOptions.empty(), token).first(); + individual = catalogManager.getIndividualManager().get(studyFqn, individual.getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(2, individual.getSamples().size()); assertEquals(2, individual.getSamples().stream().map(Sample::getId) .filter(s -> Arrays.asList(sample.getId(), sample3.getId()).contains(s)).count()); @@ -332,10 +332,10 @@ public void testUpdateIndividualSamples() throws CatalogException { updateParams = new IndividualUpdateParams().setSamples(Arrays.asList( new SampleReferenceParam().setId(sample.getId()), new SampleReferenceParam().setId(sample2.getId()))); - update = catalogManager.getIndividualManager().update(studyFqn, individual.getId(), updateParams, options, token); + update = catalogManager.getIndividualManager().update(studyFqn, individual.getId(), updateParams, options, ownerToken); assertEquals(1, update.getNumUpdated()); - individual = catalogManager.getIndividualManager().get(studyFqn, individual.getId(), QueryOptions.empty(), token).first(); + individual = catalogManager.getIndividualManager().get(studyFqn, individual.getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(2, individual.getSamples().size()); assertEquals(2, individual.getSamples().stream().map(Sample::getId) .filter(s -> Arrays.asList(sample.getId(), sample2.getId()).contains(s)).count()); @@ -346,13 +346,13 @@ public void changeIndividualIdTest() throws CatalogException { Sample sample1 = new Sample().setId("sample1"); Sample sample2 = new Sample().setId("sample2"); Sample sample3 = new Sample().setId("sample3"); - catalogManager.getSampleManager().create(studyFqn, sample1, QueryOptions.empty(), token); - catalogManager.getSampleManager().create(studyFqn, sample2, QueryOptions.empty(), token); - catalogManager.getSampleManager().create(studyFqn, sample3, QueryOptions.empty(), token); + catalogManager.getSampleManager().create(studyFqn, sample1, QueryOptions.empty(), ownerToken); + catalogManager.getSampleManager().create(studyFqn, sample2, QueryOptions.empty(), ownerToken); + catalogManager.getSampleManager().create(studyFqn, sample3, QueryOptions.empty(), ownerToken); catalogManager.getIndividualManager().create(studyFqn, new Individual().setId("individual1"), - Arrays.asList(sample1.getId(), sample2.getId(), sample3.getId()), QueryOptions.empty(), token); + Arrays.asList(sample1.getId(), sample2.getId(), sample3.getId()), QueryOptions.empty(), ownerToken); List samples = catalogManager.getSampleManager().get(studyFqn, - Arrays.asList(sample1.getId(), sample2.getId(), sample3.getId()), QueryOptions.empty(), token).getResults(); + Arrays.asList(sample1.getId(), sample2.getId(), sample3.getId()), QueryOptions.empty(), ownerToken).getResults(); for (Sample sample : samples) { assertEquals("individual1", sample.getIndividualId()); } @@ -360,30 +360,30 @@ public void changeIndividualIdTest() throws CatalogException { Sample sample4 = new Sample().setId("sample4"); Sample sample5 = new Sample().setId("sample5"); Sample sample6 = new Sample().setId("sample6"); - catalogManager.getSampleManager().create(studyFqn, sample4, QueryOptions.empty(), token); - catalogManager.getSampleManager().create(studyFqn, sample5, QueryOptions.empty(), token); - catalogManager.getSampleManager().create(studyFqn, sample6, QueryOptions.empty(), token); + catalogManager.getSampleManager().create(studyFqn, sample4, QueryOptions.empty(), ownerToken); + catalogManager.getSampleManager().create(studyFqn, sample5, QueryOptions.empty(), ownerToken); + catalogManager.getSampleManager().create(studyFqn, sample6, QueryOptions.empty(), ownerToken); catalogManager.getIndividualManager().create(studyFqn, new Individual().setId("individual2"), - Arrays.asList(sample4.getId(), sample5.getId(), sample6.getId()), QueryOptions.empty(), token); + Arrays.asList(sample4.getId(), sample5.getId(), sample6.getId()), QueryOptions.empty(), ownerToken); samples = catalogManager.getSampleManager().get(studyFqn, - Arrays.asList(sample4.getId(), sample5.getId(), sample6.getId()), QueryOptions.empty(), token).getResults(); + Arrays.asList(sample4.getId(), sample5.getId(), sample6.getId()), QueryOptions.empty(), ownerToken).getResults(); for (Sample sample : samples) { assertEquals("individual2", sample.getIndividualId()); } // Update individual id catalogManager.getIndividualManager().update(studyFqn, "individual1", new IndividualUpdateParams().setId("newId1"), - QueryOptions.empty(), token); - assertEquals(1, catalogManager.getIndividualManager().get(studyFqn, "newId1", QueryOptions.empty(), token).getNumResults()); + QueryOptions.empty(), ownerToken); + assertEquals(1, catalogManager.getIndividualManager().get(studyFqn, "newId1", QueryOptions.empty(), ownerToken).getNumResults()); catalogManager.getIndividualManager().update(studyFqn, "individual2", new IndividualUpdateParams().setId("newId2"), - QueryOptions.empty(), token); - assertEquals(1, catalogManager.getIndividualManager().get(studyFqn, "newId2", QueryOptions.empty(), token).getNumResults()); + QueryOptions.empty(), ownerToken); + assertEquals(1, catalogManager.getIndividualManager().get(studyFqn, "newId2", QueryOptions.empty(), ownerToken).getNumResults()); samples = catalogManager.getSampleManager().get(studyFqn, Arrays.asList(sample1.getId(), sample2.getId(), sample3.getId(), sample4.getId(), sample5.getId(), sample6.getId()), - QueryOptions.empty(), token).getResults(); + QueryOptions.empty(), ownerToken).getResults(); for (Sample sample : samples) { switch (sample.getId()) { case "sample1": @@ -404,15 +404,15 @@ public void changeIndividualIdTest() throws CatalogException { @Test public void changeIndividualIdTest2() throws CatalogException { - catalogManager.getIndividualManager().create(studyFqn, new Individual().setId("father"), null, QueryOptions.empty(), token); - catalogManager.getIndividualManager().create(studyFqn, new Individual().setId("mother"), null, QueryOptions.empty(), token); + catalogManager.getIndividualManager().create(studyFqn, new Individual().setId("father"), null, QueryOptions.empty(), ownerToken); + catalogManager.getIndividualManager().create(studyFqn, new Individual().setId("mother"), null, QueryOptions.empty(), ownerToken); catalogManager.getIndividualManager().create(studyFqn, new Individual() .setId("child") .setFather(new Individual().setId("father")) .setMother(new Individual().setId("mother")), - null, QueryOptions.empty(), token); + null, QueryOptions.empty(), ownerToken); Family family = catalogManager.getFamilyManager().create(studyFqn, new Family().setId("family"), - Arrays.asList("child", "mother", "father"), INCLUDE_RESULT, token).first(); + Arrays.asList("child", "mother", "father"), INCLUDE_RESULT, ownerToken).first(); assertNotNull(family.getRoles()); assertFalse(family.getRoles().isEmpty()); @@ -426,8 +426,8 @@ public void changeIndividualIdTest2() throws CatalogException { // Update child's id catalogManager.getIndividualManager().update(studyFqn, "child", new IndividualUpdateParams().setId("newId1"), - QueryOptions.empty(), token); - family = catalogManager.getFamilyManager().get(studyFqn, "family", QueryOptions.empty(), token).first(); + QueryOptions.empty(), ownerToken); + family = catalogManager.getFamilyManager().get(studyFqn, "family", QueryOptions.empty(), ownerToken).first(); assertTrue(family.getRoles().containsKey("newId1")); for (Map.Entry> entry : family.getRoles().entrySet()) { @@ -440,32 +440,32 @@ public void changeIndividualIdTest2() throws CatalogException { @Test public void testUpdateWithLockedClinicalAnalysis() throws CatalogException { Sample sample = new Sample().setId("sample1"); - catalogManager.getSampleManager().create(studyFqn, sample, QueryOptions.empty(), token); + catalogManager.getSampleManager().create(studyFqn, sample, QueryOptions.empty(), ownerToken); sample = new Sample().setId("sample2"); - catalogManager.getSampleManager().create(studyFqn, sample, QueryOptions.empty(), token); + catalogManager.getSampleManager().create(studyFqn, sample, QueryOptions.empty(), ownerToken); sample = new Sample().setId("sample3"); - catalogManager.getSampleManager().create(studyFqn, sample, QueryOptions.empty(), token); + catalogManager.getSampleManager().create(studyFqn, sample, QueryOptions.empty(), ownerToken); sample = new Sample().setId("sample4"); - catalogManager.getSampleManager().create(studyFqn, sample, QueryOptions.empty(), token); + catalogManager.getSampleManager().create(studyFqn, sample, QueryOptions.empty(), ownerToken); Individual individual = new Individual() .setId("proband") .setDisorders(Collections.singletonList(new Disorder().setId("disorder"))); catalogManager.getIndividualManager().create(studyFqn, individual, Arrays.asList("sample1", "sample2"), QueryOptions.empty(), - token); + ownerToken); individual = new Individual().setId("father"); - catalogManager.getIndividualManager().create(studyFqn, individual, Arrays.asList("sample3"), QueryOptions.empty(), token); + catalogManager.getIndividualManager().create(studyFqn, individual, Arrays.asList("sample3"), QueryOptions.empty(), ownerToken); individual = new Individual().setId("brother"); - catalogManager.getIndividualManager().create(studyFqn, individual, Arrays.asList("sample4"), QueryOptions.empty(), token); + catalogManager.getIndividualManager().create(studyFqn, individual, Arrays.asList("sample4"), QueryOptions.empty(), ownerToken); Family family = new Family().setId("family"); catalogManager.getFamilyManager().create(studyFqn, family, Arrays.asList("proband", "father", "brother"), QueryOptions.empty(), - token); + ownerToken); family.setMembers(Arrays.asList( new Individual().setId("proband").setSamples(Collections.singletonList(new Sample().setId("sample2"))), @@ -477,7 +477,7 @@ public void testUpdateWithLockedClinicalAnalysis() throws CatalogException { .setProband(new Individual().setId("proband")) .setFamily(family) .setType(ClinicalAnalysis.Type.FAMILY); - catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, QueryOptions.empty(), token); + catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, QueryOptions.empty(), ownerToken); // We will create another clinical analysis with the same information. In this test, we will not lock clinical2 clinicalAnalysis = new ClinicalAnalysis() @@ -485,16 +485,16 @@ public void testUpdateWithLockedClinicalAnalysis() throws CatalogException { .setProband(new Individual().setId("proband")) .setFamily(family) .setType(ClinicalAnalysis.Type.FAMILY); - catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, QueryOptions.empty(), token); + catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, QueryOptions.empty(), ownerToken); // Update brother not used in Clinical Analysis catalogManager.getIndividualManager().update(studyFqn, "brother", new IndividualUpdateParams() - .setName(RandomStringUtils.random(10)), new QueryOptions(), token); + .setName(RandomStringUtils.random(10)), new QueryOptions(), ownerToken); - Individual individualResult = catalogManager.getIndividualManager().get(studyFqn, "brother", QueryOptions.empty(), token).first(); + Individual individualResult = catalogManager.getIndividualManager().get(studyFqn, "brother", QueryOptions.empty(), ownerToken).first(); assertEquals(3, individualResult.getVersion()); - Family familyResult = catalogManager.getFamilyManager().get(studyFqn, "family", QueryOptions.empty(), token).first(); + Family familyResult = catalogManager.getFamilyManager().get(studyFqn, "family", QueryOptions.empty(), ownerToken).first(); assertEquals(2, familyResult.getVersion()); assertEquals(3, familyResult.getMembers().size()); assertEquals(2, familyResult.getMembers().get(0).getVersion()); @@ -502,7 +502,7 @@ public void testUpdateWithLockedClinicalAnalysis() throws CatalogException { assertEquals(3, familyResult.getMembers().get(2).getVersion()); ClinicalAnalysis clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, "clinical", QueryOptions.empty(), - token).first(); + ownerToken).first(); assertEquals(2, clinicalResult.getProband().getVersion()); assertEquals(2, clinicalResult.getProband().getSamples().get(0).getVersion()); // sample1 version assertEquals(2, clinicalResult.getFamily().getVersion()); @@ -510,7 +510,7 @@ public void testUpdateWithLockedClinicalAnalysis() throws CatalogException { assertEquals(2, clinicalResult.getFamily().getMembers().get(0).getVersion()); // proband version assertEquals(2, clinicalResult.getFamily().getMembers().get(1).getVersion()); // father version - clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, "clinical2", QueryOptions.empty(), token).first(); + clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, "clinical2", QueryOptions.empty(), ownerToken).first(); assertEquals(2, clinicalResult.getProband().getVersion()); assertEquals(2, clinicalResult.getProband().getSamples().get(0).getVersion()); // sample1 version assertEquals(2, clinicalResult.getFamily().getVersion()); @@ -520,19 +520,19 @@ public void testUpdateWithLockedClinicalAnalysis() throws CatalogException { // Update father catalogManager.getIndividualManager().update(studyFqn, "father", new IndividualUpdateParams() - .setName(RandomStringUtils.random(10)), new QueryOptions(), token); + .setName(RandomStringUtils.random(10)), new QueryOptions(), ownerToken); - individualResult = catalogManager.getIndividualManager().get(studyFqn, "father", QueryOptions.empty(), token).first(); + individualResult = catalogManager.getIndividualManager().get(studyFqn, "father", QueryOptions.empty(), ownerToken).first(); assertEquals(3, individualResult.getVersion()); - familyResult = catalogManager.getFamilyManager().get(studyFqn, "family", QueryOptions.empty(), token).first(); + familyResult = catalogManager.getFamilyManager().get(studyFqn, "family", QueryOptions.empty(), ownerToken).first(); assertEquals(3, familyResult.getVersion()); assertEquals(3, familyResult.getMembers().size()); assertEquals(2, familyResult.getMembers().get(0).getVersion()); assertEquals(3, familyResult.getMembers().get(1).getVersion()); assertEquals(3, familyResult.getMembers().get(2).getVersion()); - clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, "clinical", QueryOptions.empty(), token).first(); + clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, "clinical", QueryOptions.empty(), ownerToken).first(); assertEquals(2, clinicalResult.getProband().getVersion()); assertEquals(2, clinicalResult.getProband().getSamples().get(0).getVersion()); // sample1 version assertEquals(3, clinicalResult.getFamily().getVersion()); @@ -540,7 +540,7 @@ public void testUpdateWithLockedClinicalAnalysis() throws CatalogException { assertEquals(2, clinicalResult.getFamily().getMembers().get(0).getVersion()); // proband version assertEquals(3, clinicalResult.getFamily().getMembers().get(1).getVersion()); // father version - clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, "clinical2", QueryOptions.empty(), token).first(); + clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, "clinical2", QueryOptions.empty(), ownerToken).first(); assertEquals(2, clinicalResult.getProband().getVersion()); assertEquals(2, clinicalResult.getProband().getSamples().get(0).getVersion()); // sample1 version assertEquals(3, clinicalResult.getFamily().getVersion()); @@ -550,25 +550,25 @@ public void testUpdateWithLockedClinicalAnalysis() throws CatalogException { // LOCK CLINICAL ANALYSIS catalogManager.getClinicalAnalysisManager().update(studyFqn, "clinical", new ClinicalAnalysisUpdateParams().setLocked(true), - QueryOptions.empty(), token); - clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, "clinical", QueryOptions.empty(), token).first(); + QueryOptions.empty(), ownerToken); + clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, "clinical", QueryOptions.empty(), ownerToken).first(); assertTrue(clinicalResult.isLocked()); // Update proband catalogManager.getIndividualManager().update(studyFqn, "proband", new IndividualUpdateParams() - .setName(RandomStringUtils.random(10)), new QueryOptions(), token); + .setName(RandomStringUtils.random(10)), new QueryOptions(), ownerToken); - individualResult = catalogManager.getIndividualManager().get(studyFqn, "proband", QueryOptions.empty(), token).first(); + individualResult = catalogManager.getIndividualManager().get(studyFqn, "proband", QueryOptions.empty(), ownerToken).first(); assertEquals(3, individualResult.getVersion()); - familyResult = catalogManager.getFamilyManager().get(studyFqn, "family", QueryOptions.empty(), token).first(); + familyResult = catalogManager.getFamilyManager().get(studyFqn, "family", QueryOptions.empty(), ownerToken).first(); assertEquals(4, familyResult.getVersion()); assertEquals(3, familyResult.getMembers().size()); assertEquals(3, familyResult.getMembers().get(0).getVersion()); assertEquals(3, familyResult.getMembers().get(1).getVersion()); assertEquals(3, familyResult.getMembers().get(2).getVersion()); - clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, "clinical", QueryOptions.empty(), token).first(); + clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, "clinical", QueryOptions.empty(), ownerToken).first(); assertEquals(2, clinicalResult.getProband().getVersion()); assertEquals(2, clinicalResult.getProband().getSamples().get(0).getVersion()); // sample1 version assertEquals(3, clinicalResult.getFamily().getVersion()); @@ -576,7 +576,7 @@ public void testUpdateWithLockedClinicalAnalysis() throws CatalogException { assertEquals(2, clinicalResult.getFamily().getMembers().get(0).getVersion()); // proband version assertEquals(3, clinicalResult.getFamily().getMembers().get(1).getVersion()); // father version - clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, "clinical2", QueryOptions.empty(), token).first(); + clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, "clinical2", QueryOptions.empty(), ownerToken).first(); assertEquals(3, clinicalResult.getProband().getVersion()); assertEquals(2, clinicalResult.getProband().getSamples().get(0).getVersion()); // sample2 version assertEquals(4, clinicalResult.getFamily().getVersion()); @@ -586,19 +586,19 @@ public void testUpdateWithLockedClinicalAnalysis() throws CatalogException { // Update father catalogManager.getIndividualManager().update(studyFqn, "father", new IndividualUpdateParams() - .setName(RandomStringUtils.random(10)), new QueryOptions(), token); + .setName(RandomStringUtils.random(10)), new QueryOptions(), ownerToken); - individualResult = catalogManager.getIndividualManager().get(studyFqn, "father", QueryOptions.empty(), token).first(); + individualResult = catalogManager.getIndividualManager().get(studyFqn, "father", QueryOptions.empty(), ownerToken).first(); assertEquals(4, individualResult.getVersion()); - familyResult = catalogManager.getFamilyManager().get(studyFqn, "family", QueryOptions.empty(), token).first(); + familyResult = catalogManager.getFamilyManager().get(studyFqn, "family", QueryOptions.empty(), ownerToken).first(); assertEquals(5, familyResult.getVersion()); assertEquals(3, familyResult.getMembers().size()); assertEquals(3, familyResult.getMembers().get(0).getVersion()); assertEquals(4, familyResult.getMembers().get(1).getVersion()); assertEquals(3, familyResult.getMembers().get(2).getVersion()); - clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, "clinical", QueryOptions.empty(), token).first(); + clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, "clinical", QueryOptions.empty(), ownerToken).first(); assertEquals(2, clinicalResult.getProband().getVersion()); assertEquals(2, clinicalResult.getProband().getSamples().get(0).getVersion()); // sample1 version assertEquals(3, clinicalResult.getFamily().getVersion()); @@ -606,7 +606,7 @@ public void testUpdateWithLockedClinicalAnalysis() throws CatalogException { assertEquals(2, clinicalResult.getFamily().getMembers().get(0).getVersion()); // proband version assertEquals(3, clinicalResult.getFamily().getMembers().get(1).getVersion()); // father version - clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, "clinical2", QueryOptions.empty(), token).first(); + clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, "clinical2", QueryOptions.empty(), ownerToken).first(); assertEquals(3, clinicalResult.getProband().getVersion()); assertEquals(2, clinicalResult.getProband().getSamples().get(0).getVersion()); // sample2 version assertEquals(5, clinicalResult.getFamily().getVersion()); @@ -619,15 +619,15 @@ public void testUpdateWithLockedClinicalAnalysis() throws CatalogException { public void testUpdateIndividualQualityControl() throws CatalogException { IndividualManager individualManager = catalogManager.getIndividualManager(); DataResult individualDataResult = individualManager.create(studyFqn, new Individual().setId("Test") - .setDateOfBirth("19870214"), INCLUDE_RESULT, token); + .setDateOfBirth("19870214"), INCLUDE_RESULT, ownerToken); IndividualQualityControl qualityControl = new IndividualQualityControl(null, null, null, Collections.emptyList(), Arrays.asList(new ClinicalComment("pfurio", "message", Collections.singletonList("tag"), "today"))); DataResult update = individualManager.update(studyFqn, individualDataResult.first().getId(), - new IndividualUpdateParams().setQualityControl(qualityControl), QueryOptions.empty(), token); + new IndividualUpdateParams().setQualityControl(qualityControl), QueryOptions.empty(), ownerToken); assertEquals(1, update.getNumUpdated()); - Individual individual = individualManager.get(studyFqn, individualDataResult.first().getId(), QueryOptions.empty(), token) + Individual individual = individualManager.get(studyFqn, individualDataResult.first().getId(), QueryOptions.empty(), ownerToken) .first(); assertEquals(1, individual.getQualityControl().getComments().size()); assertEquals("pfurio", individual.getQualityControl().getComments().get(0).getAuthor()); @@ -640,37 +640,37 @@ public void testUpdateIndividualQualityControl() throws CatalogException { public void testUpdateIndividualInfo() throws CatalogException { IndividualManager individualManager = catalogManager.getIndividualManager(); DataResult individualDataResult = individualManager.create(studyFqn, new Individual().setId("Test") - .setDateOfBirth("19870214"), INCLUDE_RESULT, token); + .setDateOfBirth("19870214"), INCLUDE_RESULT, ownerToken); assertEquals(1, individualDataResult.getNumResults()); assertEquals("Test", individualDataResult.first().getId()); assertEquals("19870214", individualDataResult.first().getDateOfBirth()); DataResult update = individualManager.update(studyFqn, individualDataResult.first().getId(), - new IndividualUpdateParams().setDateOfBirth(""), QueryOptions.empty(), token); + new IndividualUpdateParams().setDateOfBirth(""), QueryOptions.empty(), ownerToken); assertEquals(1, update.getNumUpdated()); - Individual individual = individualManager.get(studyFqn, individualDataResult.first().getId(), QueryOptions.empty(), token) + Individual individual = individualManager.get(studyFqn, individualDataResult.first().getId(), QueryOptions.empty(), ownerToken) .first(); assertEquals("", individual.getDateOfBirth()); update = individualManager.update(studyFqn, individualDataResult.first().getId(), - new IndividualUpdateParams().setDateOfBirth("19870214"), QueryOptions.empty(), token); + new IndividualUpdateParams().setDateOfBirth("19870214"), QueryOptions.empty(), ownerToken); assertEquals(1, update.getNumUpdated()); - individual = individualManager.get(studyFqn, individualDataResult.first().getId(), QueryOptions.empty(), token) + individual = individualManager.get(studyFqn, individualDataResult.first().getId(), QueryOptions.empty(), ownerToken) .first(); assertEquals("19870214", individual.getDateOfBirth()); update = individualManager.update(studyFqn, individualDataResult.first().getId(), - new IndividualUpdateParams().setAttributes(Collections.singletonMap("key", "value")), QueryOptions.empty(), token); + new IndividualUpdateParams().setAttributes(Collections.singletonMap("key", "value")), QueryOptions.empty(), ownerToken); assertEquals(1, update.getNumUpdated()); - individual = individualManager.get(studyFqn, individualDataResult.first().getId(), QueryOptions.empty(), token) + individual = individualManager.get(studyFqn, individualDataResult.first().getId(), QueryOptions.empty(), ownerToken) .first(); assertEquals("value", individual.getAttributes().get("key")); update = individualManager.update(studyFqn, individualDataResult.first().getId(), - new IndividualUpdateParams().setAttributes(Collections.singletonMap("key2", "value2")), QueryOptions.empty(), token); + new IndividualUpdateParams().setAttributes(Collections.singletonMap("key2", "value2")), QueryOptions.empty(), ownerToken); assertEquals(1, update.getNumUpdated()); - individual = individualManager.get(studyFqn, individualDataResult.first().getId(), QueryOptions.empty(), token) + individual = individualManager.get(studyFqn, individualDataResult.first().getId(), QueryOptions.empty(), ownerToken) .first(); assertEquals("value", individual.getAttributes().get("key")); // Keep "key" assertEquals("value2", individual.getAttributes().get("key2")); // add new "key2" @@ -679,22 +679,22 @@ public void testUpdateIndividualInfo() throws CatalogException { thrown.expect(CatalogException.class); thrown.expectMessage("Invalid date of birth format"); individualManager.update(studyFqn, individualDataResult.first().getId(), - new IndividualUpdateParams().setDateOfBirth("198421"), QueryOptions.empty(), token); + new IndividualUpdateParams().setDateOfBirth("198421"), QueryOptions.empty(), ownerToken); } @Test public void testUpdateIndividualParents() throws CatalogException { IndividualManager individualManager = catalogManager.getIndividualManager(); - individualManager.create(studyFqn, new Individual().setId("child"), QueryOptions.empty(), token); - individualManager.create(studyFqn, new Individual().setId("father"), QueryOptions.empty(), token); - individualManager.create(studyFqn, new Individual().setId("mother"), QueryOptions.empty(), token); + individualManager.create(studyFqn, new Individual().setId("child"), QueryOptions.empty(), ownerToken); + individualManager.create(studyFqn, new Individual().setId("father"), QueryOptions.empty(), ownerToken); + individualManager.create(studyFqn, new Individual().setId("mother"), QueryOptions.empty(), ownerToken); DataResult individualDataResult = individualManager.update(studyFqn, "child", new IndividualUpdateParams().setFather(new IndividualReferenceParam("father", "")) - .setMother(new IndividualReferenceParam("mother", "")), QueryOptions.empty(), token); + .setMother(new IndividualReferenceParam("mother", "")), QueryOptions.empty(), ownerToken); assertEquals(1, individualDataResult.getNumUpdated()); - Individual individual = individualManager.get(studyFqn, "child", QueryOptions.empty(), token).first(); + Individual individual = individualManager.get(studyFqn, "child", QueryOptions.empty(), ownerToken).first(); assertEquals("mother", individual.getMother().getId()); assertEquals(1, individual.getMother().getVersion()); @@ -706,16 +706,16 @@ public void testUpdateIndividualParents() throws CatalogException { @Test public void testRemoveIndividualParents() throws CatalogException { IndividualManager individualManager = catalogManager.getIndividualManager(); - individualManager.create(studyFqn, new Individual().setId("child"), QueryOptions.empty(), token); - individualManager.create(studyFqn, new Individual().setId("father"), QueryOptions.empty(), token); - individualManager.create(studyFqn, new Individual().setId("mother"), QueryOptions.empty(), token); + individualManager.create(studyFqn, new Individual().setId("child"), QueryOptions.empty(), ownerToken); + individualManager.create(studyFqn, new Individual().setId("father"), QueryOptions.empty(), ownerToken); + individualManager.create(studyFqn, new Individual().setId("mother"), QueryOptions.empty(), ownerToken); DataResult individualDataResult = individualManager.update(studyFqn, "child", new IndividualUpdateParams().setFather(new IndividualReferenceParam("father", "")) - .setMother(new IndividualReferenceParam("mother", "")), QueryOptions.empty(), token); + .setMother(new IndividualReferenceParam("mother", "")), QueryOptions.empty(), ownerToken); assertEquals(1, individualDataResult.getNumUpdated()); - Individual individual = individualManager.get(studyFqn, "child", QueryOptions.empty(), token).first(); + Individual individual = individualManager.get(studyFqn, "child", QueryOptions.empty(), ownerToken).first(); assertEquals("mother", individual.getMother().getId()); assertEquals(1, individual.getMother().getVersion()); @@ -724,8 +724,8 @@ public void testRemoveIndividualParents() throws CatalogException { assertEquals(1, individual.getFather().getVersion()); individualManager.update(studyFqn, "child", new IndividualUpdateParams().setFather(new IndividualReferenceParam("", "")) - .setMother(new IndividualReferenceParam("", "")), QueryOptions.empty(), token); - individual = individualManager.get(studyFqn, "child", QueryOptions.empty(), token).first(); + .setMother(new IndividualReferenceParam("", "")), QueryOptions.empty(), ownerToken); + individual = individualManager.get(studyFqn, "child", QueryOptions.empty(), ownerToken).first(); assertNull(individual.getMother().getId()); assertNull(individual.getFather().getId()); @@ -735,25 +735,25 @@ public void testRemoveIndividualParents() throws CatalogException { public void testIndividualRelatives() throws CatalogException { IndividualManager individualManager = catalogManager.getIndividualManager(); individualManager.create(studyFqn, new Individual().setId("proband").setSex(SexOntologyTermAnnotation.initMale()), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); individualManager.create(studyFqn, new Individual().setId("brother").setSex(SexOntologyTermAnnotation.initMale()), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); individualManager.create(studyFqn, new Individual().setId("sister").setSex(SexOntologyTermAnnotation.initFemale()), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); individualManager.create(studyFqn, new Individual().setId("father").setSex(SexOntologyTermAnnotation.initMale()), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); individualManager.create(studyFqn, new Individual().setId("mother").setSex(SexOntologyTermAnnotation.initFemale()), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); individualManager.update(studyFqn, "proband", new IndividualUpdateParams().setFather(new IndividualReferenceParam("father", "")) - .setMother(new IndividualReferenceParam("mother", "")), QueryOptions.empty(), token); + .setMother(new IndividualReferenceParam("mother", "")), QueryOptions.empty(), ownerToken); individualManager.update(studyFqn, "brother", new IndividualUpdateParams().setFather(new IndividualReferenceParam("father", "")) - .setMother(new IndividualReferenceParam("mother", "")), QueryOptions.empty(), token); + .setMother(new IndividualReferenceParam("mother", "")), QueryOptions.empty(), ownerToken); individualManager.update(studyFqn, "sister", new IndividualUpdateParams().setFather(new IndividualReferenceParam("father", "")) - .setMother(new IndividualReferenceParam("mother", "")), QueryOptions.empty(), token); + .setMother(new IndividualReferenceParam("mother", "")), QueryOptions.empty(), ownerToken); OpenCGAResult relatives = catalogManager.getIndividualManager().relatives(studyFqn, "proband", 2, - new QueryOptions(QueryOptions.INCLUDE, IndividualDBAdaptor.QueryParams.ID.key()), token); + new QueryOptions(QueryOptions.INCLUDE, IndividualDBAdaptor.QueryParams.ID.key()), ownerToken); assertEquals(5, relatives.getNumResults()); for (Individual individual : relatives.getResults()) { @@ -769,7 +769,7 @@ public void testDeleteIndividualWithFamilies() throws CatalogException { .setId("child") .setPhenotypes(Collections.singletonList(new Phenotype().setId("phenotype1"))) .setDisorders(Collections.singletonList(new Disorder().setId("disorder1"))), - INCLUDE_RESULT, token).first(); + INCLUDE_RESULT, ownerToken).first(); Individual father = new Individual() .setId("father") .setPhenotypes(Collections.singletonList(new Phenotype().setId("phenotype2"))) @@ -781,24 +781,24 @@ public void testDeleteIndividualWithFamilies() throws CatalogException { FamilyManager familyManager = catalogManager.getFamilyManager(); familyManager.create(studyFqn, new Family().setId("family1").setMembers(Collections.singletonList(father)), - Collections.singletonList(child.getId()), QueryOptions.empty(), token); + Collections.singletonList(child.getId()), QueryOptions.empty(), ownerToken); familyManager.create(studyFqn, new Family().setId("family2").setMembers(Collections.singletonList(mother)), - Arrays.asList(father.getId(), child.getId()), QueryOptions.empty(), token); + Arrays.asList(father.getId(), child.getId()), QueryOptions.empty(), ownerToken); try { DataResult writeResult = individualManager.delete(studyFqn, new Query(IndividualDBAdaptor.QueryParams.ID.key(), "child"), - new QueryOptions(), token); + new QueryOptions(), ownerToken); fail("Expected fail"); } catch (CatalogException e) { assertTrue(e.getMessage().contains("found in the families")); } DataResult writeResult = individualManager.delete(studyFqn, new Query(IndividualDBAdaptor.QueryParams.ID.key(), "child"), - new QueryOptions(Constants.FORCE, true), token); + new QueryOptions(Constants.FORCE, true), ownerToken); assertEquals(1, writeResult.getNumDeleted()); - Family family1 = familyManager.get(studyFqn, "family1", QueryOptions.empty(), token).first(); - Family family2 = familyManager.get(studyFqn, "family2", QueryOptions.empty(), token).first(); + Family family1 = familyManager.get(studyFqn, "family1", QueryOptions.empty(), ownerToken).first(); + Family family2 = familyManager.get(studyFqn, "family2", QueryOptions.empty(), ownerToken).first(); assertEquals(1, family1.getMembers().size()); assertEquals(0, family1.getMembers().stream().filter(i -> i.getId().equals("child")).count()); @@ -820,81 +820,56 @@ public void testDeleteIndividualWithFamilies() throws CatalogException { @Test public void testGetIndividualWithSamples() throws CatalogException { IndividualManager individualManager = catalogManager.getIndividualManager(); - individualManager.create(studyFqn, new Individual().setId("individual1") - .setSamples(Arrays.asList(new Sample().setId("sample1"), new Sample().setId("sample2"), new Sample().setId( - "sample3"))), - QueryOptions.empty(), token); - individualManager.create(studyFqn, new Individual().setId("individual2") - .setSamples(Arrays.asList(new Sample().setId("sample4"), new Sample().setId("sample5"), new Sample().setId( - "sample6"))), - QueryOptions.empty(), token); - - DataResult search = individualManager.search(studyFqn, new Query(), QueryOptions.empty(), token); + DataResult search = individualManager.search(studyFqn, new Query(), QueryOptions.empty(), ownerToken); assertEquals(2, search.getNumResults()); search.getResults().forEach(i -> { - assertEquals(3, i.getSamples().size()); + assertEquals(1, i.getSamples().size()); assertTrue(org.apache.commons.lang3.StringUtils.isNotEmpty(i.getSamples().get(0).getCreationDate())); - if (i.getId().equals("individual1")) { - assertTrue(Arrays.asList("sample1", "sample2", "sample3").containsAll( - i.getSamples().stream().map(Sample::getId).collect(Collectors.toList()) - )); + if (i.getId().equals(ind1)) { + assertEquals(s_1Id, i.getSamples().get(0).getId()); } else { - assertTrue(Arrays.asList("sample4", "sample5", "sample6").containsAll( - i.getSamples().stream().map(Sample::getId).collect(Collectors.toList()) - )); + assertEquals(s_2Id, i.getSamples().get(0).getId()); } }); search = individualManager.search(studyFqn, new Query(), new QueryOptions(QueryOptions.EXCLUDE, "samples.creationDate"), - token); + ownerToken); assertEquals(2, search.getNumResults()); search.getResults().forEach(i -> { - assertEquals(3, i.getSamples().size()); + assertEquals(1, i.getSamples().size()); assertTrue(org.apache.commons.lang3.StringUtils.isEmpty(i.getSamples().get(0).getCreationDate())); - if (i.getId().equals("individual1")) { - assertTrue(Arrays.asList("sample1", "sample2", "sample3").containsAll( - i.getSamples().stream().map(Sample::getId).collect(Collectors.toList()) - )); + if (i.getId().equals(ind1)) { + assertEquals(s_1Id, i.getSamples().get(0).getId()); } else { - assertTrue(Arrays.asList("sample4", "sample5", "sample6").containsAll( - i.getSamples().stream().map(Sample::getId).collect(Collectors.toList()) - )); + assertEquals(s_2Id, i.getSamples().get(0).getId()); } }); search = individualManager.search(studyFqn, new Query(), new QueryOptions(QueryOptions.INCLUDE, "samples.id"), - token); + ownerToken); assertEquals(2, search.getNumResults()); search.getResults().forEach(i -> { - assertEquals(3, i.getSamples().size()); + assertEquals(1, i.getSamples().size()); assertTrue(org.apache.commons.lang3.StringUtils.isEmpty(i.getSamples().get(0).getCreationDate())); - if (i.getId().equals("individual1")) { - assertTrue(Arrays.asList("sample1", "sample2", "sample3").containsAll( - i.getSamples().stream().map(Sample::getId).collect(Collectors.toList()) - )); + if (i.getId().equals(ind1)) { + assertEquals(s_1Id, i.getSamples().get(0).getId()); } else { - assertTrue(Arrays.asList("sample4", "sample5", "sample6").containsAll( - i.getSamples().stream().map(Sample::getId).collect(Collectors.toList()) - )); + assertEquals(s_2Id, i.getSamples().get(0).getId()); } }); search = individualManager.search(studyFqn, new Query(), new QueryOptions(QueryOptions.INCLUDE, "id,creationDate,samples.id"), - token); + ownerToken); assertEquals(2, search.getNumResults()); search.getResults().forEach(i -> { assertTrue(org.apache.commons.lang3.StringUtils.isNotEmpty(i.getCreationDate())); assertTrue(org.apache.commons.lang3.StringUtils.isEmpty(i.getName())); - assertEquals(3, i.getSamples().size()); + assertEquals(1, i.getSamples().size()); assertTrue(org.apache.commons.lang3.StringUtils.isEmpty(i.getSamples().get(0).getCreationDate())); - if (i.getId().equals("individual1")) { - assertTrue(Arrays.asList("sample1", "sample2", "sample3").containsAll( - i.getSamples().stream().map(Sample::getId).collect(Collectors.toList()) - )); + if (i.getId().equals(ind1)) { + assertEquals(s_1Id, i.getSamples().get(0).getId()); } else { - assertTrue(Arrays.asList("sample4", "sample5", "sample6").containsAll( - i.getSamples().stream().map(Sample::getId).collect(Collectors.toList()) - )); + assertEquals(s_2Id, i.getSamples().get(0).getId()); } }); } @@ -905,19 +880,19 @@ public void incrementVersionTest() throws CatalogException { Individual dummyIndividual1 = DummyModelUtils.getDummyIndividual(null, null, null, null); Individual dummyIndividual2 = DummyModelUtils.getDummyIndividual(null, null, null, null); - catalogManager.getIndividualManager().create(studyFqn, dummyIndividual1, QueryOptions.empty(), token); - catalogManager.getIndividualManager().create(studyFqn, dummyIndividual2, QueryOptions.empty(), token); + catalogManager.getIndividualManager().create(studyFqn, dummyIndividual1, QueryOptions.empty(), ownerToken); + catalogManager.getIndividualManager().create(studyFqn, dummyIndividual2, QueryOptions.empty(), ownerToken); OpenCGAResult result = catalogManager.getIndividualManager().get(studyFqn, - Arrays.asList(dummyIndividual1.getId(), dummyIndividual2.getId()), QueryOptions.empty(), token); + Arrays.asList(dummyIndividual1.getId(), dummyIndividual2.getId()), QueryOptions.empty(), ownerToken); for (Individual individual : result.getResults()) { assertEquals(1, individual.getVersion()); } catalogManager.getIndividualManager().update(studyFqn, dummyIndividual1.getId(), new IndividualUpdateParams().setName("name"), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); result = catalogManager.getIndividualManager().get(studyFqn, - Arrays.asList(dummyIndividual1.getId(), dummyIndividual2.getId()), QueryOptions.empty(), token); + Arrays.asList(dummyIndividual1.getId(), dummyIndividual2.getId()), QueryOptions.empty(), ownerToken); assertEquals(2, result.first().getVersion()); assertEquals("name", result.first().getName()); assertEquals(1, result.getResults().get(1).getVersion()); @@ -926,7 +901,7 @@ public void incrementVersionTest() throws CatalogException { .append(IndividualDBAdaptor.QueryParams.ID.key(), dummyIndividual1.getId()) .append(Constants.ALL_VERSIONS, true); - result = catalogManager.getIndividualManager().search(studyFqn, query, QueryOptions.empty(), token); + result = catalogManager.getIndividualManager().search(studyFqn, query, QueryOptions.empty(), ownerToken); assertEquals(2, result.getNumResults()); assertEquals(dummyIndividual1.getId(), result.getResults().get(0).getName()); assertEquals(1, result.getResults().get(0).getVersion()); @@ -954,27 +929,27 @@ public void memberReferenceTest() throws CatalogException { family.setMembers(null); QueryOptions options = new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true); - individual1 = catalogManager.getIndividualManager().create(studyFqn, individual1, options, token).first(); + individual1 = catalogManager.getIndividualManager().create(studyFqn, individual1, options, ownerToken).first(); assertEquals(1, individual1.getVersion()); assertEquals(2, individual1.getSamples().size()); assertEquals(2, individual1.getSamples().stream().map(Sample::getVersion).filter(v -> v == 1).count()); assertEquals(2, individual1.getSamples().stream().map(Sample::getIndividualId).filter(i -> i.equals(individualId1)).count()); - individual2 = catalogManager.getIndividualManager().create(studyFqn, individual2, options, token).first(); + individual2 = catalogManager.getIndividualManager().create(studyFqn, individual2, options, ownerToken).first(); assertEquals(1, individual2.getVersion()); assertEquals(2, individual2.getSamples().size()); assertEquals(2, individual2.getSamples().stream().map(Sample::getVersion).filter(v -> v == 1).count()); assertEquals(2, individual2.getSamples().stream().map(Sample::getIndividualId).filter(i -> i.equals(individualId2)).count()); family = catalogManager.getFamilyManager().create(studyFqn, family, Arrays.asList(individual1.getId(), individual2.getId()), - options, token).first(); + options, ownerToken).first(); assertEquals(2, family.getMembers().size()); assertEquals(1, family.getVersion()); assertEquals(2, family.getMembers().stream().map(Individual::getVersion).filter(v -> v == 2).count()); // Update individual 2 individual2 = catalogManager.getIndividualManager().update(studyFqn, individual2.getId(), new IndividualUpdateParams() - .setName("blabla"), options, token).first(); + .setName("blabla"), options, ownerToken).first(); assertEquals(2, individual2.getSamples().size()); assertEquals(3, individual2.getVersion()); assertEquals(2, individual2.getSamples().stream().map(Sample::getVersion).filter(v -> v == 1).count()); @@ -982,7 +957,7 @@ public void memberReferenceTest() throws CatalogException { assertEquals(1, individual2.getFamilyIds().size()); assertEquals(family.getId(), individual2.getFamilyIds().get(0)); - family = catalogManager.getFamilyManager().get(studyFqn, family.getId(), QueryOptions.empty(), token).first(); + family = catalogManager.getFamilyManager().get(studyFqn, family.getId(), QueryOptions.empty(), ownerToken).first(); assertTrue(family.getRoles().containsKey(individualId1)); assertFalse(family.getRoles().containsKey("blabla")); assertEquals(2, family.getMembers().size()); @@ -998,7 +973,7 @@ public void memberReferenceTest() throws CatalogException { // Update id from individual1 individual1 = catalogManager.getIndividualManager().update(studyFqn, individual1.getId(), new IndividualUpdateParams() - .setId("blabla"), options, token).first(); + .setId("blabla"), options, ownerToken).first(); assertEquals(2, individual1.getSamples().size()); assertEquals(3, individual1.getVersion()); assertEquals(2, individual1.getSamples().stream().map(Sample::getVersion).filter(v -> v == 2).count()); @@ -1006,14 +981,14 @@ public void memberReferenceTest() throws CatalogException { assertEquals(1, individual1.getFamilyIds().size()); assertEquals(family.getId(), individual1.getFamilyIds().get(0)); - family = catalogManager.getFamilyManager().get(studyFqn, family.getId(), QueryOptions.empty(), token).first(); + family = catalogManager.getFamilyManager().get(studyFqn, family.getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(2, family.getMembers().size()); assertEquals(2, family.getMembers().stream().map(Individual::getVersion).filter(v -> v == 3).count()); assertFalse(family.getRoles().containsKey(individualId1)); assertTrue(family.getRoles().containsKey("blabla")); List samples = catalogManager.getSampleManager().get(studyFqn, Arrays.asList(sample1.getId(), sample2.getId()), - QueryOptions.empty(), token).getResults(); + QueryOptions.empty(), ownerToken).getResults(); for (Sample sample : samples) { assertEquals("blabla", sample.getIndividualId()); assertEquals(2, sample.getVersion()); @@ -1027,12 +1002,12 @@ public void updateInUseInCATest() throws CatalogException { Family family = DummyModelUtils.getDummyCaseFamily("family1"); for (int i = family.getMembers().size() - 1; i >= 0; i--) { - catalogManager.getIndividualManager().create(studyFqn, family.getMembers().get(i), QueryOptions.empty(), token); + catalogManager.getIndividualManager().create(studyFqn, family.getMembers().get(i), QueryOptions.empty(), ownerToken); } List members = family.getMembers().stream().map(Individual::getId).collect(Collectors.toList()); family.setMembers(null); - family = catalogManager.getFamilyManager().create(studyFqn, family, members, options, token).first(); + family = catalogManager.getFamilyManager().create(studyFqn, family, members, options, ownerToken).first(); // Unlocked cases ClinicalAnalysis case1 = DummyModelUtils.getDummyClinicalAnalysis(family.getMembers().get(0), family, null); @@ -1042,29 +1017,29 @@ public void updateInUseInCATest() throws CatalogException { ClinicalAnalysis case3 = DummyModelUtils.getDummyClinicalAnalysis(family.getMembers().get(0), family, null) .setLocked(true); - case1 = catalogManager.getClinicalAnalysisManager().create(studyFqn, case1, options, token).first(); + case1 = catalogManager.getClinicalAnalysisManager().create(studyFqn, case1, options, ownerToken).first(); assertFalse(case1.isLocked()); - case2 = catalogManager.getClinicalAnalysisManager().create(studyFqn, case2, options, token).first(); + case2 = catalogManager.getClinicalAnalysisManager().create(studyFqn, case2, options, ownerToken).first(); assertFalse(case2.isLocked()); - case3 = catalogManager.getClinicalAnalysisManager().create(studyFqn, case3, options, token).first(); + case3 = catalogManager.getClinicalAnalysisManager().create(studyFqn, case3, options, ownerToken).first(); assertTrue(case3.isLocked()); String oldProbandId = case3.getProband().getId(); // Update proband information Individual proband = catalogManager.getIndividualManager().update(studyFqn, oldProbandId, - new IndividualUpdateParams().setId("blabla"), options, token).first(); + new IndividualUpdateParams().setId("blabla"), options, ownerToken).first(); assertEquals("blabla", proband.getId()); assertEquals(3, proband.getVersion()); - family = catalogManager.getFamilyManager().get(studyFqn, family.getId(), QueryOptions.empty(), token).first(); + family = catalogManager.getFamilyManager().get(studyFqn, family.getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(2, family.getVersion()); assertTrue(family.getRoles().containsKey("blabla")); assertEquals(1, family.getMembers().stream().filter(i -> i.getId().equals("blabla")).count()); assertEquals(1, family.getMembers().stream().filter(i -> i.getId().equals("blabla")).filter(i -> i.getVersion() == 3).count()); OpenCGAResult result = catalogManager.getClinicalAnalysisManager().get(studyFqn, - Arrays.asList(case1.getId(), case2.getId(), case3.getId()), QueryOptions.empty(), token); + Arrays.asList(case1.getId(), case2.getId(), case3.getId()), QueryOptions.empty(), ownerToken); case1 = result.getResults().get(0); case2 = result.getResults().get(1); case3 = result.getResults().get(2); @@ -1089,12 +1064,12 @@ public void updateDeleteInUseInCATest() throws CatalogException { Family family = DummyModelUtils.getDummyCaseFamily("family1"); for (int i = family.getMembers().size() - 1; i >= 0; i--) { - catalogManager.getIndividualManager().create(studyFqn, family.getMembers().get(i), QueryOptions.empty(), token); + catalogManager.getIndividualManager().create(studyFqn, family.getMembers().get(i), QueryOptions.empty(), ownerToken); } List members = family.getMembers().stream().map(Individual::getId).collect(Collectors.toList()); family.setMembers(null); - family = catalogManager.getFamilyManager().create(studyFqn, family, members, options, token).first(); + family = catalogManager.getFamilyManager().create(studyFqn, family, members, options, ownerToken).first(); // Unlocked cases ClinicalAnalysis case1 = DummyModelUtils.getDummyClinicalAnalysis(family.getMembers().get(0), family, null); @@ -1104,28 +1079,28 @@ public void updateDeleteInUseInCATest() throws CatalogException { ClinicalAnalysis case3 = DummyModelUtils.getDummyClinicalAnalysis(family.getMembers().get(0), family, null) .setLocked(true); - case1 = catalogManager.getClinicalAnalysisManager().create(studyFqn, case1, options, token).first(); + case1 = catalogManager.getClinicalAnalysisManager().create(studyFqn, case1, options, ownerToken).first(); assertFalse(case1.isLocked()); - case2 = catalogManager.getClinicalAnalysisManager().create(studyFqn, case2, options, token).first(); + case2 = catalogManager.getClinicalAnalysisManager().create(studyFqn, case2, options, ownerToken).first(); assertFalse(case2.isLocked()); - case3 = catalogManager.getClinicalAnalysisManager().create(studyFqn, case3, options, token).first(); + case3 = catalogManager.getClinicalAnalysisManager().create(studyFqn, case3, options, ownerToken).first(); assertTrue(case3.isLocked()); // Delete proband try { catalogManager.getIndividualManager().delete(studyFqn, Collections.singletonList(case1.getProband().getId()), - new QueryOptions(ParamConstants.FORCE, true), token); + new QueryOptions(ParamConstants.FORCE, true), ownerToken); } catch (CatalogException e) { assertTrue(e.getMessage().contains("in use in 3 cases")); } // unlock case3 catalogManager.getClinicalAnalysisManager().update(studyFqn, case3.getId(), new ClinicalAnalysisUpdateParams().setLocked(false), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); try { catalogManager.getIndividualManager().delete(studyFqn, Collections.singletonList(case1.getProband().getId()), - new QueryOptions(ParamConstants.FORCE, true), token); + new QueryOptions(ParamConstants.FORCE, true), ownerToken); } catch (CatalogException e) { assertTrue(e.getMessage().contains("in use in 3 cases")); } @@ -1136,30 +1111,32 @@ public void updateDeleteInUseInCATest() throws CatalogException { public void viewSampleFilesFromIndividualTest() throws CatalogException { // Link VCF file. This VCF file will automatically create sample NA19600 String vcfFile = getClass().getResource("/biofiles/variant-test-file.vcf.gz").getFile(); - catalogManager.getFileManager().link(studyFqn, new FileLinkParams(vcfFile, "", "", "", null, null, null, null, null), false, token); + catalogManager.getFileManager().link(studyFqn, new FileLinkParams(vcfFile, "", "", "", null, null, null, null, null), false, ownerToken); Sample sample = catalogManager.getSampleManager().get(studyFqn, "NA19600", - new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.FILE_IDS.key()), token).first(); + new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.FILE_IDS.key()), ownerToken).first(); assertEquals(1, sample.getFileIds().size()); assertEquals("variant-test-file.vcf.gz", sample.getFileIds().get(0)); // Create individual catalogManager.getIndividualManager().create(studyFqn, new Individual().setId("individual"), Collections.singletonList(sample.getId()), - QueryOptions.empty(), token); - Individual individual = catalogManager.getIndividualManager().get(studyFqn, "individual", QueryOptions.empty(), token).first(); + + QueryOptions.empty(), ownerToken); + Individual individual = catalogManager.getIndividualManager().get(studyFqn, "individual", QueryOptions.empty(), ownerToken).first(); + assertEquals(1, individual.getSamples().get(0).getFileIds().size()); assertEquals("variant-test-file.vcf.gz", individual.getSamples().get(0).getFileIds().get(0)); // Link BAM file (related to NA19600 sample) String bamFile = getClass().getResource("/biofiles/NA19600.chrom20.small.bam").getFile(); - catalogManager.getFileManager().link(studyFqn, new FileLinkParams(bamFile, "", "", "", null, null, null, null, null), false, token); + catalogManager.getFileManager().link(studyFqn, new FileLinkParams(bamFile, "", "", "", null, null, null, null, null), false, ownerToken); sample = catalogManager.getSampleManager().get(studyFqn, "NA19600", - new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.FILE_IDS.key()), token).first(); + new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.FILE_IDS.key()), ownerToken).first(); assertEquals(2, sample.getFileIds().size()); assertTrue(Arrays.asList("variant-test-file.vcf.gz", "NA19600.chrom20.small.bam").containsAll(sample.getFileIds())); - individual = catalogManager.getIndividualManager().get(studyFqn, "individual", QueryOptions.empty(), token).first(); + individual = catalogManager.getIndividualManager().get(studyFqn, "individual", QueryOptions.empty(), ownerToken).first(); assertEquals(2, individual.getSamples().get(0).getFileIds().size()); assertTrue(Arrays.asList("variant-test-file.vcf.gz", "NA19600.chrom20.small.bam").containsAll(individual.getSamples().get(0).getFileIds())); } diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/InterpretationManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/InterpretationManagerTest.java index f3e32569bb0..80b630d34ee 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/InterpretationManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/InterpretationManagerTest.java @@ -1,89 +1,53 @@ package org.opencb.opencga.catalog.managers; import org.apache.commons.lang3.RandomStringUtils; -import org.apache.commons.lang3.StringUtils; -import org.junit.After; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; -import org.junit.rules.ExpectedException; -import org.opencb.biodata.models.common.Status; import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.commons.test.GenericTest; -import org.opencb.opencga.TestParamConstants; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.models.clinical.*; import org.opencb.opencga.core.models.common.StatusParam; import org.opencb.opencga.core.models.family.Family; -import org.opencb.opencga.core.models.file.File; -import org.opencb.opencga.core.models.file.FileLinkParams; import org.opencb.opencga.core.models.individual.Individual; import org.opencb.opencga.core.models.panel.Panel; import org.opencb.opencga.core.models.panel.PanelReferenceParam; import org.opencb.opencga.core.models.sample.Sample; -import org.opencb.opencga.core.models.user.Account; +import org.opencb.opencga.core.models.study.StudyAclParams; +import org.opencb.opencga.core.models.study.StudyPermissions; +import org.opencb.opencga.core.models.study.configuration.ClinicalAnalysisStudyConfiguration; import org.opencb.opencga.core.testclassification.duration.MediumTests; -import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; import static org.junit.Assert.*; @Category(MediumTests.class) -public class InterpretationManagerTest extends GenericTest { +public class InterpretationManagerTest extends AbstractManagerTest { - public final static String STUDY = "user@1000G:phase1"; - @Rule - public ExpectedException thrown = ExpectedException.none(); - - @Rule - public CatalogManagerExternalResource catalogManagerResource = new CatalogManagerExternalResource(); - - protected CatalogManager catalogManager; - private String opencgaToken; - protected String sessionIdUser; private FamilyManager familyManager; - private static final QueryOptions INCLUDE_RESULT = new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true); - @Before - public void setUp() throws IOException, CatalogException { - catalogManager = catalogManagerResource.getCatalogManager(); + public void setUp() throws Exception { + super.setUp(); familyManager = catalogManager.getFamilyManager(); - setUpCatalogManager(catalogManager); - } - - public void setUpCatalogManager(CatalogManager catalogManager) throws IOException, CatalogException { - opencgaToken = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); - - catalogManager.getUserManager().create("user", "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, "", null, Account.AccountType.FULL, opencgaToken); - sessionIdUser = catalogManager.getUserManager().login("user", TestParamConstants.PASSWORD).getToken(); - - catalogManager.getUserManager().create("user2", "User Name2", "mail2@ebi.ac.uk", TestParamConstants.PASSWORD, "", null, Account.AccountType.GUEST, - opencgaToken); - - String projectId = catalogManager.getProjectManager().create("1000G", "Project about some genomes", "", "Homo sapiens", - null, "GRCh38", INCLUDE_RESULT, sessionIdUser).first().getId(); - catalogManager.getStudyManager().create(projectId, "phase1", null, "Phase 1", "Done", null, null, null, null, null, sessionIdUser); - } - - @After - public void tearDown() throws Exception { } private DataResult createDummyFamily() throws CatalogException { Family family = DummyModelUtils.getDummyFamily("family"); - return familyManager.create(STUDY, family, INCLUDE_RESULT, sessionIdUser); + return familyManager.create(studyFqn, family, INCLUDE_RESULT, ownerToken); } private DataResult createDummyEnvironment(boolean createFamily, boolean createDefaultInterpretation) throws CatalogException { ClinicalAnalysis clinicalAnalysis = new ClinicalAnalysis() - .setStatus(new Status().setId(ClinicalAnalysisStatus.READY_FOR_INTERPRETATION)) + .setStatus(new ClinicalStatus().setId("READY_FOR_INTERPRETATION")) .setId("analysis" + RandomStringUtils.randomAlphanumeric(3)) .setDescription("My description").setType(ClinicalAnalysis.Type.FAMILY) .setProband(new Individual().setId("child1").setSamples(Arrays.asList(new Sample().setId("sample2")))); @@ -94,80 +58,61 @@ private DataResult createDummyEnvironment(boolean createFamily clinicalAnalysis.setFamily(new Family().setId("family") .setMembers(Arrays.asList(new Individual().setId("child1").setSamples(Arrays.asList(new Sample().setId("sample2")))))); - return catalogManager.getClinicalAnalysisManager().create(STUDY, clinicalAnalysis, !createDefaultInterpretation, - INCLUDE_RESULT, sessionIdUser); - } - - private List registerDummyFiles() throws CatalogException { - List files = new LinkedList<>(); - - String vcfFile = getClass().getResource("/biofiles/variant-test-file.vcf.gz").getFile(); - files.add(catalogManager.getFileManager().link(STUDY, new FileLinkParams(vcfFile, "", "", "", null, null, null, null, - null), false, sessionIdUser).first()); - vcfFile = getClass().getResource("/biofiles/family.vcf").getFile(); - files.add(catalogManager.getFileManager().link(STUDY, new FileLinkParams(vcfFile, "", "", "", null, null, null, null, - null), false, sessionIdUser).first()); - String bamFile = getClass().getResource("/biofiles/HG00096.chrom20.small.bam").getFile(); - files.add(catalogManager.getFileManager().link(STUDY, new FileLinkParams(bamFile, "", "", "", null, null, null, null, - null), false, sessionIdUser).first()); - bamFile = getClass().getResource("/biofiles/NA19600.chrom20.small.bam").getFile(); - files.add(catalogManager.getFileManager().link(STUDY, new FileLinkParams(bamFile, "", "", "", null, null, null, null, - null), false, sessionIdUser).first()); - - return files; + return catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, !createDefaultInterpretation, + INCLUDE_RESULT, ownerToken); } @Test public void deleteLockedInterpretationTest() throws CatalogException { ClinicalAnalysis ca = createDummyEnvironment(true, false).first(); - catalogManager.getInterpretationManager().create(STUDY, ca.getId(), new Interpretation().setLocked(true), - ParamUtils.SaveInterpretationAs.PRIMARY, QueryOptions.empty(), sessionIdUser); - ca = catalogManager.getClinicalAnalysisManager().get(STUDY, ca.getId(), QueryOptions.empty(), sessionIdUser).first(); + catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation().setLocked(true), + ParamUtils.SaveInterpretationAs.PRIMARY, QueryOptions.empty(), ownerToken); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, ca.getId(), QueryOptions.empty(), ownerToken).first(); - Interpretation interpretation = catalogManager.getInterpretationManager().get(STUDY, ca.getInterpretation().getId(), - QueryOptions.empty(), sessionIdUser).first(); + Interpretation interpretation = catalogManager.getInterpretationManager().get(studyFqn, ca.getInterpretation().getId(), + QueryOptions.empty(), ownerToken).first(); assertTrue(interpretation.isLocked()); // Try to delete interpretation try { - catalogManager.getInterpretationManager().delete(STUDY, ca.getId(), Collections.singletonList(ca.getInterpretation().getId()), - sessionIdUser); + catalogManager.getInterpretationManager().delete(studyFqn, ca.getId(), Collections.singletonList(ca.getInterpretation().getId()), + ownerToken); fail("Interpretation is locked so it should not allow this"); } catch (CatalogException e) { assertTrue(e.getMessage().contains("locked")); } // Unlock interpretation - catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getInterpretation().getId(), - new InterpretationUpdateParams().setLocked(false), null, QueryOptions.empty(), sessionIdUser); - interpretation = catalogManager.getInterpretationManager().get(STUDY, ca.getInterpretation().getId(), - QueryOptions.empty(), sessionIdUser).first(); + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getInterpretation().getId(), + new InterpretationUpdateParams().setLocked(false), null, QueryOptions.empty(), ownerToken); + interpretation = catalogManager.getInterpretationManager().get(studyFqn, ca.getInterpretation().getId(), + QueryOptions.empty(), ownerToken).first(); assertFalse(interpretation.isLocked()); // Delete interpretation - catalogManager.getInterpretationManager().delete(STUDY, ca.getId(), Collections.singletonList(ca.getInterpretation().getId()), - sessionIdUser); - ca = catalogManager.getClinicalAnalysisManager().get(STUDY, ca.getId(), QueryOptions.empty(), sessionIdUser).first(); + catalogManager.getInterpretationManager().delete(studyFqn, ca.getId(), Collections.singletonList(ca.getInterpretation().getId()), + ownerToken); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, ca.getId(), QueryOptions.empty(), ownerToken).first(); assertNull(ca.getInterpretation()); } @Test public void automaticallyLockInterpretationTest() throws CatalogException { ClinicalAnalysis ca = createDummyEnvironment(true, false).first(); - Interpretation interpretation = catalogManager.getInterpretationManager().create(STUDY, ca.getId(), - new Interpretation(), ParamUtils.SaveInterpretationAs.PRIMARY, INCLUDE_RESULT, sessionIdUser).first(); - assertTrue(StringUtils.isEmpty(interpretation.getStatus().getId())); + Interpretation interpretation = catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), + new Interpretation(), ParamUtils.SaveInterpretationAs.PRIMARY, INCLUDE_RESULT, ownerToken).first(); + assertEquals(ClinicalStatusValue.ClinicalStatusType.NOT_STARTED, interpretation.getStatus().getType()); assertFalse(interpretation.isLocked()); - interpretation = catalogManager.getInterpretationManager().update(STUDY, ca.getId(), interpretation.getId(), - new InterpretationUpdateParams().setStatus(new StatusParam("READY")), null, INCLUDE_RESULT, sessionIdUser).first(); + interpretation = catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), interpretation.getId(), + new InterpretationUpdateParams().setStatus(new StatusParam("READY")), null, INCLUDE_RESULT, ownerToken).first(); assertEquals("READY", interpretation.getStatus().getId()); assertTrue(interpretation.isLocked()); - interpretation = catalogManager.getInterpretationManager().create(STUDY, ca.getId(), + interpretation = catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation() - .setStatus(new Status("REJECTED", "", "", "")), - ParamUtils.SaveInterpretationAs.PRIMARY, INCLUDE_RESULT, sessionIdUser).first(); + .setStatus(new ClinicalStatus("REJECTED", "", null, "", "", "", "")), + ParamUtils.SaveInterpretationAs.PRIMARY, INCLUDE_RESULT, ownerToken).first(); assertEquals("REJECTED", interpretation.getStatus().getId()); assertTrue(interpretation.isLocked()); } @@ -176,91 +121,124 @@ public void automaticallyLockInterpretationTest() throws CatalogException { public void interpretationLockedTest() throws CatalogException { ClinicalAnalysis ca = createDummyEnvironment(true, false).first(); - catalogManager.getInterpretationManager().create(STUDY, ca.getId(), new Interpretation().setLocked(true), - ParamUtils.SaveInterpretationAs.PRIMARY, QueryOptions.empty(), sessionIdUser); - catalogManager.getInterpretationManager().create(STUDY, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.SECONDARY, - QueryOptions.empty(), sessionIdUser); - catalogManager.getInterpretationManager().create(STUDY, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.SECONDARY, - QueryOptions.empty(), sessionIdUser); + catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation().setLocked(true), + ParamUtils.SaveInterpretationAs.PRIMARY, QueryOptions.empty(), ownerToken); + catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.SECONDARY, + QueryOptions.empty(), ownerToken); + catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation(), ParamUtils.SaveInterpretationAs.SECONDARY, + QueryOptions.empty(), ownerToken); - ca = catalogManager.getClinicalAnalysisManager().get(STUDY, ca.getId(), QueryOptions.empty(), sessionIdUser).first(); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, ca.getId(), QueryOptions.empty(), ownerToken).first(); assertTrue(ca.getInterpretation().isLocked()); for (Interpretation secondaryInterpretation : ca.getSecondaryInterpretations()) { assertFalse(secondaryInterpretation.isLocked()); } + // Add ADMIN permissions to the user2 + catalogManager.getStudyManager().updateAcl(studyFqn, normalUserId2, + new StudyAclParams(StudyPermissions.Permissions.ADMIN_CLINICAL_ANALYSIS.name(), null), ParamUtils.AclAction.SET, ownerToken); + // Try to update interpretation 1 try { - catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getInterpretation().getId(), - new InterpretationUpdateParams().setDescription("blabla"), null, QueryOptions.empty(), sessionIdUser); + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getInterpretation().getId(), + new InterpretationUpdateParams().setDescription("blabla"), null, QueryOptions.empty(), normalToken1); fail("Interpretation is locked so it should not allow this"); } catch (CatalogException e) { - assertTrue(e.getMessage().contains("locked")); + assertTrue(e.getMessage().contains(ClinicalAnalysisPermissions.ADMIN.name())); } + // Try to update interpretation 1 + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getInterpretation().getId(), + new InterpretationUpdateParams().setDescription("blabla"), null, QueryOptions.empty(), normalToken2); + Interpretation interpretation = catalogManager.getInterpretationManager().get(studyFqn, ca.getInterpretation().getId(), + QueryOptions.empty(), ownerToken).first(); + assertEquals("blabla", interpretation.getDescription()); + assertTrue(interpretation.isLocked()); + + // Try to update interpretation 1 + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getInterpretation().getId(), + new InterpretationUpdateParams().setDescription("blabla2"), null, QueryOptions.empty(), ownerToken); + interpretation = catalogManager.getInterpretationManager().get(studyFqn, ca.getInterpretation().getId(), QueryOptions.empty(), ownerToken).first(); + assertEquals("blabla2", interpretation.getDescription()); + assertTrue(interpretation.isLocked()); + // Update interpretation 2 - catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getSecondaryInterpretations().get(0).getId(), - new InterpretationUpdateParams().setDescription("blabla"), null, QueryOptions.empty(), sessionIdUser); - Interpretation interpretation2 = catalogManager.getInterpretationManager().get(STUDY, - ca.getSecondaryInterpretations().get(0).getId(), QueryOptions.empty(), sessionIdUser).first(); + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getSecondaryInterpretations().get(0).getId(), + new InterpretationUpdateParams().setDescription("blabla"), null, QueryOptions.empty(), ownerToken); + Interpretation interpretation2 = catalogManager.getInterpretationManager().get(studyFqn, + ca.getSecondaryInterpretations().get(0).getId(), QueryOptions.empty(), ownerToken).first(); assertEquals("blabla", interpretation2.getDescription()); assertFalse(interpretation2.isLocked()); - catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getSecondaryInterpretations().get(0).getId(), - new InterpretationUpdateParams().setDescription("bloblo").setLocked(true), null, QueryOptions.empty(), sessionIdUser); - interpretation2 = catalogManager.getInterpretationManager().get(STUDY, - ca.getSecondaryInterpretations().get(0).getId(), QueryOptions.empty(), sessionIdUser).first(); + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getSecondaryInterpretations().get(0).getId(), + new InterpretationUpdateParams().setDescription("bloblo").setLocked(true), null, QueryOptions.empty(), ownerToken); + interpretation2 = catalogManager.getInterpretationManager().get(studyFqn, + ca.getSecondaryInterpretations().get(0).getId(), QueryOptions.empty(), ownerToken).first(); assertEquals("bloblo", interpretation2.getDescription()); assertTrue(interpretation2.isLocked()); - // Try to lock again and update interpretation 2 - try { - catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getSecondaryInterpretations().get(0).getId(), - new InterpretationUpdateParams().setDescription("blabla").setLocked(true), null, QueryOptions.empty(), sessionIdUser); - fail("Interpretation was already locked so it should not allow this"); - } catch (CatalogException e) { - assertTrue(e.getMessage().contains("locked")); - } - // Unlock and update interpretation 2 - catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getSecondaryInterpretations().get(0).getId(), - new InterpretationUpdateParams().setDescription("blabla").setLocked(false), null, QueryOptions.empty(), sessionIdUser); - interpretation2 = catalogManager.getInterpretationManager().get(STUDY, - ca.getSecondaryInterpretations().get(0).getId(), QueryOptions.empty(), sessionIdUser).first(); + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getSecondaryInterpretations().get(0).getId(), + new InterpretationUpdateParams().setDescription("blabla").setLocked(false), null, QueryOptions.empty(), ownerToken); + interpretation2 = catalogManager.getInterpretationManager().get(studyFqn, + ca.getSecondaryInterpretations().get(0).getId(), QueryOptions.empty(), ownerToken).first(); assertEquals("blabla", interpretation2.getDescription()); assertFalse(interpretation2.isLocked()); // Lock and update interpretation 2 - catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getSecondaryInterpretations().get(0).getId(), - new InterpretationUpdateParams().setDescription("bloblo").setLocked(true), null, QueryOptions.empty(), sessionIdUser); - interpretation2 = catalogManager.getInterpretationManager().get(STUDY, - ca.getSecondaryInterpretations().get(0).getId(), QueryOptions.empty(), sessionIdUser).first(); + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getSecondaryInterpretations().get(0).getId(), + new InterpretationUpdateParams().setDescription("bloblo").setLocked(true), null, QueryOptions.empty(), ownerToken); + interpretation2 = catalogManager.getInterpretationManager().get(studyFqn, + ca.getSecondaryInterpretations().get(0).getId(), QueryOptions.empty(), ownerToken).first(); assertEquals("bloblo", interpretation2.getDescription()); assertTrue(interpretation2.isLocked()); // Lock case - catalogManager.getClinicalAnalysisManager().update(STUDY, ca.getId(), new ClinicalAnalysisUpdateParams().setLocked(true), - QueryOptions.empty(), sessionIdUser); - ca = catalogManager.getClinicalAnalysisManager().get(STUDY, ca.getId(), QueryOptions.empty(), sessionIdUser).first(); + catalogManager.getClinicalAnalysisManager().update(studyFqn, ca.getId(), new ClinicalAnalysisUpdateParams().setLocked(true), + QueryOptions.empty(), ownerToken); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, ca.getId(), QueryOptions.empty(), ownerToken).first(); assertTrue(ca.isLocked()); assertTrue(ca.getInterpretation().isLocked()); for (Interpretation secondaryInterpretation : ca.getSecondaryInterpretations()) { assertTrue(secondaryInterpretation.isLocked()); } + // Try to update the interpretation 1 + try { + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getInterpretation().getId(), + new InterpretationUpdateParams().setDescription("new description"), null, QueryOptions.empty(), normalToken1); + fail("Case and Interpretation are locked so it should not allow this"); + } catch (CatalogException e) { + assertTrue(e.getMessage().contains(ClinicalAnalysisPermissions.ADMIN.name()) && e.getMessage().toLowerCase().contains("permission denied")); + } + + // Try to update the interpretation 1 (ADMIN permission) + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getInterpretation().getId(), + new InterpretationUpdateParams().setDescription("new description"), null, QueryOptions.empty(), normalToken2); + interpretation = catalogManager.getInterpretationManager().get(studyFqn, ca.getInterpretation().getId(), QueryOptions.empty(), ownerToken).first(); + assertEquals("new description", interpretation.getDescription()); + assertTrue(interpretation.isLocked()); + + // Try to update the interpretation 1 (owner user) + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getInterpretation().getId(), + new InterpretationUpdateParams().setDescription("new description2"), null, QueryOptions.empty(), ownerToken); + interpretation = catalogManager.getInterpretationManager().get(studyFqn, ca.getInterpretation().getId(), QueryOptions.empty(), ownerToken).first(); + assertEquals("new description2", interpretation.getDescription()); + assertTrue(interpretation.isLocked()); + // Try to unlock interpretation 1 try { - catalogManager.getInterpretationManager().update(STUDY, ca.getId(), ca.getInterpretation().getId(), - new InterpretationUpdateParams().setLocked(false), null, QueryOptions.empty(), sessionIdUser); + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), ca.getInterpretation().getId(), + new InterpretationUpdateParams().setLocked(false), null, QueryOptions.empty(), ownerToken); fail("Case is locked so it should not allow this"); } catch (CatalogException e) { assertTrue(e.getMessage().contains("locked") && e.getMessage().toLowerCase().contains("case")); } // Unlock case - catalogManager.getClinicalAnalysisManager().update(STUDY, ca.getId(), new ClinicalAnalysisUpdateParams().setLocked(false), - QueryOptions.empty(), sessionIdUser); - ca = catalogManager.getClinicalAnalysisManager().get(STUDY, ca.getId(), QueryOptions.empty(), sessionIdUser).first(); + catalogManager.getClinicalAnalysisManager().update(studyFqn, ca.getId(), new ClinicalAnalysisUpdateParams().setLocked(false), + QueryOptions.empty(), ownerToken); + ca = catalogManager.getClinicalAnalysisManager().get(studyFqn, ca.getId(), QueryOptions.empty(), ownerToken).first(); assertFalse(ca.isLocked()); assertTrue(ca.getInterpretation().isLocked()); for (Interpretation secondaryInterpretation : ca.getSecondaryInterpretations()) { @@ -268,6 +246,43 @@ public void interpretationLockedTest() throws CatalogException { } } + @Test + public void interpretationStatusTest() throws CatalogException { + ClinicalAnalysis ca = createDummyEnvironment(true, false).first(); + + Interpretation interpretation = catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), new Interpretation(), + ParamUtils.SaveInterpretationAs.PRIMARY, INCLUDE_RESULT, ownerToken).first(); + + // Create 2 allowed statuses of type CLOSED + ClinicalAnalysisStudyConfiguration studyConfiguration = ClinicalAnalysisStudyConfiguration.defaultConfiguration(); + List statusValueList = new ArrayList<>(); + for (ClinicalStatusValue status : studyConfiguration.getInterpretation().getStatus()) { + if (!status.getType().equals(ClinicalStatusValue.ClinicalStatusType.CLOSED)) { + statusValueList.add(status); + } + } + // Add two statuses of type CLOSED + statusValueList.add(new ClinicalStatusValue("closed1", "my desc", ClinicalStatusValue.ClinicalStatusType.CLOSED)); + statusValueList.add(new ClinicalStatusValue("closed2", "my desc", ClinicalStatusValue.ClinicalStatusType.CLOSED)); + studyConfiguration.getInterpretation().setStatus(statusValueList); + catalogManager.getClinicalAnalysisManager().configureStudy(studyFqn, studyConfiguration, studyAdminToken1); + + // Update status to one of the new statuses + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), interpretation.getId(), + new InterpretationUpdateParams().setStatus(new StatusParam("closed1")), null, QueryOptions.empty(), studyAdminToken1); + interpretation = catalogManager.getInterpretationManager().get(studyFqn, interpretation.getId(), QueryOptions.empty(), studyAdminToken1).first(); + assertEquals("closed1", interpretation.getStatus().getId()); + assertEquals(ClinicalStatusValue.ClinicalStatusType.CLOSED, interpretation.getStatus().getType()); + assertTrue(interpretation.isLocked()); + + // Update status to the other new CLOSED status + catalogManager.getInterpretationManager().update(studyFqn, ca.getId(), interpretation.getId(), + new InterpretationUpdateParams().setStatus(new StatusParam("closed2")), null, QueryOptions.empty(), studyAdminToken1); + assertEquals("closed1", interpretation.getStatus().getId()); + assertEquals(ClinicalStatusValue.ClinicalStatusType.CLOSED, interpretation.getStatus().getType()); + assertTrue(interpretation.isLocked()); + } + @Test public void createInterpretationWithSubsetOfPanels() throws CatalogException { ClinicalAnalysis ca = createDummyEnvironment(true, false).first(); @@ -276,24 +291,24 @@ public void createInterpretationWithSubsetOfPanels() throws CatalogException { for (int i = 0; i < 3; i++) { Panel panel = new Panel().setId("panel" + i); panelReferenceParamList.add(new PanelReferenceParam(panel.getId())); - catalogManager.getPanelManager().create(STUDY, panel, QueryOptions.empty(), sessionIdUser); + catalogManager.getPanelManager().create(studyFqn, panel, QueryOptions.empty(), ownerToken); } // Add panels to the case and set panelLock to true ClinicalAnalysisUpdateParams updateParams = new ClinicalAnalysisUpdateParams() .setPanels(panelReferenceParamList); - catalogManager.getClinicalAnalysisManager().update(STUDY, ca.getId(), updateParams, QueryOptions.empty(), sessionIdUser); + catalogManager.getClinicalAnalysisManager().update(studyFqn, ca.getId(), updateParams, QueryOptions.empty(), ownerToken); updateParams = new ClinicalAnalysisUpdateParams() - .setPanelLock(true); - catalogManager.getClinicalAnalysisManager().update(STUDY, ca.getId(), updateParams, QueryOptions.empty(), sessionIdUser); + .setPanelLocked(true); + catalogManager.getClinicalAnalysisManager().update(studyFqn, ca.getId(), updateParams, QueryOptions.empty(), ownerToken); // Create interpretation with just panel1 InterpretationCreateParams interpretationCreateParams = new InterpretationCreateParams() .setPanels(panelReferenceParamList.subList(0, 1)); - Interpretation interpretation = catalogManager.getInterpretationManager().create(STUDY, ca.getId(), + Interpretation interpretation = catalogManager.getInterpretationManager().create(studyFqn, ca.getId(), interpretationCreateParams.toClinicalInterpretation(), ParamUtils.SaveInterpretationAs.PRIMARY, - new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true), sessionIdUser).first(); + new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true), ownerToken).first(); assertEquals(1, interpretation.getPanels().size()); assertEquals(panelReferenceParamList.get(0).getId(), interpretation.getPanels().get(0).getId()); } diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/NoteManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/NoteManagerTest.java new file mode 100644 index 00000000000..9d1afcadb2b --- /dev/null +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/NoteManagerTest.java @@ -0,0 +1,397 @@ +package org.opencb.opencga.catalog.managers; + +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.catalog.db.api.NoteDBAdaptor; +import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.utils.Constants; +import org.opencb.opencga.catalog.utils.ParamUtils; +import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.models.notes.Note; +import org.opencb.opencga.core.models.notes.NoteCreateParams; +import org.opencb.opencga.core.models.notes.NoteUpdateParams; +import org.opencb.opencga.core.models.organizations.Organization; +import org.opencb.opencga.core.models.study.Study; +import org.opencb.opencga.core.response.OpenCGAResult; +import org.opencb.opencga.core.testclassification.duration.MediumTests; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.*; + +@Category(MediumTests.class) +public class NoteManagerTest extends AbstractManagerTest { + + @Test + public void createOrganizationNoteTest() throws CatalogException { + NoteCreateParams noteCreateParams = new NoteCreateParams() + .setId("note1") + .setVisibility(Note.Visibility.PRIVATE) + .setValueType(Note.Type.STRING) + .setValue("hello"); + Note note = catalogManager.getNotesManager().createOrganizationNote(noteCreateParams, INCLUDE_RESULT, ownerToken).first(); + assertEquals(noteCreateParams.getId(), note.getId()); + assertEquals(orgOwnerUserId, note.getUserId()); + + noteCreateParams.setId("note2"); + note = catalogManager.getNotesManager().createOrganizationNote(noteCreateParams, INCLUDE_RESULT, orgAdminToken1).first(); + assertEquals(noteCreateParams.getId(), note.getId()); + assertEquals(orgAdminUserId1, note.getUserId()); + + thrown.expect(CatalogAuthorizationException.class); + thrown.expectMessage("denied"); + catalogManager.getNotesManager().createOrganizationNote(noteCreateParams, INCLUDE_RESULT, studyAdminToken1).first(); + } + + @Test + public void updateOrganizationNoteTest() throws CatalogException { + NoteCreateParams noteCreateParams = new NoteCreateParams() + .setId("note1") + .setVisibility(Note.Visibility.PRIVATE) + .setValueType(Note.Type.STRING) + .setValue("hello"); + Note note = catalogManager.getNotesManager().createOrganizationNote(noteCreateParams, INCLUDE_RESULT, ownerToken).first(); + assertEquals(1, note.getVersion()); + + NoteUpdateParams noteUpdateParams = new NoteUpdateParams() + .setTags(Arrays.asList("tag1", "tag2")); + note = catalogManager.getNotesManager().updateOrganizationNote(note.getId(), noteUpdateParams, INCLUDE_RESULT, ownerToken).first(); + assertEquals(2, note.getVersion()); + assertEquals(orgOwnerUserId, note.getUserId()); + assertEquals(2, note.getTags().size()); + assertArrayEquals(noteUpdateParams.getTags().toArray(), note.getTags().toArray()); + + noteUpdateParams = new NoteUpdateParams() + .setVisibility(Note.Visibility.PUBLIC); + note = catalogManager.getNotesManager().updateOrganizationNote(note.getId(), noteUpdateParams, INCLUDE_RESULT, orgAdminToken1).first(); + assertEquals(3, note.getVersion()); + assertEquals(orgAdminUserId1, note.getUserId()); + assertEquals(noteUpdateParams.getVisibility(), note.getVisibility()); + + thrown.expect(CatalogAuthorizationException.class); + thrown.expectMessage("denied"); + catalogManager.getNotesManager().updateOrganizationNote(note.getId(), noteUpdateParams, INCLUDE_RESULT, studyAdminToken1).first(); + } + + @Test + public void getOrganizationNoteTest() throws CatalogException { + NoteCreateParams noteCreateParams1 = new NoteCreateParams() + .setId("note1") + .setVisibility(Note.Visibility.PRIVATE) + .setValueType(Note.Type.STRING) + .setValue("hello"); + catalogManager.getNotesManager().createOrganizationNote(noteCreateParams1, INCLUDE_RESULT, ownerToken).first(); + + NoteCreateParams noteCreateParams2 = new NoteCreateParams() + .setId("note2") + .setVisibility(Note.Visibility.PUBLIC) + .setValueType(Note.Type.STRING) + .setValue("hello"); + catalogManager.getNotesManager().createOrganizationNote(noteCreateParams2, INCLUDE_RESULT, ownerToken).first(); + + Organization organization = catalogManager.getOrganizationManager().get(organizationId, null, ownerToken).first(); + assertEquals(2, organization.getNotes().size()); + assertEquals(noteCreateParams1.getId(), organization.getNotes().get(0).getId()); + assertEquals(noteCreateParams2.getId(), organization.getNotes().get(1).getId()); + + OpenCGAResult result = catalogManager.getNotesManager().searchOrganizationNote(new Query(), new QueryOptions(), ownerToken); + assertEquals(2, result.getNumResults()); + assertEquals(noteCreateParams1.getId(), result.getResults().get(0).getId()); + assertEquals(noteCreateParams2.getId(), result.getResults().get(1).getId()); + + // Check accessibility to private notes + organization = catalogManager.getOrganizationManager().get(organizationId, null, orgAdminToken1).first(); + assertEquals(2, organization.getNotes().size()); + assertEquals(noteCreateParams1.getId(), organization.getNotes().get(0).getId()); + assertEquals(noteCreateParams2.getId(), organization.getNotes().get(1).getId()); + + result = catalogManager.getNotesManager().searchOrganizationNote(new Query(), new QueryOptions(), orgAdminToken1); + assertEquals(2, result.getNumResults()); + assertEquals(noteCreateParams1.getId(), result.getResults().get(0).getId()); + assertEquals(noteCreateParams2.getId(), result.getResults().get(1).getId()); + + organization = catalogManager.getOrganizationManager().get(organizationId, null, studyAdminToken1).first(); + assertEquals(1, organization.getNotes().size()); + assertEquals(noteCreateParams2.getId(), organization.getNotes().get(0).getId()); + + result = catalogManager.getNotesManager().searchOrganizationNote(new Query(), new QueryOptions(), studyAdminToken1); + assertEquals(1, result.getNumResults()); + assertEquals(noteCreateParams2.getId(), result.getResults().get(0).getId()); + + organization = catalogManager.getOrganizationManager().get(organizationId, null, normalToken1).first(); + assertEquals(1, organization.getNotes().size()); + assertEquals(noteCreateParams2.getId(), organization.getNotes().get(0).getId()); + + result = catalogManager.getNotesManager().searchOrganizationNote(new Query(), new QueryOptions(), normalToken1); + assertEquals(1, result.getNumResults()); + assertEquals(noteCreateParams2.getId(), result.getResults().get(0).getId()); + } + + @Test + public void createStudyNoteTest() throws CatalogException { + NoteCreateParams noteCreateParams = new NoteCreateParams() + .setId("note1") + .setVisibility(Note.Visibility.PRIVATE) + .setValueType(Note.Type.STRING) + .setValue("hello"); + Note note = catalogManager.getNotesManager().createStudyNote(studyFqn, noteCreateParams, INCLUDE_RESULT, ownerToken).first(); + assertEquals(noteCreateParams.getId(), note.getId()); + assertEquals(orgOwnerUserId, note.getUserId()); + + noteCreateParams.setId("note2"); + note = catalogManager.getNotesManager().createStudyNote(studyFqn, noteCreateParams, INCLUDE_RESULT, orgAdminToken1).first(); + assertEquals(noteCreateParams.getId(), note.getId()); + assertEquals(orgAdminUserId1, note.getUserId()); + + noteCreateParams.setId("note3"); + note = catalogManager.getNotesManager().createStudyNote(studyFqn, noteCreateParams, INCLUDE_RESULT, studyAdminToken1).first(); + assertEquals(noteCreateParams.getId(), note.getId()); + assertEquals(studyAdminUserId1, note.getUserId()); + + thrown.expect(CatalogAuthorizationException.class); + thrown.expectMessage("denied"); + catalogManager.getNotesManager().createStudyNote(studyFqn, noteCreateParams, INCLUDE_RESULT, normalToken1).first(); + } + + @Test + public void updateStudyNoteTest() throws CatalogException { + NoteCreateParams noteCreateParams = new NoteCreateParams() + .setId("note1") + .setVisibility(Note.Visibility.PRIVATE) + .setValueType(Note.Type.STRING) + .setValue("hello"); + Note note = catalogManager.getNotesManager().createStudyNote(studyFqn, noteCreateParams, INCLUDE_RESULT, ownerToken).first(); + assertEquals(1, note.getVersion()); + + NoteUpdateParams noteUpdateParams = new NoteUpdateParams() + .setTags(Arrays.asList("tag1", "tag2")); + note = catalogManager.getNotesManager().updateStudyNote(studyFqn, note.getId(), noteUpdateParams, INCLUDE_RESULT, ownerToken).first(); + assertEquals(2, note.getVersion()); + assertEquals(orgOwnerUserId, note.getUserId()); + assertEquals(2, note.getTags().size()); + assertArrayEquals(noteUpdateParams.getTags().toArray(), note.getTags().toArray()); + + noteUpdateParams = new NoteUpdateParams() + .setVisibility(Note.Visibility.PUBLIC); + note = catalogManager.getNotesManager().updateStudyNote(studyFqn, note.getId(), noteUpdateParams, INCLUDE_RESULT, orgAdminToken1).first(); + assertEquals(3, note.getVersion()); + assertEquals(orgAdminUserId1, note.getUserId()); + assertEquals(noteUpdateParams.getVisibility(), note.getVisibility()); + + noteUpdateParams = new NoteUpdateParams() + .setValue("my new value"); + note = catalogManager.getNotesManager().updateStudyNote(studyFqn, note.getId(), noteUpdateParams, INCLUDE_RESULT, studyAdminToken1).first(); + assertEquals(4, note.getVersion()); + assertEquals(studyAdminUserId1, note.getUserId()); + assertEquals(noteUpdateParams.getValue(), note.getValue()); + + thrown.expect(CatalogAuthorizationException.class); + thrown.expectMessage("denied"); + catalogManager.getNotesManager().updateStudyNote(studyFqn, note.getId(), noteUpdateParams, INCLUDE_RESULT, normalToken1); + } + + @Test + public void noteTagsUpdateTest() throws CatalogException { + NoteCreateParams noteCreateParams = new NoteCreateParams() + .setId("note1") + .setVisibility(Note.Visibility.PRIVATE) + .setValueType(Note.Type.STRING) + .setTags(Arrays.asList("tag1", "tag2")) + .setValue("hello"); + Note note = catalogManager.getNotesManager().createStudyNote(studyFqn, noteCreateParams, INCLUDE_RESULT, ownerToken).first(); + assertEquals(2, note.getTags().size()); + assertArrayEquals(Arrays.asList("tag1", "tag2").toArray(), note.getTags().toArray()); + + QueryOptions queryOptions = new QueryOptions(); + Map actionMap = new HashMap<>(); + actionMap.put(NoteDBAdaptor.QueryParams.TAGS.key(), ParamUtils.BasicUpdateAction.ADD); + queryOptions.put(Constants.ACTIONS, actionMap); + queryOptions.put(ParamConstants.INCLUDE_RESULT_PARAM, true); + NoteUpdateParams updateParams = new NoteUpdateParams().setTags(Arrays.asList("tag3", "tag1")); + note = catalogManager.getNotesManager().updateStudyNote(studyFqn, note.getId(), updateParams, queryOptions, ownerToken).first(); + assertEquals(3, note.getTags().size()); + assertArrayEquals(Arrays.asList("tag1", "tag2", "tag3").toArray(), note.getTags().toArray()); + + // Remove tag1 and tag2 + actionMap.put(NoteDBAdaptor.QueryParams.TAGS.key(), ParamUtils.BasicUpdateAction.REMOVE); + queryOptions.put(Constants.ACTIONS, actionMap); + updateParams = new NoteUpdateParams().setTags(Arrays.asList("tag1", "tag2")); + note = catalogManager.getNotesManager().updateStudyNote(studyFqn, note.getId(), updateParams, queryOptions, ownerToken).first(); + assertEquals(1, note.getTags().size()); + assertArrayEquals(Arrays.asList("tag3").toArray(), note.getTags().toArray()); + + // Set new list of tags + actionMap.put(NoteDBAdaptor.QueryParams.TAGS.key(), ParamUtils.BasicUpdateAction.SET); + queryOptions.put(Constants.ACTIONS, actionMap); + updateParams = new NoteUpdateParams().setTags(Arrays.asList("tag4", "tag5")); + note = catalogManager.getNotesManager().updateStudyNote(studyFqn, note.getId(), updateParams, queryOptions, ownerToken).first(); + assertEquals(2, note.getTags().size()); + assertArrayEquals(Arrays.asList("tag4", "tag5").toArray(), note.getTags().toArray()); + } + + @Test + public void getStudyNoteTest() throws CatalogException { + NoteCreateParams noteCreateParams1 = new NoteCreateParams() + .setId("note1") + .setVisibility(Note.Visibility.PRIVATE) + .setValueType(Note.Type.STRING) + .setValue("hello"); + catalogManager.getNotesManager().createStudyNote(studyFqn, noteCreateParams1, QueryOptions.empty(), ownerToken); + + NoteCreateParams noteCreateParams2 = new NoteCreateParams() + .setId("note2") + .setVisibility(Note.Visibility.PUBLIC) + .setValueType(Note.Type.STRING) + .setValue("hello"); + catalogManager.getNotesManager().createStudyNote(studyFqn, noteCreateParams2, QueryOptions.empty(), ownerToken); + + Study study = catalogManager.getStudyManager().get(studyFqn, null, ownerToken).first(); + assertEquals(2, study.getNotes().size()); + assertEquals(noteCreateParams1.getId(), study.getNotes().get(0).getId()); + assertEquals(noteCreateParams2.getId(), study.getNotes().get(1).getId()); + + OpenCGAResult result = catalogManager.getNotesManager().searchStudyNote(studyFqn, new Query(), new QueryOptions(), ownerToken); + assertEquals(2, result.getNumResults()); + assertEquals(noteCreateParams1.getId(), result.getResults().get(0).getId()); + assertEquals(noteCreateParams2.getId(), result.getResults().get(1).getId()); + + // Check accessibility to private notes + study = catalogManager.getStudyManager().get(studyFqn, null, orgAdminToken1).first(); + assertEquals(2, study.getNotes().size()); + assertEquals(noteCreateParams1.getId(), study.getNotes().get(0).getId()); + assertEquals(noteCreateParams2.getId(), study.getNotes().get(1).getId()); + + result = catalogManager.getNotesManager().searchStudyNote(studyFqn, new Query(), new QueryOptions(), orgAdminToken1); + assertEquals(2, result.getNumResults()); + assertEquals(noteCreateParams1.getId(), result.getResults().get(0).getId()); + assertEquals(noteCreateParams2.getId(), result.getResults().get(1).getId()); + + study = catalogManager.getStudyManager().get(studyFqn, null, studyAdminToken1).first(); + assertEquals(2, study.getNotes().size()); + assertEquals(noteCreateParams1.getId(), study.getNotes().get(0).getId()); + assertEquals(noteCreateParams2.getId(), study.getNotes().get(1).getId()); + + result = catalogManager.getNotesManager().searchStudyNote(studyFqn, new Query(), new QueryOptions(), studyAdminToken1); + assertEquals(2, result.getNumResults()); + assertEquals(noteCreateParams1.getId(), result.getResults().get(0).getId()); + assertEquals(noteCreateParams2.getId(), result.getResults().get(1).getId()); + + study = catalogManager.getStudyManager().get(studyFqn, null, normalToken1).first(); + assertEquals(1, study.getNotes().size()); + assertEquals(noteCreateParams2.getId(), study.getNotes().get(0).getId()); + + result = catalogManager.getNotesManager().searchStudyNote(studyFqn, new Query(), new QueryOptions(), normalToken1); + assertEquals(1, result.getNumResults()); + assertEquals(noteCreateParams2.getId(), result.getResults().get(0).getId()); + } + + @Test + public void getNoteTest() throws CatalogException { + NoteCreateParams study1Note1 = new NoteCreateParams() + .setId("note1") + .setVisibility(Note.Visibility.PRIVATE) + .setValueType(Note.Type.STRING) + .setValue("study1Note1"); + catalogManager.getNotesManager().createStudyNote(studyFqn, study1Note1, QueryOptions.empty(), ownerToken); + + NoteCreateParams study1Note2 = new NoteCreateParams() + .setId("note2") + .setVisibility(Note.Visibility.PUBLIC) + .setValueType(Note.Type.STRING) + .setValue("study1Note2"); + catalogManager.getNotesManager().createStudyNote(studyFqn, study1Note2, QueryOptions.empty(), ownerToken); + + NoteCreateParams study2Note1 = new NoteCreateParams() + .setId("note1") + .setVisibility(Note.Visibility.PUBLIC) + .setValueType(Note.Type.STRING) + .setValue("study2Note1"); + catalogManager.getNotesManager().createStudyNote(studyFqn2, study2Note1, QueryOptions.empty(), ownerToken); + + NoteCreateParams orgNote1 = new NoteCreateParams() + .setId("note1") + .setVisibility(Note.Visibility.PRIVATE) + .setValueType(Note.Type.STRING) + .setValue("orgNote1"); + catalogManager.getNotesManager().createOrganizationNote(orgNote1, INCLUDE_RESULT, ownerToken); + + NoteCreateParams orgNote2 = new NoteCreateParams() + .setId("note2") + .setVisibility(Note.Visibility.PUBLIC) + .setValueType(Note.Type.STRING) + .setValue("orgNote2"); + catalogManager.getNotesManager().createOrganizationNote(orgNote2, INCLUDE_RESULT, ownerToken); + + Study study = catalogManager.getStudyManager().get(studyFqn, null, ownerToken).first(); + assertEquals(2, study.getNotes().size()); + assertEquals(study1Note1.getId(), study.getNotes().get(0).getId()); + assertEquals(study1Note1.getValue(), study.getNotes().get(0).getValue()); + assertEquals(study1Note2.getId(), study.getNotes().get(1).getId()); + assertEquals(study1Note2.getValue(), study.getNotes().get(1).getValue()); + + study = catalogManager.getStudyManager().get(studyFqn2, null, ownerToken).first(); + assertEquals(1, study.getNotes().size()); + assertEquals(study2Note1.getId(), study.getNotes().get(0).getId()); + assertEquals(study2Note1.getValue(), study.getNotes().get(0).getValue()); + + Organization organization = catalogManager.getOrganizationManager().get(organizationId, QueryOptions.empty(), ownerToken).first(); + assertEquals(2, organization.getNotes().size()); + assertEquals(orgNote1.getId(), organization.getNotes().get(0).getId()); + assertEquals(orgNote1.getValue(), organization.getNotes().get(0).getValue()); + assertEquals(orgNote2.getId(), organization.getNotes().get(1).getId()); + assertEquals(orgNote2.getValue(), organization.getNotes().get(1).getValue()); + } + + @Test + public void deleteNoteTest() throws CatalogException { + NoteCreateParams study1Note1 = new NoteCreateParams() + .setId("study1Note1") + .setVisibility(Note.Visibility.PRIVATE) + .setValueType(Note.Type.STRING) + .setValue("study1Note1"); + catalogManager.getNotesManager().createStudyNote(studyFqn, study1Note1, QueryOptions.empty(), ownerToken); + + NoteCreateParams study1Note2 = new NoteCreateParams() + .setId("study1Note2") + .setVisibility(Note.Visibility.PUBLIC) + .setValueType(Note.Type.STRING) + .setValue("study1Note2"); + catalogManager.getNotesManager().createStudyNote(studyFqn, study1Note2, QueryOptions.empty(), ownerToken); + + NoteCreateParams study2Note1 = new NoteCreateParams() + .setId("study2Note1") + .setVisibility(Note.Visibility.PUBLIC) + .setValueType(Note.Type.STRING) + .setValue("study2Note1"); + catalogManager.getNotesManager().createStudyNote(studyFqn2, study2Note1, QueryOptions.empty(), ownerToken); + + NoteCreateParams orgNote1 = new NoteCreateParams() + .setId("orgNote1") + .setVisibility(Note.Visibility.PRIVATE) + .setValueType(Note.Type.STRING) + .setValue("orgNote1"); + catalogManager.getNotesManager().createOrganizationNote(orgNote1, INCLUDE_RESULT, ownerToken); + + NoteCreateParams orgNote2 = new NoteCreateParams() + .setId("orgNote2") + .setVisibility(Note.Visibility.PUBLIC) + .setValueType(Note.Type.STRING) + .setValue("orgNote2"); + catalogManager.getNotesManager().createOrganizationNote(orgNote2, INCLUDE_RESULT, ownerToken); + + assertThrows(CatalogException.class, () -> catalogManager.getNotesManager().deleteOrganizationNote(study1Note1.getId(), null, ownerToken)); + + OpenCGAResult result = catalogManager.getNotesManager().searchStudyNote(studyFqn, new Query(NoteDBAdaptor.QueryParams.ID.key(), study1Note1.getId()), QueryOptions.empty(), ownerToken); + assertEquals(1, result.getNumResults()); + assertEquals(study1Note1.getId(), result.first().getId()); + catalogManager.getNotesManager().deleteStudyNote(studyFqn, study1Note1.getId(), null, ownerToken); + result = catalogManager.getNotesManager().searchStudyNote(studyFqn, new Query(NoteDBAdaptor.QueryParams.ID.key(), study1Note1.getId()), QueryOptions.empty(), ownerToken); + assertEquals(0, result.getNumResults()); + } + +} diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/OrganizationManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/OrganizationManagerTest.java new file mode 100644 index 00000000000..a72d52eace3 --- /dev/null +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/OrganizationManagerTest.java @@ -0,0 +1,433 @@ +package org.opencb.opencga.catalog.managers; + +import org.apache.commons.collections4.CollectionUtils; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.catalog.auth.authentication.CatalogAuthenticationManager; +import org.opencb.opencga.catalog.db.api.OrganizationDBAdaptor; +import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; +import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; +import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.exceptions.CatalogParameterException; +import org.opencb.opencga.catalog.utils.Constants; +import org.opencb.opencga.catalog.utils.ParamUtils; +import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.config.AuthenticationOrigin; +import org.opencb.opencga.core.config.Optimizations; +import org.opencb.opencga.core.models.organizations.*; +import org.opencb.opencga.core.models.project.Project; +import org.opencb.opencga.core.models.study.Group; +import org.opencb.opencga.core.models.study.Study; +import org.opencb.opencga.core.models.user.OrganizationUserUpdateParams; +import org.opencb.opencga.core.models.user.User; +import org.opencb.opencga.core.models.user.UserQuota; +import org.opencb.opencga.core.response.OpenCGAResult; +import org.opencb.opencga.core.testclassification.duration.MediumTests; + +import java.util.*; + +import static org.junit.Assert.*; + +@Category(MediumTests.class) +public class OrganizationManagerTest extends AbstractManagerTest { + + @Test + public void ensureAuthOriginExistsTest() throws CatalogException { + Organization organization = catalogManager.getOrganizationManager().get(organizationId, null, ownerToken).first(); + assertFalse(organization.getConfiguration().getAuthenticationOrigins().isEmpty()); + assertNotNull(organization.getConfiguration().getAuthenticationOrigins().get(0)); + } + + @Test + public void updateConfigurationAuthorizationTest() throws CatalogException { + OrganizationConfiguration configuration = new OrganizationConfiguration() + .setOptimizations(new Optimizations(true)); + + // Owner can update configuration + catalogManager.getOrganizationManager().updateConfiguration(organizationId, configuration, QueryOptions.empty(), ownerToken); + + // Admin can update configuration + catalogManager.getOrganizationManager().updateConfiguration(organizationId, configuration, QueryOptions.empty(), orgAdminToken1); + + // Admin can update configuration + catalogManager.getOrganizationManager().updateConfiguration(organizationId, configuration, QueryOptions.empty(), orgAdminToken2); + + // Study admin cannot update configuration + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getOrganizationManager().updateConfiguration(organizationId, + configuration, QueryOptions.empty(), studyAdminToken1)); + + // Normal user cannot update configuration + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getOrganizationManager().updateConfiguration(organizationId, + configuration, QueryOptions.empty(), normalToken1)); + } + + @Test + public void ensureAuthOriginCannotBeRemovedTest() throws CatalogException { + Organization organization = catalogManager.getOrganizationManager().get(organizationId, null, ownerToken).first(); + + OrganizationConfiguration configuration = new OrganizationConfiguration() + .setAuthenticationOrigins(organization.getConfiguration().getAuthenticationOrigins()); + Map actionMap = new HashMap<>(); + actionMap.put(OrganizationDBAdaptor.AUTH_ORIGINS_FIELD, ParamUtils.UpdateAction.REMOVE); + QueryOptions options = new QueryOptions(Constants.ACTIONS, actionMap); + + CatalogException catalogException = assertThrows(CatalogException.class, () -> catalogManager.getOrganizationManager() + .updateConfiguration(organizationId, configuration, options, ownerToken)); + assertTrue(catalogException.getMessage().contains("user account uses")); + } + + @Test + public void authOriginActionTest() throws CatalogException { + OrganizationConfiguration configuration = new OrganizationConfiguration() + .setAuthenticationOrigins(Collections.singletonList(new AuthenticationOrigin("myId", + AuthenticationOrigin.AuthenticationType.SSO, null, null))); + Map actionMap = new HashMap<>(); + actionMap.put(OrganizationDBAdaptor.AUTH_ORIGINS_FIELD, ParamUtils.UpdateAction.ADD); + QueryOptions options = new QueryOptions() + .append(ParamConstants.INCLUDE_RESULT_PARAM, true) + .append(Constants.ACTIONS, actionMap); + + OrganizationConfiguration configurationResult = catalogManager.getOrganizationManager().updateConfiguration(organizationId, + configuration, options, ownerToken).first(); + assertEquals(2, configurationResult.getAuthenticationOrigins().size()); + for (AuthenticationOrigin authenticationOrigin : configurationResult.getAuthenticationOrigins()) { + if (authenticationOrigin.getId().equals("myId")) { + assertEquals(AuthenticationOrigin.AuthenticationType.SSO, authenticationOrigin.getType()); + } else { + assertEquals(AuthenticationOrigin.AuthenticationType.OPENCGA, authenticationOrigin.getType()); + } + } + + // Remove authOrigin + actionMap.put(OrganizationDBAdaptor.AUTH_ORIGINS_FIELD, ParamUtils.UpdateAction.REMOVE); + options.put(Constants.ACTIONS, actionMap); + configurationResult = catalogManager.getOrganizationManager().updateConfiguration(organizationId, configuration, options, + ownerToken).first(); + assertEquals(1, configurationResult.getAuthenticationOrigins().size()); + assertEquals(AuthenticationOrigin.AuthenticationType.OPENCGA, configurationResult.getAuthenticationOrigins().get(0).getType()); + + // Set authOrigin + List authenticationOriginList = new ArrayList<>(); + authenticationOriginList.add(new AuthenticationOrigin("myId", AuthenticationOrigin.AuthenticationType.SSO, null, null)); + authenticationOriginList.add(configurationResult.getAuthenticationOrigins().get(0)); + actionMap.put(OrganizationDBAdaptor.AUTH_ORIGINS_FIELD, ParamUtils.UpdateAction.SET); + options.put(Constants.ACTIONS, actionMap); + configuration.setAuthenticationOrigins(authenticationOriginList); + configurationResult = catalogManager.getOrganizationManager().updateConfiguration(organizationId, configuration, options, + ownerToken).first(); + assertEquals(2, configurationResult.getAuthenticationOrigins().size()); + for (AuthenticationOrigin authenticationOrigin : configurationResult.getAuthenticationOrigins()) { + if (authenticationOrigin.getId().equals("myId")) { + assertEquals(AuthenticationOrigin.AuthenticationType.SSO, authenticationOrigin.getType()); + } else { + assertEquals(AuthenticationOrigin.AuthenticationType.OPENCGA, authenticationOrigin.getType()); + } + } + + // Add existing authOrigin + actionMap.put(OrganizationDBAdaptor.AUTH_ORIGINS_FIELD, ParamUtils.UpdateAction.ADD); + options.put(Constants.ACTIONS, actionMap); + CatalogException catalogException = assertThrows(CatalogException.class, () -> catalogManager.getOrganizationManager() + .updateConfiguration(organizationId, configuration, options, ownerToken)); + assertTrue(catalogException.getMessage().contains("REPLACE")); + + // Replace existing authOrigin + actionMap.put(OrganizationDBAdaptor.AUTH_ORIGINS_FIELD, ParamUtils.UpdateAction.REPLACE); + options.put(Constants.ACTIONS, actionMap); + configuration.setAuthenticationOrigins(Collections.singletonList( + new AuthenticationOrigin(CatalogAuthenticationManager.OPENCGA, AuthenticationOrigin.AuthenticationType.OPENCGA, null, new ObjectMap("key", "value")))); + configurationResult = catalogManager.getOrganizationManager().updateConfiguration(organizationId, configuration, options, ownerToken).first(); + assertEquals(2, configurationResult.getAuthenticationOrigins().size()); + for (AuthenticationOrigin authenticationOrigin : configurationResult.getAuthenticationOrigins()) { + if (authenticationOrigin.getId().equals("myId")) { + assertEquals(AuthenticationOrigin.AuthenticationType.SSO, authenticationOrigin.getType()); + } else { + assertEquals(AuthenticationOrigin.AuthenticationType.OPENCGA, authenticationOrigin.getType()); + assertTrue(authenticationOrigin.getOptions().containsKey("key")); + assertEquals("value", authenticationOrigin.getOptions().get("key")); + } + } + } + + @Test + public void tokenUpdateTest() throws CatalogException { + TokenConfiguration tokenConfiguration = TokenConfiguration.init(); + OrganizationConfiguration configuration = new OrganizationConfiguration().setToken(tokenConfiguration); + OrganizationConfiguration configurationResult = catalogManager.getOrganizationManager().updateConfiguration(organizationId, + configuration, INCLUDE_RESULT, ownerToken).first(); + assertEquals(tokenConfiguration.getSecretKey(), configurationResult.getToken().getSecretKey()); + + assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getOrganizationManager().get(organizationId, null, ownerToken)); + } + + @Test + public void createNewOrganizationTest() throws CatalogException { + OpenCGAResult result = catalogManager.getOrganizationManager().create(new OrganizationCreateParams().setId("org2"), INCLUDE_RESULT, opencgaToken); + assertEquals(1, result.getNumInserted()); + assertEquals(1, result.getNumMatches()); + assertEquals("org2", result.first().getId()); + } + + @Test + public void organizationInfoIncludeProjectsTest() throws CatalogException { + Organization organization = catalogManager.getOrganizationManager().get(organizationId, QueryOptions.empty(), ownerToken).first(); + assertEquals(3, organization.getProjects().size()); + for (Project project : organization.getProjects()) { + assertNotNull(project.getFqn()); + assertNotNull(project.getName()); + assertNotNull(project.getCreationDate()); + } + + QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, "name"); + organization = catalogManager.getOrganizationManager().get(organizationId, options, ownerToken).first(); + assertNull(organization.getProjects()); + + options = new QueryOptions(QueryOptions.EXCLUDE, "projects"); + organization = catalogManager.getOrganizationManager().get(organizationId, options, ownerToken).first(); + assertNull(organization.getProjects()); + + options = new QueryOptions(QueryOptions.INCLUDE, "projects.fqn"); + organization = catalogManager.getOrganizationManager().get(organizationId, options, ownerToken).first(); + assertEquals(3, organization.getProjects().size()); + for (Project project : organization.getProjects()) { + assertNotNull(project.getFqn()); + assertNull(project.getName()); + assertNull(project.getCreationDate()); + } + + options = new QueryOptions(QueryOptions.EXCLUDE, "projects.name"); + organization = catalogManager.getOrganizationManager().get(organizationId, options, ownerToken).first(); + assertEquals(3, organization.getProjects().size()); + for (Project project : organization.getProjects()) { + assertNotNull(project.getFqn()); + assertNull(project.getName()); + assertNotNull(project.getCreationDate()); + } + } + + @Test + public void organizationInfoProjectsAuthTest() throws CatalogException { + Organization organization = catalogManager.getOrganizationManager().get(organizationId, QueryOptions.empty(), ownerToken).first(); + assertEquals(3, organization.getProjects().size()); + + organization = catalogManager.getOrganizationManager().get(organizationId, QueryOptions.empty(), orgAdminToken1).first(); + assertEquals(3, organization.getProjects().size()); + + organization = catalogManager.getOrganizationManager().get(organizationId, QueryOptions.empty(), studyAdminToken1).first(); + assertEquals(2, organization.getProjects().size()); + assertArrayEquals(Arrays.asList(projectFqn1, projectFqn2).toArray(), + organization.getProjects().stream().map(Project::getFqn).toArray()); + + organization = catalogManager.getOrganizationManager().get(organizationId, QueryOptions.empty(), normalToken1).first(); + assertEquals(1, organization.getProjects().size()); + assertEquals(projectFqn1, organization.getProjects().get(0).getFqn()); + + organization = catalogManager.getOrganizationManager().get(organizationId, QueryOptions.empty(), noAccessToken1).first(); + assertEquals(1, organization.getProjects().size()); + assertEquals(projectFqn1, organization.getProjects().get(0).getFqn()); + } + + @Test + public void validateUserUpdateParamsTest() { + OrganizationUserUpdateParams expiredDateParam = new OrganizationUserUpdateParams() + .setAccount(new OrganizationUserUpdateParams.Account("20200101100000")); + CatalogParameterException exception = assertThrows(CatalogParameterException.class, () -> catalogManager.getOrganizationManager() + .updateUser(organizationId, normalUserId1, expiredDateParam, INCLUDE_RESULT, ownerToken)); + assertTrue(exception.getMessage().contains("expired")); + + OrganizationUserUpdateParams invalidMailParam = new OrganizationUserUpdateParams() + .setEmail("invalidEmail"); + exception = assertThrows(CatalogParameterException.class, () -> catalogManager.getOrganizationManager().updateUser(organizationId, + normalUserId1, invalidMailParam, INCLUDE_RESULT, ownerToken)); + assertTrue(exception.getMessage().contains("not valid")); + } + + @Test + public void updateUserInformationTest() throws CatalogException { + Date date = TimeUtils.getDate(); + Calendar cl = Calendar.getInstance(); + cl.setTime(date); + cl.add(Calendar.YEAR, 1); + String expirationTime = TimeUtils.getTime(cl.getTime()); + + OrganizationUserUpdateParams userUpdateParams = new OrganizationUserUpdateParams() + .setName("newName") + .setEmail("mail@mail.com") + .setAccount(new OrganizationUserUpdateParams.Account(expirationTime)) + .setQuota(new UserQuota(1000, 1000000, 1000, 1000000)) + .setAttributes(Collections.singletonMap("key1", "value1")); + updateAndAssertChanges(organizationId, userUpdateParams, opencgaToken); + + userUpdateParams = new OrganizationUserUpdateParams() + .setName("newName2") + .setEmail("mai2l@mail.com") + .setAccount(new OrganizationUserUpdateParams.Account(expirationTime)) + .setQuota(new UserQuota(1001, 1010000, 1010, 1100000)) + .setAttributes(Collections.singletonMap("key2", "value2")); + updateAndAssertChanges(null, userUpdateParams, ownerToken); + + userUpdateParams = new OrganizationUserUpdateParams() + .setName("newName3") + .setEmail("mai3l@mail.com") + .setAccount(new OrganizationUserUpdateParams.Account(expirationTime)) + .setQuota(new UserQuota(3001, 1010300, 1013, 1300000)) + .setAttributes(Collections.singletonMap("key3", "value3")); + updateAndAssertChanges(null, userUpdateParams, orgAdminToken1); + + thrown.expect(CatalogAuthorizationException.class); + catalogManager.getOrganizationManager().updateUser(organizationId, normalUserId1, userUpdateParams, INCLUDE_RESULT, normalToken1); + } + + private void updateAndAssertChanges(String orgId, OrganizationUserUpdateParams userUpdateParams, String token) throws CatalogException { + User user = catalogManager.getOrganizationManager().updateUser(orgId, normalUserId1, userUpdateParams, INCLUDE_RESULT, token).first(); + assertEquals(userUpdateParams.getName(), user.getName()); + assertEquals(userUpdateParams.getEmail(), user.getEmail()); + assertEquals(userUpdateParams.getAccount().getExpirationDate(), user.getInternal().getAccount().getExpirationDate()); + assertEquals(userUpdateParams.getQuota().getCpuUsage(), user.getQuota().getCpuUsage()); + assertEquals(userUpdateParams.getQuota().getDiskUsage(), user.getQuota().getDiskUsage()); + assertEquals(userUpdateParams.getQuota().getMaxCpu(), user.getQuota().getMaxCpu()); + assertEquals(userUpdateParams.getQuota().getMaxDisk(), user.getQuota().getMaxDisk()); + for (String key : userUpdateParams.getAttributes().keySet()) { + assertEquals(userUpdateParams.getAttributes().get(key), user.getAttributes().get(key)); + } + } + + @Test + public void migrationExecutionProjectionTest() throws CatalogException { + Organization organization = catalogManager.getOrganizationManager().get(organizationId, null, ownerToken).first(); + assertNotNull(organization.getInternal()); + assertTrue(CollectionUtils.isNotEmpty(organization.getInternal().getMigrationExecutions())); + + QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, OrganizationDBAdaptor.QueryParams.INTERNAL.key()); + organization = catalogManager.getOrganizationManager().get(organizationId, options, ownerToken).first(); + assertNotNull(organization.getInternal()); + assertTrue(CollectionUtils.isNotEmpty(organization.getInternal().getMigrationExecutions())); + assertNull(organization.getName()); + + options = new QueryOptions(QueryOptions.INCLUDE, OrganizationDBAdaptor.QueryParams.INTERNAL_MIGRATION_EXECUTIONS.key()); + organization = catalogManager.getOrganizationManager().get(organizationId, options, ownerToken).first(); + assertNotNull(organization.getInternal()); + assertTrue(CollectionUtils.isNotEmpty(organization.getInternal().getMigrationExecutions())); + assertNull(organization.getName()); + + options = new QueryOptions(QueryOptions.INCLUDE, OrganizationDBAdaptor.QueryParams.NAME.key()); + organization = catalogManager.getOrganizationManager().get(organizationId, options, ownerToken).first(); + assertNull(organization.getInternal()); + assertNotNull(organization.getName()); + + options = new QueryOptions(QueryOptions.EXCLUDE, OrganizationDBAdaptor.QueryParams.INTERNAL.key()); + organization = catalogManager.getOrganizationManager().get(organizationId, options, ownerToken).first(); + assertNull(organization.getInternal()); + assertNotNull(organization.getName()); + + options = new QueryOptions(QueryOptions.EXCLUDE, OrganizationDBAdaptor.QueryParams.INTERNAL_MIGRATION_EXECUTIONS.key()); + organization = catalogManager.getOrganizationManager().get(organizationId, options, ownerToken).first(); + assertNotNull(organization.getInternal()); + assertNull(organization.getInternal().getMigrationExecutions()); + assertNotNull(organization.getName()); + + options = new QueryOptions(QueryOptions.EXCLUDE, OrganizationDBAdaptor.QueryParams.OWNER.key()); + organization = catalogManager.getOrganizationManager().get(organizationId, options, ownerToken).first(); + assertNotNull(organization.getInternal()); + assertTrue(CollectionUtils.isNotEmpty(organization.getInternal().getMigrationExecutions())); + assertNotNull(organization.getName()); + assertNull(organization.getOwner()); + } + + @Test + public void updateOrganizationTest() throws CatalogException { + // Owner update + Organization organization = catalogManager.getOrganizationManager().update(organizationId, + new OrganizationUpdateParams().setName("name"), INCLUDE_RESULT, ownerToken).first(); + assertEquals("name", organization.getName()); + + // Admin update + organization = catalogManager.getOrganizationManager().update(organizationId, + new OrganizationUpdateParams().setName("name2"), INCLUDE_RESULT, orgAdminToken1).first(); + assertEquals("name2", organization.getName()); + + // Normal user update + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getOrganizationManager().update(organizationId, + new OrganizationUpdateParams().setName("name2"), INCLUDE_RESULT, normalToken1)); + + // Admin update owner + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getOrganizationManager().update(organizationId, + new OrganizationUpdateParams().setOwner(normalUserId2), INCLUDE_RESULT, orgAdminToken1)); + // Admin update admins + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getOrganizationManager().update(organizationId, + new OrganizationUpdateParams().setAdmins(Collections.singletonList(normalUserId1)), INCLUDE_RESULT, orgAdminToken1)); + + // Owner changes owner + organization = catalogManager.getOrganizationManager().update(organizationId, new OrganizationUpdateParams().setOwner(normalUserId2), + INCLUDE_RESULT, ownerToken).first(); + assertEquals(normalUserId2, organization.getOwner()); + + QueryOptions studyOptions = new QueryOptions() + .append(QueryOptions.INCLUDE, StudyDBAdaptor.QueryParams.GROUPS.key()) + .append(ParamConstants.INCLUDE_RESULT_PARAM, true); + OpenCGAResult result = catalogManager.getStudyManager().searchInOrganization(organizationId, new Query(), studyOptions, + normalToken2); + assertEquals(3, result.getNumResults()); + for (Study study : result.getResults()) { + for (Group group : study.getGroups()) { + if (ParamConstants.ADMINS_GROUP.equals(group.getId())) { + assertTrue(group.getUserIds().contains(normalUserId2)); + assertFalse(group.getUserIds().contains(orgOwnerUserId)); + assertFalse(group.getUserIds().contains(normalUserId1)); + } + } + } + + // Previous owner changes admins + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getOrganizationManager().update(organizationId, + new OrganizationUpdateParams().setOwner(normalUserId2), INCLUDE_RESULT, ownerToken)); + + // Current owner changes admins + Map actionMap = new HashMap<>(); + actionMap.put(OrganizationDBAdaptor.QueryParams.ADMINS.key(), ParamUtils.AddRemoveAction.ADD); + QueryOptions options = new QueryOptions() + .append(Constants.ACTIONS, actionMap) + .append(ParamConstants.INCLUDE_RESULT_PARAM, true); + organization = catalogManager.getOrganizationManager().update(organizationId, + new OrganizationUpdateParams().setAdmins(Collections.singletonList(normalUserId1)), options, normalToken2).first(); + assertEquals(3, organization.getAdmins().size()); + assertTrue(organization.getAdmins().contains(normalUserId1)); + + // Check study admins + result = catalogManager.getStudyManager().searchInOrganization(organizationId, new Query(), studyOptions, normalToken2); + assertEquals(3, result.getNumResults()); + for (Study study : result.getResults()) { + for (Group group : study.getGroups()) { + if (ParamConstants.ADMINS_GROUP.equals(group.getId())) { + assertTrue(group.getUserIds().contains(normalUserId1)); + } + } + } + } + + @Test + public void secretKeyIsAlwaysHiddenTest() throws CatalogException { + Organization organization = catalogManager.getOrganizationManager().get(organizationId, null, ownerToken).first(); + assertNotNull(organization.getName()); + assertNull(organization.getConfiguration().getToken()); + assertNull(organization.getConfiguration().getAuthenticationOrigins().get(0).getOptions()); + + QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, OrganizationDBAdaptor.QueryParams.CONFIGURATION.key()); + organization = catalogManager.getOrganizationManager().get(organizationId, options, ownerToken).first(); + assertNull(organization.getName()); + assertNull(organization.getConfiguration().getToken()); + assertNull(organization.getConfiguration().getAuthenticationOrigins().get(0).getOptions()); + + options = new QueryOptions(QueryOptions.EXCLUDE, OrganizationDBAdaptor.QueryParams.NAME.key()); + organization = catalogManager.getOrganizationManager().get(organizationId, options, ownerToken).first(); + assertNull(organization.getName()); + assertNull(organization.getConfiguration().getToken()); + assertNull(organization.getConfiguration().getAuthenticationOrigins().get(0).getOptions()); + } + +} diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/PanelManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/PanelManagerTest.java index a055d368557..43191dc6838 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/PanelManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/PanelManagerTest.java @@ -17,10 +17,8 @@ package org.opencb.opencga.catalog.managers; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; -import org.junit.rules.ExpectedException; import org.opencb.biodata.models.clinical.ClinicalProperty; import org.opencb.biodata.models.clinical.interpretation.CancerPanel; import org.opencb.biodata.models.clinical.interpretation.DiseasePanel; @@ -28,8 +26,6 @@ import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.commons.test.GenericTest; -import org.opencb.opencga.TestParamConstants; import org.opencb.opencga.catalog.db.api.PanelDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.utils.Constants; @@ -41,7 +37,6 @@ import org.opencb.opencga.core.models.family.Family; import org.opencb.opencga.core.models.panel.Panel; import org.opencb.opencga.core.models.panel.PanelUpdateParams; -import org.opencb.opencga.core.models.user.Account; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.MediumTests; @@ -53,46 +48,21 @@ import static org.junit.Assert.*; @Category(MediumTests.class) -public class PanelManagerTest extends GenericTest { - - private String studyFqn = "user@1000G:phase1"; - @Rule - public ExpectedException thrown = ExpectedException.none(); - - @Rule - public CatalogManagerExternalResource catalogManagerResource = new CatalogManagerExternalResource(); - - protected CatalogManager catalogManager; - protected String sessionIdUser; +public class PanelManagerTest extends AbstractManagerTest { private PanelManager panelManager; - private String opencgaToken; - - private static final QueryOptions INCLUDE_RESULT = new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true); @Before - public void setUp() throws IOException, CatalogException { - catalogManager = catalogManagerResource.getCatalogManager(); + public void setUp() throws Exception { + super.setUp(); panelManager = catalogManager.getPanelManager(); - setUpCatalogManager(catalogManager); - } - - private void setUpCatalogManager(CatalogManager catalogManager) throws IOException, CatalogException { - opencgaToken = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); - - catalogManager.getUserManager().create("user", "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, "", null, Account.AccountType.FULL, opencgaToken); - sessionIdUser = catalogManager.getUserManager().login("user", TestParamConstants.PASSWORD).getToken(); - - String projectId = catalogManager.getProjectManager().create("1000G", "Project about some genomes", "", "Homo sapiens", - null, "GRCh38", INCLUDE_RESULT, sessionIdUser).first().getId(); - catalogManager.getStudyManager().create(projectId, "phase1", null, "Phase 1", "Done", null, null, null, null, null, sessionIdUser); } @Test public void createTest() throws IOException, CatalogException { Panel panel = Panel.load(getClass().getResource("/disease_panels/panel1.json").openStream()); - DataResult diseasePanelDataResult = panelManager.create(studyFqn, panel, INCLUDE_RESULT, sessionIdUser); + DataResult diseasePanelDataResult = panelManager.create(studyFqn, panel, INCLUDE_RESULT, ownerToken); assertEquals(1, diseasePanelDataResult.getNumResults()); assertEquals(panel.getId(), diseasePanelDataResult.first().getId()); assertEquals(panel.toString(), diseasePanelDataResult.first().toString()); @@ -100,10 +70,10 @@ public void createTest() throws IOException, CatalogException { @Test public void importFromSource() throws CatalogException { - OpenCGAResult cancer = panelManager.importFromSource(studyFqn, "gene-census", null, sessionIdUser); + OpenCGAResult cancer = panelManager.importFromSource(studyFqn, "gene-census", null, ownerToken); assertEquals(1, cancer.getNumInserted()); - OpenCGAResult panelApp = panelManager.importFromSource(studyFqn, "panelapp", "Thoracic_aortic_aneurysm_and_dissection-PanelAppId-700,VACTERL-like_phenotypes-PanelAppId-101", sessionIdUser); + OpenCGAResult panelApp = panelManager.importFromSource(studyFqn, "panelapp", "Thoracic_aortic_aneurysm_and_dissection-PanelAppId-700,VACTERL-like_phenotypes-PanelAppId-101", ownerToken); assertEquals(2, panelApp.getNumInserted()); } @@ -111,20 +81,20 @@ public void importFromSource() throws CatalogException { public void importFromSourceInvalidId() throws CatalogException { thrown.expect(CatalogException.class); thrown.expectMessage("Unknown panel"); - panelManager.importFromSource(studyFqn, "gene-census", "ZSR222", sessionIdUser); + panelManager.importFromSource(studyFqn, "gene-census", "ZSR222", ownerToken); } @Test public void importFromInvalidSource() throws CatalogException { thrown.expect(CatalogException.class); thrown.expectMessage("Unknown source"); - panelManager.importFromSource(studyFqn, "gene-census-wrong", null, sessionIdUser); + panelManager.importFromSource(studyFqn, "gene-census-wrong", null, ownerToken); } @Test public void updateTest() throws CatalogException { - panelManager.importFromSource(studyFqn, "gene-census", null, sessionIdUser); - Panel panel = panelManager.get(studyFqn, "gene-census", QueryOptions.empty(), sessionIdUser).first(); + panelManager.importFromSource(studyFqn, "gene-census", null, ownerToken); + Panel panel = panelManager.get(studyFqn, "gene-census", QueryOptions.empty(), ownerToken).first(); assertEquals(1, panel.getVersion()); assertEquals((int) panel.getStats().get("numberOfRegions"), panel.getVariants().size()); assertEquals((int) panel.getStats().get("numberOfVariants"), panel.getVariants().size()); @@ -146,10 +116,10 @@ public void updateTest() throws CatalogException { .setVariants(Collections.singletonList(variantPanel)) .setGenes(Collections.singletonList(genePanel)); - DataResult updateResult = panelManager.update(studyFqn, "gene-census", updateParams, null, sessionIdUser); + DataResult updateResult = panelManager.update(studyFqn, "gene-census", updateParams, null, ownerToken); assertEquals(1, updateResult.getNumUpdated()); - Panel updatedPanel = panelManager.get(studyFqn, "gene-census", QueryOptions.empty(), sessionIdUser).first(); + Panel updatedPanel = panelManager.get(studyFqn, "gene-census", QueryOptions.empty(), ownerToken).first(); assertEquals(2, updatedPanel.getVersion()); assertEquals("author", updatedPanel.getSource().getAuthor()); assertEquals(1, updatedPanel.getRegions().size()); @@ -167,51 +137,51 @@ public void updateTest() throws CatalogException { Query query = new Query() .append(PanelDBAdaptor.QueryParams.VERSION.key(), 1); - panel = panelManager.get(studyFqn, Collections.singletonList("gene-census"), query, QueryOptions.empty(), false, sessionIdUser).first(); + panel = panelManager.get(studyFqn, Collections.singletonList("gene-census"), query, QueryOptions.empty(), false, ownerToken).first(); assertEquals("gene-census", panel.getId()); assertEquals(1, panel.getVersion()); } @Test public void deletePanelTest() throws CatalogException { - panelManager.importFromSource(studyFqn, "gene-census", null, sessionIdUser); - Panel panel = panelManager.get(studyFqn, "gene-census", QueryOptions.empty(), sessionIdUser).first(); + panelManager.importFromSource(studyFqn, "gene-census", null, ownerToken); + Panel panel = panelManager.get(studyFqn, "gene-census", QueryOptions.empty(), ownerToken).first(); assertEquals(1, panel.getVersion()); - OpenCGAResult result = panelManager.delete(studyFqn, Collections.singletonList("gene-census"), QueryOptions.empty(), sessionIdUser); + OpenCGAResult result = panelManager.delete(studyFqn, Collections.singletonList("gene-census"), QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumDeleted()); - result = panelManager.get(studyFqn, Collections.singletonList("gene-census"), QueryOptions.empty(), true, sessionIdUser); + result = panelManager.get(studyFqn, Collections.singletonList("gene-census"), QueryOptions.empty(), true, ownerToken); assertEquals(0, result.getNumResults()); Query query = new Query() .append(ParamConstants.DELETED_PARAM, true); - result = panelManager.get(studyFqn, Collections.singletonList("gene-census"), query, QueryOptions.empty(), false, sessionIdUser); + result = panelManager.get(studyFqn, Collections.singletonList("gene-census"), query, QueryOptions.empty(), false, ownerToken); assertEquals(1, result.getNumResults()); } @Test public void deletePanelWithVersionsTest() throws CatalogException { - panelManager.importFromSource(studyFqn, "gene-census", null, sessionIdUser); - Panel panel = panelManager.get(studyFqn, "gene-census", QueryOptions.empty(), sessionIdUser).first(); + panelManager.importFromSource(studyFqn, "gene-census", null, ownerToken); + Panel panel = panelManager.get(studyFqn, "gene-census", QueryOptions.empty(), ownerToken).first(); assertEquals(1, panel.getVersion()); PanelUpdateParams updateParams = new PanelUpdateParams() .setSource(new DiseasePanel.SourcePanel().setAuthor("author")) .setDisorders(Collections.singletonList(new OntologyTerm().setId("ontologyTerm"))); - DataResult updateResult = panelManager.update(studyFqn, "gene-census", updateParams, null, sessionIdUser); + DataResult updateResult = panelManager.update(studyFqn, "gene-census", updateParams, null, ownerToken); assertEquals(1, updateResult.getNumUpdated()); - OpenCGAResult result = panelManager.delete(studyFqn, Collections.singletonList("gene-census"), QueryOptions.empty(), sessionIdUser); + OpenCGAResult result = panelManager.delete(studyFqn, Collections.singletonList("gene-census"), QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumDeleted()); - result = panelManager.get(studyFqn, Collections.singletonList("gene-census"), QueryOptions.empty(), true, sessionIdUser); + result = panelManager.get(studyFqn, Collections.singletonList("gene-census"), QueryOptions.empty(), true, ownerToken); assertEquals(0, result.getNumResults()); Query query = new Query() .append(Constants.ALL_VERSIONS, true) .append(ParamConstants.DELETED_PARAM, true); - result = panelManager.get(studyFqn, Collections.singletonList("gene-census"), query, QueryOptions.empty(), false, sessionIdUser); + result = panelManager.get(studyFqn, Collections.singletonList("gene-census"), query, QueryOptions.empty(), false, ownerToken); assertEquals(2, result.getNumResults()); } @@ -249,65 +219,65 @@ public void panelIncrementVersionWithCasesAndInterpretations() throws CatalogExc .setLocked(true); Interpretation interpretation9 = DummyModelUtils.getDummyInterpretation(Collections.singletonList(panel3)); - catalogManager.getPanelManager().create(studyFqn, panel1, QueryOptions.empty(), sessionIdUser); - catalogManager.getPanelManager().create(studyFqn, panel2, QueryOptions.empty(), sessionIdUser); - catalogManager.getPanelManager().create(studyFqn, panel3, QueryOptions.empty(), sessionIdUser); - catalogManager.getPanelManager().create(studyFqn, panel4, QueryOptions.empty(), sessionIdUser); + catalogManager.getPanelManager().create(studyFqn, panel1, QueryOptions.empty(), ownerToken); + catalogManager.getPanelManager().create(studyFqn, panel2, QueryOptions.empty(), ownerToken); + catalogManager.getPanelManager().create(studyFqn, panel3, QueryOptions.empty(), ownerToken); + catalogManager.getPanelManager().create(studyFqn, panel4, QueryOptions.empty(), ownerToken); - catalogManager.getFamilyManager().create(studyFqn, family1, QueryOptions.empty(), sessionIdUser); + catalogManager.getFamilyManager().create(studyFqn, family1, QueryOptions.empty(), ownerToken); - catalogManager.getClinicalAnalysisManager().create(studyFqn, case1, QueryOptions.empty(), sessionIdUser); - catalogManager.getClinicalAnalysisManager().create(studyFqn, case2, QueryOptions.empty(), sessionIdUser); - catalogManager.getClinicalAnalysisManager().create(studyFqn, case3, QueryOptions.empty(), sessionIdUser); - catalogManager.getClinicalAnalysisManager().create(studyFqn, case4, QueryOptions.empty(), sessionIdUser); + catalogManager.getClinicalAnalysisManager().create(studyFqn, case1, QueryOptions.empty(), ownerToken); + catalogManager.getClinicalAnalysisManager().create(studyFqn, case2, QueryOptions.empty(), ownerToken); + catalogManager.getClinicalAnalysisManager().create(studyFqn, case3, QueryOptions.empty(), ownerToken); + catalogManager.getClinicalAnalysisManager().create(studyFqn, case4, QueryOptions.empty(), ownerToken); // case1 catalogManager.getInterpretationManager().create(studyFqn, case1.getId(), interpretation1, - ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), sessionIdUser); + ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), ownerToken); catalogManager.getInterpretationManager().create(studyFqn, case1.getId(), interpretation2, - ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), sessionIdUser); + ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), ownerToken); catalogManager.getInterpretationManager().create(studyFqn, case1.getId(), interpretation3, - ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), sessionIdUser); + ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), ownerToken); // case 2 catalogManager.getInterpretationManager().create(studyFqn, case2.getId(), interpretation4, - ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), sessionIdUser); + ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), ownerToken); catalogManager.getInterpretationManager().create(studyFqn, case2.getId(), interpretation5, - ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), sessionIdUser); - catalogManager.getClinicalAnalysisManager().update(studyFqn, case2.getId(), new ClinicalAnalysisUpdateParams().setPanelLock(true), - QueryOptions.empty(), sessionIdUser); + ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), ownerToken); + catalogManager.getClinicalAnalysisManager().update(studyFqn, case2.getId(), new ClinicalAnalysisUpdateParams().setPanelLocked(true), + QueryOptions.empty(), ownerToken); // case 3 catalogManager.getInterpretationManager().create(studyFqn, case3.getId(), interpretation6, - ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), sessionIdUser); + ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), ownerToken); catalogManager.getInterpretationManager().create(studyFqn, case3.getId(), interpretation7, - ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), sessionIdUser); + ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), ownerToken); catalogManager.getClinicalAnalysisManager().update(studyFqn, case3.getId(), new ClinicalAnalysisUpdateParams().setLocked(true), - QueryOptions.empty(), sessionIdUser); + QueryOptions.empty(), ownerToken); // case 4 catalogManager.getInterpretationManager().create(studyFqn, case4.getId(), interpretation8, - ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), sessionIdUser); + ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), ownerToken); catalogManager.getInterpretationManager().create(studyFqn, case4.getId(), interpretation9, - ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), sessionIdUser); + ParamUtils.SaveInterpretationAs.SECONDARY, QueryOptions.empty(), ownerToken); catalogManager.getClinicalAnalysisManager().update(studyFqn, case4.getId(), new ClinicalAnalysisUpdateParams().setLocked(true), - QueryOptions.empty(), sessionIdUser); + QueryOptions.empty(), ownerToken); // Update panel1 ... - panelManager.update(studyFqn, panel1.getId(), new PanelUpdateParams().setName("name"), QueryOptions.empty(), sessionIdUser); - OpenCGAResult resultPanel = panelManager.get(studyFqn, panel1.getId(), QueryOptions.empty(), sessionIdUser); + panelManager.update(studyFqn, panel1.getId(), new PanelUpdateParams().setName("name"), QueryOptions.empty(), ownerToken); + OpenCGAResult resultPanel = panelManager.get(studyFqn, panel1.getId(), QueryOptions.empty(), ownerToken); assertEquals(2, resultPanel.first().getVersion()); assertEquals("name", resultPanel.first().getName()); Query query = new Query() .append(PanelDBAdaptor.QueryParams.ID.key(), panel1.getId()) .append(PanelDBAdaptor.QueryParams.VERSION.key(), 1); - resultPanel = panelManager.search(studyFqn, query, QueryOptions.empty(), sessionIdUser); + resultPanel = panelManager.search(studyFqn, query, QueryOptions.empty(), ownerToken); assertEquals(1, resultPanel.first().getVersion()); assertNotEquals("name", resultPanel.first().getName()); OpenCGAResult result = catalogManager.getClinicalAnalysisManager().get(studyFqn, - Arrays.asList(case1.getId(), case2.getId(), case3.getId(), case4.getId()), QueryOptions.empty(), sessionIdUser); + Arrays.asList(case1.getId(), case2.getId(), case3.getId(), case4.getId()), QueryOptions.empty(), ownerToken); case1 = result.getResults().get(0); case2 = result.getResults().get(1); case3 = result.getResults().get(2); @@ -381,34 +351,34 @@ public void geneIdAndNameQueryTest() throws CatalogException { .setId("panel2") .setGenes(Arrays.asList(createGenePanel("geneId1", "geneName1"), createGenePanel("geneId3", "geneName3"))); - panelManager.create(studyFqn, panel1, QueryOptions.empty(), sessionIdUser); - panelManager.create(studyFqn, panel2, QueryOptions.empty(), sessionIdUser); + panelManager.create(studyFqn, panel1, QueryOptions.empty(), ownerToken); + panelManager.create(studyFqn, panel2, QueryOptions.empty(), ownerToken); - OpenCGAResult result = panelManager.search(studyFqn, new Query(PanelDBAdaptor.QueryParams.GENES.key(), "geneName2"), QueryOptions.empty(), sessionIdUser); + OpenCGAResult result = panelManager.search(studyFqn, new Query(PanelDBAdaptor.QueryParams.GENES.key(), "geneName2"), QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumResults()); assertEquals("panel1", result.first().getId()); - result = panelManager.search(studyFqn, new Query(PanelDBAdaptor.QueryParams.GENES.key(), "geneId2"), QueryOptions.empty(), sessionIdUser); + result = panelManager.search(studyFqn, new Query(PanelDBAdaptor.QueryParams.GENES.key(), "geneId2"), QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumResults()); assertEquals("panel1", result.first().getId()); - result = panelManager.search(studyFqn, new Query(PanelDBAdaptor.QueryParams.GENES.key(), "geneName1"), QueryOptions.empty(), sessionIdUser); + result = panelManager.search(studyFqn, new Query(PanelDBAdaptor.QueryParams.GENES.key(), "geneName1"), QueryOptions.empty(), ownerToken); assertEquals(2, result.getNumResults()); assertTrue(result.getResults().stream().map(Panel::getId).collect(Collectors.toList()).containsAll(Arrays.asList("panel1", "panel2"))); - result = panelManager.search(studyFqn, new Query(PanelDBAdaptor.QueryParams.GENES.key(), "geneId1"), QueryOptions.empty(), sessionIdUser); + result = panelManager.search(studyFqn, new Query(PanelDBAdaptor.QueryParams.GENES.key(), "geneId1"), QueryOptions.empty(), ownerToken); assertEquals(2, result.getNumResults()); assertTrue(result.getResults().stream().map(Panel::getId).collect(Collectors.toList()).containsAll(Arrays.asList("panel1", "panel2"))); - result = panelManager.search(studyFqn, new Query(PanelDBAdaptor.QueryParams.GENES.key(), "geneId3"), QueryOptions.empty(), sessionIdUser); + result = panelManager.search(studyFqn, new Query(PanelDBAdaptor.QueryParams.GENES.key(), "geneId3"), QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumResults()); assertEquals("panel2", result.first().getId()); - result = panelManager.search(studyFqn, new Query(PanelDBAdaptor.QueryParams.GENES.key(), "geneName3"), QueryOptions.empty(), sessionIdUser); + result = panelManager.search(studyFqn, new Query(PanelDBAdaptor.QueryParams.GENES.key(), "geneName3"), QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumResults()); assertEquals("panel2", result.first().getId()); - result = panelManager.search(studyFqn, new Query(PanelDBAdaptor.QueryParams.GENES.key(), "geneId4"), QueryOptions.empty(), sessionIdUser); + result = panelManager.search(studyFqn, new Query(PanelDBAdaptor.QueryParams.GENES.key(), "geneId4"), QueryOptions.empty(), ownerToken); assertEquals(0, result.getNumResults()); } diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ProjectManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ProjectManagerTest.java index 611a3a6204b..85e13914da5 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ProjectManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ProjectManagerTest.java @@ -18,16 +18,12 @@ import com.fasterxml.jackson.core.JsonProcessingException; import org.apache.commons.collections4.CollectionUtils; -import org.junit.Before; -import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; -import org.junit.rules.ExpectedException; import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.commons.test.GenericTest; import org.opencb.opencga.TestParamConstants; import org.opencb.opencga.catalog.db.api.ProjectDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; @@ -35,15 +31,17 @@ import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.config.storage.CellBaseConfiguration; +import org.opencb.opencga.core.models.organizations.OrganizationCreateParams; +import org.opencb.opencga.core.models.organizations.OrganizationUpdateParams; import org.opencb.opencga.core.models.project.Project; import org.opencb.opencga.core.models.project.ProjectCreateParams; import org.opencb.opencga.core.models.project.ProjectOrganism; import org.opencb.opencga.core.models.study.GroupUpdateParams; -import org.opencb.opencga.core.models.user.Account; +import org.opencb.opencga.core.models.study.Study; +import org.opencb.opencga.core.models.user.User; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.MediumTests; -import java.io.IOException; import java.util.Collections; import static org.junit.Assert.*; @@ -52,68 +50,18 @@ * Created by pfurio on 28/11/16. */ @Category(MediumTests.class) -public class ProjectManagerTest extends GenericTest { - - @Rule - public ExpectedException thrown = ExpectedException.none(); - - @Rule - public CatalogManagerExternalResource catalogManagerResource = new CatalogManagerExternalResource(); - - protected CatalogManager catalogManager; - private String opencgaToken; - protected String sessionIdUser; - protected String sessionIdUser2; - protected String sessionIdUser3; - private String project1; - private String project2; - private String project3; - private String studyId; - private String studyId2; - private String studyId3; - - private static final QueryOptions INCLUDE_RESULT = new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true); - - @Before - public void setUp() throws IOException, CatalogException { - catalogManager = catalogManagerResource.getCatalogManager(); - setUpCatalogManager(catalogManager); - } - - public void setUpCatalogManager(CatalogManager catalogManager) throws IOException, CatalogException { - opencgaToken = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); - - catalogManager.getUserManager().create("user", "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, "", null, Account.AccountType.FULL, opencgaToken); - catalogManager.getUserManager().create("user2", "User2 Name", "mail2@ebi.ac.uk", TestParamConstants.PASSWORD, "", null, Account.AccountType.FULL, opencgaToken); - catalogManager.getUserManager().create("user3", "User3 Name", "user.2@e.mail", TestParamConstants.PASSWORD, "ACME", null, Account.AccountType.FULL, opencgaToken); - - sessionIdUser = catalogManager.getUserManager().login("user", TestParamConstants.PASSWORD).getToken(); - sessionIdUser2 = catalogManager.getUserManager().login("user2", TestParamConstants.PASSWORD).getToken(); - sessionIdUser3 = catalogManager.getUserManager().login("user3", TestParamConstants.PASSWORD).getToken(); - - project1 = catalogManager.getProjectManager().create("1000G", "Project about some genomes", "", "Homo sapiens", - null, "GRCh38", INCLUDE_RESULT, sessionIdUser).first().getId(); - project2 = catalogManager.getProjectManager().create("pmp", "Project Management Project", "life art intelligent system", - "Homo sapiens", null, "GRCh38", INCLUDE_RESULT, sessionIdUser2).first().getId(); - project3 = catalogManager.getProjectManager().create("p1", "project 1", "", "Homo sapiens", - null, "GRCh38", INCLUDE_RESULT, sessionIdUser3).first().getId(); - - studyId = catalogManager.getStudyManager().create(project1, "phase1", null, "Phase 1", "Done", null, null, null, null, INCLUDE_RESULT, sessionIdUser).first().getFqn(); - studyId2 = catalogManager.getStudyManager().create(project1, "phase3", null, "Phase 3", "d", null, null, null, null, INCLUDE_RESULT, sessionIdUser).first().getFqn(); - - studyId3 = catalogManager.getStudyManager().create(project2, "s1", null, "Study 1", "", null, null, null, null, INCLUDE_RESULT, sessionIdUser2).first().getFqn(); - } +public class ProjectManagerTest extends AbstractManagerTest { @Test public void searchProjectByStudy() throws CatalogException { - OpenCGAResult result = catalogManager.getProjectManager().search(new Query(ProjectDBAdaptor.QueryParams.STUDY.key(), "phase1"), null, sessionIdUser); + OpenCGAResult result = catalogManager.getProjectManager().search(organizationId, new Query(ProjectDBAdaptor.QueryParams.STUDY.key(), "phase1"), null, ownerToken); assertEquals(1, result.getNumResults()); assertEquals(project1, result.first().getId()); } @Test public void getOwnProjectNoStudies() throws CatalogException { - DataResult projectDataResult = catalogManager.getProjectManager().get(project3, null, sessionIdUser3); + DataResult projectDataResult = catalogManager.getProjectManager().get(project3, null, orgAdminToken2); assertEquals(1, projectDataResult.getNumResults()); } @@ -121,52 +69,64 @@ public void getOwnProjectNoStudies() throws CatalogException { public void getOtherUsersProject() throws CatalogException { thrown.expect(CatalogException.class); thrown.expectMessage("cannot view"); - catalogManager.getProjectManager().get(project1, null, sessionIdUser3); + catalogManager.getProjectManager().get(project2, null, noAccessToken1); } @Test public void searchSampleNoPermissions() throws CatalogException { - // User3 looks for any sample without providing any project or study and he has not been granted permissions anywhere + // First, remove anonymous user access to the study + catalogManager.getStudyManager().updateGroup(studyFqn, ParamConstants.MEMBERS_GROUP, ParamUtils.BasicUpdateAction.REMOVE, + new GroupUpdateParams(Collections.singletonList(ParamConstants.ANONYMOUS_USER_ID)), ownerToken); + + // User1 looks for any sample without providing any project or study and he has not been granted permissions anywhere thrown.expect(CatalogAuthorizationException.class); thrown.expectMessage("cannot view any study"); - catalogManager.getSampleManager().search("", new Query(), QueryOptions.empty(), sessionIdUser3); + catalogManager.getSampleManager().search("", new Query(), QueryOptions.empty(), noAccessToken1); } @Test public void searchProjects() throws CatalogException { - - catalogManager.getUserManager().create("userid", "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, "", null, Account.AccountType.FULL, opencgaToken); - String token = catalogManager.getUserManager().login("userid", TestParamConstants.PASSWORD).getToken(); - OpenCGAResult projectOpenCGAResult = catalogManager.getProjectManager().search(new Query(), QueryOptions.empty(), token); + String org2 = "otherOrg"; + catalogManager.getOrganizationManager().create(new OrganizationCreateParams().setId(org2), QueryOptions.empty(), opencgaToken); + catalogManager.getUserManager().create(new User().setId("userFromOrg2").setName("name").setOrganization(org2), + TestParamConstants.PASSWORD, opencgaToken); + catalogManager.getOrganizationManager().update(org2, new OrganizationUpdateParams().setOwner("userFromOrg2"), null, opencgaToken); + String owner2Token = catalogManager.getUserManager().login(org2, "userFromOrg2", TestParamConstants.PASSWORD).getToken(); + Project p = catalogManager.getProjectManager().create(new ProjectCreateParams() + .setId("project") + .setOrganism(new ProjectOrganism("Homo sapiens", "GRCh38")), + INCLUDE_RESULT, owner2Token).first(); + Study study = catalogManager.getStudyManager().create(p.getFqn(), new Study().setId("study"), INCLUDE_RESULT, owner2Token).first(); + + catalogManager.getUserManager().create("userid", "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, org2, null, owner2Token); + String token = catalogManager.getUserManager().login(org2, "userid", TestParamConstants.PASSWORD).getToken(); + OpenCGAResult projectOpenCGAResult = catalogManager.getProjectManager().search(org2, new Query(), QueryOptions.empty(), token); assertTrue(projectOpenCGAResult.getResults().isEmpty()); assertEquals(0, projectOpenCGAResult.getEvents().size()); String otherUser = "user_tmp"; - catalogManager.getUserManager().create(otherUser, "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, "", null, Account.AccountType.FULL, opencgaToken); - String otherUsertoken = catalogManager.getUserManager().login(otherUser, TestParamConstants.PASSWORD).getToken(); + catalogManager.getUserManager().create(otherUser, "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, org2, null, owner2Token); + String otherUsertoken = catalogManager.getUserManager().login(org2, otherUser, TestParamConstants.PASSWORD).getToken(); OpenCGAResult result = catalogManager.getProjectManager() - .search(new Query(), QueryOptions.empty(), otherUsertoken); + .search(org2, new Query(), QueryOptions.empty(), otherUsertoken); assertTrue(result.getResults().isEmpty()); assertEquals(0, result.getEvents().size()); - // Create a new study in project2 with some dummy permissions for user - String s2 = catalogManager.getStudyManager().create(project2, "s2", null, "Study 2", "", null, null, null, null, INCLUDE_RESULT, sessionIdUser2).first().getId(); - catalogManager.getStudyManager().updateGroup(s2, "@members", ParamUtils.BasicUpdateAction.ADD, - new GroupUpdateParams(Collections.singletonList(otherUser)), sessionIdUser2); + // Give some permissions to the user in the study + catalogManager.getStudyManager().updateGroup(study.getFqn(), "@members", ParamUtils.BasicUpdateAction.ADD, + new GroupUpdateParams(Collections.singletonList(otherUser)), owner2Token); - result = catalogManager.getProjectManager() - .search(new Query(), QueryOptions.empty(), otherUsertoken); + result = catalogManager.getProjectManager().search(org2, new Query(), QueryOptions.empty(), otherUsertoken); assertEquals(1, result.getNumResults()); - assertEquals(project2, result.first().getId()); - assertEquals("user2@pmp", result.first().getFqn()); - + assertEquals(p.getId(), result.first().getId()); + assertEquals(p.getFqn(), result.first().getFqn()); } @Test public void searchProjectsUsingInclude() throws CatalogException { - OpenCGAResult projects = catalogManager.getProjectManager().search(new Query(), - new QueryOptions(QueryOptions.INCLUDE, ProjectDBAdaptor.QueryParams.ID.key()), sessionIdUser); - assertEquals(1, projects.getNumResults()); + OpenCGAResult projects = catalogManager.getProjectManager().search(organizationId, new Query(), + new QueryOptions(QueryOptions.INCLUDE, ProjectDBAdaptor.QueryParams.ID.key()), ownerToken); + assertEquals(3, projects.getNumResults()); for (Project project : projects.getResults()) { assertNotNull(project.getId()); assertNull(project.getDescription()); @@ -175,75 +135,35 @@ public void searchProjectsUsingInclude() throws CatalogException { assertTrue(CollectionUtils.isEmpty(project.getStudies())); } - projects = catalogManager.getProjectManager().search(new Query(), - new QueryOptions(QueryOptions.INCLUDE, ProjectDBAdaptor.QueryParams.STUDIES.key()), sessionIdUser); - assertEquals(1, projects.getNumResults()); + projects = catalogManager.getProjectManager().search(organizationId, new Query(), + new QueryOptions(QueryOptions.INCLUDE, ProjectDBAdaptor.QueryParams.STUDIES.key()), ownerToken); + assertEquals(3, projects.getNumResults()); for (Project project : projects.getResults()) { - assertNull(project.getId()); assertNull(project.getDescription()); assertNull(project.getName()); assertNotNull(project.getStudies()); - assertTrue(CollectionUtils.isNotEmpty(project.getStudies())); + if ("p1".equals(project.getId())) { + assertTrue(CollectionUtils.isEmpty(project.getStudies())); + } else { + assertTrue(CollectionUtils.isNotEmpty(project.getStudies())); + } } - projects = catalogManager.getProjectManager().search(new Query(), - new QueryOptions(QueryOptions.EXCLUDE, ProjectDBAdaptor.QueryParams.NAME.key()), sessionIdUser); - assertEquals(1, projects.getNumResults()); + projects = catalogManager.getProjectManager().search(organizationId, new Query(), + new QueryOptions(QueryOptions.EXCLUDE, ProjectDBAdaptor.QueryParams.NAME.key()), ownerToken); + assertEquals(3, projects.getNumResults()); for (Project project : projects.getResults()) { assertNotNull(project.getId()); assertNull(project.getName()); assertNotNull(project.getDescription()); assertNotNull(project.getStudies()); - assertFalse(CollectionUtils.isEmpty(project.getStudies())); - } - } - - @Test - public void getSharedProjects() throws CatalogException { - try { - OpenCGAResult user = catalogManager.getProjectManager().getSharedProjects("user", null, sessionIdUser); - } catch (CatalogAuthorizationException e) { - // Correct - } - - // Create a new study in project2 with some dummy permissions for user - String s2 = catalogManager.getStudyManager().create(project2, "s2", null, "Study 2", "", null, null, null, null, INCLUDE_RESULT, sessionIdUser2).first().getId(); - catalogManager.getStudyManager().updateGroup(s2, "@members", ParamUtils.BasicUpdateAction.ADD, - new GroupUpdateParams(Collections.singletonList("user")), sessionIdUser2); - - DataResult queryResult = catalogManager.getProjectManager().getSharedProjects("user", null, sessionIdUser); - assertEquals(1, queryResult.getNumResults()); - assertEquals(1, queryResult.first().getStudies().size()); - assertEquals("s2", queryResult.first().getStudies().get(0).getId()); - - // Add permissions to a group were user belongs - catalogManager.getStudyManager().createGroup(studyId3, "@member", Collections.singletonList("user"), sessionIdUser2); - - queryResult = catalogManager.getProjectManager().getSharedProjects("user", null, sessionIdUser); - assertEquals(1, queryResult.getNumResults()); - assertEquals(2, queryResult.first().getStudies().size()); - assertEquals("user2@pmp", queryResult.first().getFqn()); - - // Add permissions to user in a study of user3 - String s3 = catalogManager.getStudyManager().create(project3, "s3", null, "StudyProject3", "", null, null, null, null, INCLUDE_RESULT, sessionIdUser3).first().getId(); - catalogManager.getStudyManager().updateGroup(String.valueOf(s3), "@members", ParamUtils.BasicUpdateAction.ADD, - new GroupUpdateParams(Collections.singletonList("user")), sessionIdUser3); - - queryResult = catalogManager.getProjectManager().getSharedProjects("user", null, sessionIdUser); - assertEquals(2, queryResult.getNumResults()); - for (Project project : queryResult.getResults()) { - if (project.getId().equals(project2)) { - assertEquals(2, project.getStudies().size()); - } else { - assertEquals(1, project.getStudies().size()); - } } } @Test public void updateOrganismInProject() throws CatalogException { Project pr = catalogManager.getProjectManager().create("project2", "Project about some genomes", "", "Homo sapiens", - null, "GRCh38", INCLUDE_RESULT, sessionIdUser).first(); + null, "GRCh38", INCLUDE_RESULT, ownerToken).first(); assertEquals("Homo sapiens", pr.getOrganism().getScientificName()); assertEquals("", pr.getOrganism().getCommonName()); @@ -252,9 +172,9 @@ public void updateOrganismInProject() throws CatalogException { ObjectMap objectMap = new ObjectMap(); objectMap.put(ProjectDBAdaptor.QueryParams.ORGANISM_COMMON_NAME.key(), "common"); - OpenCGAResult update = catalogManager.getProjectManager().update(pr.getId(), objectMap, INCLUDE_RESULT, sessionIdUser); + OpenCGAResult update = catalogManager.getProjectManager().update(pr.getId(), objectMap, INCLUDE_RESULT, ownerToken); assertEquals(1, update.getNumResults()); - OpenCGAResult queryResult = catalogManager.getProjectManager().get(pr.getId(), null, sessionIdUser); + OpenCGAResult queryResult = catalogManager.getProjectManager().get(pr.getId(), null, ownerToken); assertEquals("Homo sapiens", queryResult.first().getOrganism().getScientificName()); assertEquals("common", queryResult.first().getOrganism().getCommonName()); @@ -265,11 +185,11 @@ public void updateOrganismInProject() throws CatalogException { thrown.expect(CatalogException.class); thrown.expectMessage("Cannot update organism"); - catalogManager.getProjectManager().update(pr.getId(), objectMap, null, sessionIdUser); + catalogManager.getProjectManager().update(pr.getId(), objectMap, null, ownerToken); } @Test - public void createProjectCheckCellbase() throws CatalogException, JsonProcessingException { + public void createProjectCheckCellbase() throws CatalogException { Project pr = catalogManager.getProjectManager() .create(new ProjectCreateParams() .setId("Project_1") @@ -278,7 +198,7 @@ public void createProjectCheckCellbase() throws CatalogException, JsonProcessing .setCellbase(new CellBaseConfiguration( ParamConstants.CELLBASE_URL, "v5.0")), - INCLUDE_RESULT, sessionIdUser).first(); + INCLUDE_RESULT, ownerToken).first(); assertNull(pr.getCellbase().getDataRelease()); pr = catalogManager.getProjectManager() @@ -289,7 +209,7 @@ public void createProjectCheckCellbase() throws CatalogException, JsonProcessing .setCellbase(new CellBaseConfiguration( ParamConstants.CELLBASE_URL, "5.0")), - INCLUDE_RESULT, sessionIdUser).first(); + INCLUDE_RESULT, ownerToken).first(); assertNull(pr.getCellbase().getDataRelease()); pr = catalogManager.getProjectManager() @@ -300,7 +220,7 @@ public void createProjectCheckCellbase() throws CatalogException, JsonProcessing .setCellbase(new CellBaseConfiguration( ParamConstants.CELLBASE_URL, ParamConstants.CELLBASE_VERSION)), - INCLUDE_RESULT, sessionIdUser).first(); + INCLUDE_RESULT, ownerToken).first(); assertNotNull(pr.getCellbase().getDataRelease()); } @@ -313,29 +233,29 @@ public void createProjectWrongCellbase() throws CatalogException, JsonProcessing .setName("Project about some genomes") .setOrganism(new ProjectOrganism("Homo sapiens", "grch38")) .setCellbase(new CellBaseConfiguration(ParamConstants.CELLBASE_URL, ParamConstants.CELLBASE_VERSION, "NON_EXISTING_DR", null)), - INCLUDE_RESULT, sessionIdUser).first(); + INCLUDE_RESULT, ownerToken).first(); } @Test public void updateCellbaseInProject() throws CatalogException, JsonProcessingException { Project pr = catalogManager.getProjectManager().create("project2", "Project about some genomes", "", "Homo sapiens", - null, "GRCh38", INCLUDE_RESULT, sessionIdUser).first(); + null, "GRCh38", INCLUDE_RESULT, ownerToken).first(); assertNotNull(pr.getCellbase()); assertEquals("https://ws.zettagenomics.com/cellbase", pr.getCellbase().getUrl()); assertEquals(ParamConstants.CELLBASE_VERSION, pr.getCellbase().getVersion()); CellBaseConfiguration cb = new CellBaseConfiguration("https://ws.opencb.org/cellbase", "v3"); OpenCGAResult update = catalogManager.getProjectManager().setCellbaseConfiguration(pr.getId(), - new CellBaseConfiguration("https://ws.opencb.org/cellbase", "v3"), false, sessionIdUser); + new CellBaseConfiguration("https://ws.opencb.org/cellbase", "v3"), false, ownerToken); assertEquals(1, update.getNumUpdated()); - Project project = catalogManager.getProjectManager().get(pr.getId(), QueryOptions.empty(), sessionIdUser).first(); + Project project = catalogManager.getProjectManager().get(pr.getId(), QueryOptions.empty(), ownerToken).first(); assertNotNull(pr.getCellbase()); assertEquals(cb.getUrl(), project.getCellbase().getUrl()); assertEquals(cb.getVersion(), project.getCellbase().getVersion()); thrown.expectMessage("Unable to access cellbase url"); catalogManager.getProjectManager().setCellbaseConfiguration(pr.getId(), - new CellBaseConfiguration("https://ws.opencb.org/cellbase", "v3"), true, sessionIdUser); + new CellBaseConfiguration("https://ws.opencb.org/cellbase", "v3"), true, ownerToken); } } diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/SampleManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/SampleManagerTest.java index e9134475dd7..69042379f29 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/SampleManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/SampleManagerTest.java @@ -40,7 +40,6 @@ import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager; import org.opencb.opencga.catalog.db.api.DBIterator; import org.opencb.opencga.catalog.db.api.IndividualDBAdaptor; -import org.opencb.opencga.catalog.db.api.ProjectDBAdaptor; import org.opencb.opencga.catalog.db.api.SampleDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; @@ -61,12 +60,10 @@ import org.opencb.opencga.core.models.individual.IndividualAclParams; import org.opencb.opencga.core.models.individual.IndividualPermissions; import org.opencb.opencga.core.models.individual.IndividualUpdateParams; -import org.opencb.opencga.core.models.project.Project; import org.opencb.opencga.core.models.sample.*; import org.opencb.opencga.core.models.study.*; import org.opencb.opencga.core.models.summaries.FeatureCount; import org.opencb.opencga.core.models.summaries.VariableSetSummary; -import org.opencb.opencga.core.models.user.Account; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.MediumTests; @@ -87,34 +84,31 @@ public class SampleManagerTest extends AbstractManagerTest { @Test public void testSampleVersioning() throws CatalogException { - Query query = new Query(ProjectDBAdaptor.QueryParams.USER_ID.key(), "user"); - String projectId = catalogManager.getProjectManager().search(query, null, token).first().getId(); - List descriptions = Arrays.asList(RandomStringUtils.random(5), RandomStringUtils.random(5), RandomStringUtils.random(5)); catalogManager.getSampleManager().create(studyFqn, - new Sample().setId("testSample").setDescription("description"), null, token); + new Sample().setId("testSample").setDescription("description"), null, ownerToken); catalogManager.getSampleManager().update(studyFqn, "testSample", new SampleUpdateParams().setDescription(descriptions.get(0)), - new QueryOptions(), token); + new QueryOptions(), ownerToken); catalogManager.getSampleManager().update(studyFqn, "testSample", new SampleUpdateParams().setDescription(descriptions.get(1)), - new QueryOptions(), token); + new QueryOptions(), ownerToken); - catalogManager.getProjectManager().incrementRelease(projectId, token); + catalogManager.getProjectManager().incrementRelease(project1, ownerToken); // We create something to have a gap in the release - catalogManager.getSampleManager().create(studyFqn, new Sample().setId("dummy"), null, token); + catalogManager.getSampleManager().create(studyFqn, new Sample().setId("dummy"), null, ownerToken); - catalogManager.getProjectManager().incrementRelease(projectId, token); + catalogManager.getProjectManager().incrementRelease(project1, ownerToken); catalogManager.getSampleManager().update(studyFqn, "testSample", new SampleUpdateParams().setDescription(descriptions.get(2)), - new QueryOptions(), token); + new QueryOptions(), ownerToken); catalogManager.getSampleManager().update(studyFqn, "testSample", - new SampleUpdateParams().setDescription("new description"), null, token); + new SampleUpdateParams().setDescription("new description"), null, ownerToken); // We want the whole history of the sample - query = new Query() + Query query = new Query() .append(SampleDBAdaptor.QueryParams.ID.key(), "testSample") .append(Constants.ALL_VERSIONS, true); - DataResult sampleDataResult = catalogManager.getSampleManager().search(studyFqn, query, null, token); + DataResult sampleDataResult = catalogManager.getSampleManager().search(studyFqn, query, null, ownerToken); assertEquals(5, sampleDataResult.getNumResults()); assertEquals("description", sampleDataResult.getResults().get(0).getDescription()); assertEquals(descriptions.get(0), sampleDataResult.getResults().get(1).getDescription()); @@ -125,7 +119,7 @@ public void testSampleVersioning() throws CatalogException { query = new Query() .append(SampleDBAdaptor.QueryParams.VERSION.key(), "all"); sampleDataResult = catalogManager.getSampleManager().get(studyFqn, Collections.singletonList("testSample"), - query, null, false, token); + query, null, false, ownerToken); assertEquals(5, sampleDataResult.getNumResults()); assertEquals("description", sampleDataResult.getResults().get(0).getDescription()); assertEquals(descriptions.get(0), sampleDataResult.getResults().get(1).getDescription()); @@ -137,7 +131,7 @@ public void testSampleVersioning() throws CatalogException { query = new Query() .append(SampleDBAdaptor.QueryParams.ID.key(), "testSample") .append(SampleDBAdaptor.QueryParams.SNAPSHOT.key(), 1); - sampleDataResult = catalogManager.getSampleManager().search(studyFqn, query, null, token); + sampleDataResult = catalogManager.getSampleManager().search(studyFqn, query, null, ownerToken); assertEquals(1, sampleDataResult.getNumResults()); assertEquals(3, sampleDataResult.first().getVersion()); @@ -145,14 +139,14 @@ public void testSampleVersioning() throws CatalogException { query = new Query() .append(SampleDBAdaptor.QueryParams.ID.key(), "testSample") .append(SampleDBAdaptor.QueryParams.SNAPSHOT.key(), 2); - sampleDataResult = catalogManager.getSampleManager().search(studyFqn, query, null, token); + sampleDataResult = catalogManager.getSampleManager().search(studyFqn, query, null, ownerToken); assertEquals(1, sampleDataResult.getNumResults()); assertEquals(3, sampleDataResult.first().getVersion()); // We want the last version of the sample query = new Query() .append(SampleDBAdaptor.QueryParams.ID.key(), "testSample"); - sampleDataResult = catalogManager.getSampleManager().search(studyFqn, query, null, token); + sampleDataResult = catalogManager.getSampleManager().search(studyFqn, query, null, ownerToken); assertEquals(1, sampleDataResult.getNumResults()); assertEquals(5, sampleDataResult.first().getVersion()); @@ -160,7 +154,7 @@ public void testSampleVersioning() throws CatalogException { query = new Query() .append(SampleDBAdaptor.QueryParams.ID.key(), "testSample") .append(SampleDBAdaptor.QueryParams.VERSION.key(), 2); - sampleDataResult = catalogManager.getSampleManager().search(studyFqn, query, null, token); + sampleDataResult = catalogManager.getSampleManager().search(studyFqn, query, null, ownerToken); assertEquals(1, sampleDataResult.getNumResults()); assertEquals(2, sampleDataResult.first().getVersion()); @@ -168,34 +162,34 @@ public void testSampleVersioning() throws CatalogException { query = new Query() .append(SampleDBAdaptor.QueryParams.ID.key(), "testSample") .append(SampleDBAdaptor.QueryParams.VERSION.key(), 1); - sampleDataResult = catalogManager.getSampleManager().search(studyFqn, query, null, token); + sampleDataResult = catalogManager.getSampleManager().search(studyFqn, query, null, ownerToken); assertEquals(1, sampleDataResult.getNumResults()); assertEquals(1, sampleDataResult.first().getVersion()); DataResult testSample = catalogManager.getSampleManager() - .get(studyFqn, Collections.singletonList("testSample"), new Query(Constants.ALL_VERSIONS, true), null, false, token); + .get(studyFqn, Collections.singletonList("testSample"), new Query(Constants.ALL_VERSIONS, true), null, false, ownerToken); assertEquals(5, testSample.getResults().size()); } @Test public void testCustomCreationDate() throws CatalogException { Sample s1 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("s1").setCreationDate("20140101120000"), - INCLUDE_RESULT, token).first(); + INCLUDE_RESULT, ownerToken).first(); assertEquals("20140101120000", s1.getCreationDate()); OpenCGAResult search = catalogManager.getSampleManager().search(studyFqn, - new Query(SampleDBAdaptor.QueryParams.CREATION_DATE.key(), "<2015"), QueryOptions.empty(), token); + new Query(SampleDBAdaptor.QueryParams.CREATION_DATE.key(), "<2015"), QueryOptions.empty(), ownerToken); assertEquals(1, search.getNumResults()); assertEquals("s1", search.first().getId()); catalogManager.getSampleManager().update(studyFqn, "s1", new SampleUpdateParams().setCreationDate("20160101120000"), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); search = catalogManager.getSampleManager().search(studyFqn, - new Query(SampleDBAdaptor.QueryParams.CREATION_DATE.key(), "<2015"), QueryOptions.empty(), token); + new Query(SampleDBAdaptor.QueryParams.CREATION_DATE.key(), "<2015"), QueryOptions.empty(), ownerToken); assertEquals(0, search.getNumResults()); search = catalogManager.getSampleManager().search(studyFqn, - new Query(SampleDBAdaptor.QueryParams.CREATION_DATE.key(), "<201602"), QueryOptions.empty(), token); + new Query(SampleDBAdaptor.QueryParams.CREATION_DATE.key(), "<201602"), QueryOptions.empty(), ownerToken); assertEquals(1, search.getNumResults()); assertEquals("s1", search.first().getId()); } @@ -205,24 +199,24 @@ public void testCustomModificationDate() throws CatalogException { Date date = TimeUtils.toDate(TimeUtils.getTime()); Sample s1 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("s1").setModificationDate("20140101120000"), - INCLUDE_RESULT, token).first(); + INCLUDE_RESULT, ownerToken).first(); assertEquals("20140101120000", s1.getModificationDate()); Date date1 = TimeUtils.toDate(s1.getInternal().getLastModified()); assertTrue(date1.after(date) || date1.equals(date)); OpenCGAResult search = catalogManager.getSampleManager().search(studyFqn, - new Query(SampleDBAdaptor.QueryParams.MODIFICATION_DATE.key(), "<2015"), QueryOptions.empty(), token); + new Query(SampleDBAdaptor.QueryParams.MODIFICATION_DATE.key(), "<2015"), QueryOptions.empty(), ownerToken); assertEquals(1, search.getNumResults()); assertEquals("s1", search.first().getId()); catalogManager.getSampleManager().update(studyFqn, "s1", new SampleUpdateParams().setModificationDate("20160101120000"), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); search = catalogManager.getSampleManager().search(studyFqn, - new Query(SampleDBAdaptor.QueryParams.MODIFICATION_DATE.key(), "<2015"), QueryOptions.empty(), token); + new Query(SampleDBAdaptor.QueryParams.MODIFICATION_DATE.key(), "<2015"), QueryOptions.empty(), ownerToken); assertEquals(0, search.getNumResults()); search = catalogManager.getSampleManager().search(studyFqn, - new Query(SampleDBAdaptor.QueryParams.MODIFICATION_DATE.key(), "<201602"), QueryOptions.empty(), token); + new Query(SampleDBAdaptor.QueryParams.MODIFICATION_DATE.key(), "<201602"), QueryOptions.empty(), ownerToken); assertEquals(1, search.getNumResults()); assertEquals("s1", search.first().getId()); @@ -233,36 +227,33 @@ public void testCustomModificationDate() throws CatalogException { @Test public void testSampleVersioningWithWeirdId() throws CatalogException { - Query query = new Query(ProjectDBAdaptor.QueryParams.USER_ID.key(), "user"); - String projectId = catalogManager.getProjectManager().search(query, null, token).first().getId(); - String sampleId = "test__Sa..mp--le"; List descriptions = Arrays.asList(RandomStringUtils.random(5), RandomStringUtils.random(5), RandomStringUtils.random(5)); catalogManager.getSampleManager().create(studyFqn, - new Sample().setId(sampleId).setDescription("description"), null, token); + new Sample().setId(sampleId).setDescription("description"), null, ownerToken); catalogManager.getSampleManager().update(studyFqn, sampleId, new SampleUpdateParams().setDescription(descriptions.get(0)), - new QueryOptions(), token); + new QueryOptions(), ownerToken); catalogManager.getSampleManager().update(studyFqn, sampleId, new SampleUpdateParams().setDescription(descriptions.get(1)), - new QueryOptions(), token); + new QueryOptions(), ownerToken); - catalogManager.getProjectManager().incrementRelease(projectId, token); + catalogManager.getProjectManager().incrementRelease(project1, ownerToken); // We create something to have a gap in the release - catalogManager.getSampleManager().create(studyFqn, new Sample().setId("du--m---m..y"), null, token); + catalogManager.getSampleManager().create(studyFqn, new Sample().setId("du--m---m..y"), null, ownerToken); - catalogManager.getProjectManager().incrementRelease(projectId, token); + catalogManager.getProjectManager().incrementRelease(project1, ownerToken); catalogManager.getSampleManager().update(studyFqn, sampleId, new SampleUpdateParams().setDescription(descriptions.get(2)), - new QueryOptions(), token); + new QueryOptions(), ownerToken); catalogManager.getSampleManager().update(studyFqn, sampleId, - new SampleUpdateParams().setDescription("new description"), null, token); + new SampleUpdateParams().setDescription("new description"), null, ownerToken); // We want the whole history of the sample - query = new Query() + Query query = new Query() .append(SampleDBAdaptor.QueryParams.ID.key(), sampleId) .append(Constants.ALL_VERSIONS, true); - DataResult sampleDataResult = catalogManager.getSampleManager().search(studyFqn, query, null, token); + DataResult sampleDataResult = catalogManager.getSampleManager().search(studyFqn, query, null, ownerToken); assertEquals(5, sampleDataResult.getNumResults()); assertEquals("description", sampleDataResult.getResults().get(0).getDescription()); assertEquals(descriptions.get(0), sampleDataResult.getResults().get(1).getDescription()); @@ -273,7 +264,7 @@ public void testSampleVersioningWithWeirdId() throws CatalogException { query = new Query() .append(SampleDBAdaptor.QueryParams.VERSION.key(), "all"); sampleDataResult = catalogManager.getSampleManager().get(studyFqn, Collections.singletonList(sampleId), - query, null, false, token); + query, null, false, ownerToken); assertEquals(5, sampleDataResult.getNumResults()); assertEquals("description", sampleDataResult.getResults().get(0).getDescription()); assertEquals(descriptions.get(0), sampleDataResult.getResults().get(1).getDescription()); @@ -285,7 +276,7 @@ public void testSampleVersioningWithWeirdId() throws CatalogException { query = new Query() .append(SampleDBAdaptor.QueryParams.ID.key(), sampleId) .append(SampleDBAdaptor.QueryParams.SNAPSHOT.key(), 1); - sampleDataResult = catalogManager.getSampleManager().search(studyFqn, query, null, token); + sampleDataResult = catalogManager.getSampleManager().search(studyFqn, query, null, ownerToken); assertEquals(1, sampleDataResult.getNumResults()); assertEquals(3, sampleDataResult.first().getVersion()); @@ -293,14 +284,14 @@ public void testSampleVersioningWithWeirdId() throws CatalogException { query = new Query() .append(SampleDBAdaptor.QueryParams.ID.key(), sampleId) .append(SampleDBAdaptor.QueryParams.SNAPSHOT.key(), 2); - sampleDataResult = catalogManager.getSampleManager().search(studyFqn, query, null, token); + sampleDataResult = catalogManager.getSampleManager().search(studyFqn, query, null, ownerToken); assertEquals(1, sampleDataResult.getNumResults()); assertEquals(3, sampleDataResult.first().getVersion()); // We want the last version of the sample query = new Query() .append(SampleDBAdaptor.QueryParams.ID.key(), sampleId); - sampleDataResult = catalogManager.getSampleManager().search(studyFqn, query, null, token); + sampleDataResult = catalogManager.getSampleManager().search(studyFqn, query, null, ownerToken); assertEquals(1, sampleDataResult.getNumResults()); assertEquals(5, sampleDataResult.first().getVersion()); @@ -308,7 +299,7 @@ public void testSampleVersioningWithWeirdId() throws CatalogException { query = new Query() .append(SampleDBAdaptor.QueryParams.ID.key(), sampleId) .append(SampleDBAdaptor.QueryParams.VERSION.key(), 2); - sampleDataResult = catalogManager.getSampleManager().search(studyFqn, query, null, token); + sampleDataResult = catalogManager.getSampleManager().search(studyFqn, query, null, ownerToken); assertEquals(1, sampleDataResult.getNumResults()); assertEquals(2, sampleDataResult.first().getVersion()); @@ -316,12 +307,12 @@ public void testSampleVersioningWithWeirdId() throws CatalogException { query = new Query() .append(SampleDBAdaptor.QueryParams.ID.key(), sampleId) .append(SampleDBAdaptor.QueryParams.VERSION.key(), 1); - sampleDataResult = catalogManager.getSampleManager().search(studyFqn, query, null, token); + sampleDataResult = catalogManager.getSampleManager().search(studyFqn, query, null, ownerToken); assertEquals(1, sampleDataResult.getNumResults()); assertEquals(1, sampleDataResult.first().getVersion()); DataResult testSample = catalogManager.getSampleManager() - .get(studyFqn, Collections.singletonList(sampleId), new Query(Constants.ALL_VERSIONS, true), null, false, token); + .get(studyFqn, Collections.singletonList(sampleId), new Query(Constants.ALL_VERSIONS, true), null, false, ownerToken); assertEquals(5, testSample.getResults().size()); } @@ -332,7 +323,7 @@ public void searchByInternalAnnotationSetTest() throws CatalogException { variables.add(new Variable().setId("a").setType(Variable.VariableType.STRING)); variables.add(new Variable().setId("b").setType(Variable.VariableType.MAP_INTEGER).setAllowedKeys(Arrays.asList("b1", "b2"))); VariableSet variableSet = new VariableSet("myInternalVset", "", false, false, true, "", variables, null, 1, null); - catalogManager.getStudyManager().createVariableSet(studyFqn, variableSet, token); + catalogManager.getStudyManager().createVariableSet(studyFqn, variableSet, ownerToken); Map annotations = new HashMap<>(); annotations.put("a", "hello"); @@ -347,7 +338,7 @@ public void searchByInternalAnnotationSetTest() throws CatalogException { Sample sample = new Sample() .setId("sample") .setAnnotationSets(Arrays.asList(annotationSet, annotationSet2)); - Sample sampleResult = catalogManager.getSampleManager().create(studyFqn, sample, INCLUDE_RESULT, token).first(); + Sample sampleResult = catalogManager.getSampleManager().create(studyFqn, sample, INCLUDE_RESULT, ownerToken).first(); for (AnnotationSet aSet : sampleResult.getAnnotationSets()) { assertNotEquals(variableSet.getId(), aSet.getVariableSetId()); } @@ -366,54 +357,54 @@ public void searchByInternalAnnotationSetTest() throws CatalogException { Sample sample2 = new Sample() .setId("sample2") .setAnnotationSets(Arrays.asList(annotationSet, annotationSet2)); - sampleResult = catalogManager.getSampleManager().create(studyFqn, sample2, INCLUDE_RESULT, token).first(); + sampleResult = catalogManager.getSampleManager().create(studyFqn, sample2, INCLUDE_RESULT, ownerToken).first(); for (AnnotationSet aSet : sampleResult.getAnnotationSets()) { assertNotEquals(variableSet.getId(), aSet.getVariableSetId()); } // Query by one of the annotations Query query = new Query(Constants.ANNOTATION, "myInternalVset:a=hello"); - assertEquals(1, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); - assertEquals("sample", catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, token).first() + assertEquals(1, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); + assertEquals("sample", catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, ownerToken).first() .getId()); query = new Query(Constants.ANNOTATION, "myInternalVset:b.b1=" + (Integer.MAX_VALUE + 1L)); - assertEquals(1, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); - assertEquals("sample", catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, token).first() + assertEquals(1, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); + assertEquals("sample", catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, ownerToken).first() .getId()); query = new Query(Constants.ANNOTATION, "b.b1=14"); - assertEquals(1, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); - assertEquals("sample2", catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, token).first() + assertEquals(1, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); + assertEquals("sample2", catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, ownerToken).first() .getId()); query = new Query(Constants.ANNOTATION, "a=goodbye"); - assertEquals(1, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); - assertEquals("sample2", catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, token).first() + assertEquals(1, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); + assertEquals("sample2", catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, ownerToken).first() .getId()); // Update sample annotation to be exactly the same as sample2 ObjectMap action = new ObjectMap(SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key(), ParamUtils.BasicUpdateAction.SET); QueryOptions options = new QueryOptions(Constants.ACTIONS, action); catalogManager.getSampleManager().update(studyFqn, sample.getId(), - new SampleUpdateParams().setAnnotationSets(Arrays.asList(annotationSet, annotationSet2)), options, token); + new SampleUpdateParams().setAnnotationSets(Arrays.asList(annotationSet, annotationSet2)), options, ownerToken); query = new Query(Constants.ANNOTATION, "myInternalVset:a=hello"); - assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); query = new Query(Constants.ANNOTATION, "myInternalVset:b.b1=4"); - assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); query = new Query(Constants.ANNOTATION, "b.b1=14"); - assertEquals(2, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(2, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); assertTrue(Arrays.asList("sample", "sample2") - .containsAll(catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, token) + .containsAll(catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, ownerToken) .getResults().stream().map(Sample::getId).collect(Collectors.toList()))); query = new Query(Constants.ANNOTATION, "a=goodbye"); - assertEquals(2, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(2, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); assertTrue(Arrays.asList("sample", "sample2") - .containsAll(catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, token) + .containsAll(catalogManager.getSampleManager().search(studyFqn, query, SampleManager.INCLUDE_SAMPLE_IDS, ownerToken) .getResults().stream().map(Sample::getId).collect(Collectors.toList()))); } @@ -424,9 +415,9 @@ public void testMultipleUpdate() throws CatalogException { new SampleUpdateParams() .setSomatic(true) .setQualityControl(new SampleQualityControl()), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); OpenCGAResult result = catalogManager.getSampleManager().get(studyFqn, Arrays.asList("s_1", "s_2", "s_3"), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); assertEquals(3, result.getNumResults()); for (Sample sample : result.getResults()) { assertTrue(sample.isSomatic()); @@ -439,7 +430,7 @@ public void testMultipleUpdate() throws CatalogException { @Test public void updateQualityControlTest1() throws CatalogException { Sample sample = new Sample().setId("sample"); - catalogManager.getSampleManager().create(studyFqn, sample, QueryOptions.empty(), token); + catalogManager.getSampleManager().create(studyFqn, sample, QueryOptions.empty(), ownerToken); List sampleQcVariantStats = new ArrayList<>(); SampleVariantStats sampleVariantStats = new SampleVariantStats(); @@ -456,38 +447,38 @@ public void updateQualityControlTest1() throws CatalogException { SampleQualityControl qualityControl = new SampleQualityControl(null, null, metrics); OpenCGAResult result = catalogManager.getSampleManager().update(studyFqn, "sample", - new SampleUpdateParams().setQualityControl(qualityControl), QueryOptions.empty(), token); + new SampleUpdateParams().setQualityControl(qualityControl), QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumUpdated()); Query query = new Query() .append(ParamConstants.SAMPLE_VARIANT_STATS_ID_PARAM, "v1") .append(ParamConstants.SAMPLE_VARIANT_STATS_COUNT_PARAM, 20); - assertEquals(1, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(1, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); query = new Query() .append(ParamConstants.SAMPLE_VARIANT_STATS_ID_PARAM, "v2") .append(ParamConstants.SAMPLE_VARIANT_STATS_COUNT_PARAM, 10); - assertEquals(1, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(1, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); query = new Query() .append(ParamConstants.SAMPLE_VARIANT_STATS_ID_PARAM, "v1") .append(ParamConstants.SAMPLE_VARIANT_STATS_COUNT_PARAM, 15); - assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); query = new Query() .append(ParamConstants.SAMPLE_VARIANT_STATS_ID_PARAM, "v1") .append(ParamConstants.SAMPLE_VARIANT_STATS_TI_TV_RATIO_PARAM, 13.2); - assertEquals(1, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(1, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); query = new Query() .append(ParamConstants.SAMPLE_VARIANT_STATS_ID_PARAM, "v2") .append(ParamConstants.SAMPLE_VARIANT_STATS_TI_TV_RATIO_PARAM, 15.2); - assertEquals(1, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(1, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); query = new Query() .append(ParamConstants.SAMPLE_VARIANT_STATS_ID_PARAM, "v2") .append(ParamConstants.SAMPLE_VARIANT_STATS_TI_TV_RATIO_PARAM, 3.5); - assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); // Change values sampleQcVariantStats = new ArrayList<>(); @@ -500,120 +491,120 @@ public void updateQualityControlTest1() throws CatalogException { // And update sample result = catalogManager.getSampleManager().update(studyFqn, "sample", new SampleUpdateParams().setQualityControl(qualityControl), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumUpdated()); // Check same values as before but the results should be now different query = new Query() .append(ParamConstants.SAMPLE_VARIANT_STATS_ID_PARAM, "v1") .append(ParamConstants.SAMPLE_VARIANT_STATS_COUNT_PARAM, 20); - assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); query = new Query() .append(ParamConstants.SAMPLE_VARIANT_STATS_ID_PARAM, "v2") .append(ParamConstants.SAMPLE_VARIANT_STATS_COUNT_PARAM, 10); - assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); query = new Query() .append(ParamConstants.SAMPLE_VARIANT_STATS_ID_PARAM, "v1") .append(ParamConstants.SAMPLE_VARIANT_STATS_COUNT_PARAM, 15); - assertEquals(1, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(1, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); query = new Query() .append(ParamConstants.SAMPLE_VARIANT_STATS_ID_PARAM, "v1") .append(ParamConstants.SAMPLE_VARIANT_STATS_TI_TV_RATIO_PARAM, 13.2); - assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); query = new Query() .append(ParamConstants.SAMPLE_VARIANT_STATS_ID_PARAM, "v1") .append(ParamConstants.SAMPLE_VARIANT_STATS_TI_TV_RATIO_PARAM, 15.2); - assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); query = new Query() .append(ParamConstants.SAMPLE_VARIANT_STATS_ID_PARAM, "v1") .append(ParamConstants.SAMPLE_VARIANT_STATS_TI_TV_RATIO_PARAM, 3.5); - assertEquals(1, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(1, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); // Update any other sample field to validate it doesn't affect quality control result = catalogManager.getSampleManager().update(studyFqn, "sample", new SampleUpdateParams().setDescription("my description"), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumUpdated()); // Check same values as before but the results should be now different query = new Query() .append(ParamConstants.SAMPLE_VARIANT_STATS_ID_PARAM, "v1") .append(ParamConstants.SAMPLE_VARIANT_STATS_COUNT_PARAM, 20); - assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); query = new Query() .append(ParamConstants.SAMPLE_VARIANT_STATS_ID_PARAM, "v2") .append(ParamConstants.SAMPLE_VARIANT_STATS_COUNT_PARAM, 10); - assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); query = new Query() .append(ParamConstants.SAMPLE_VARIANT_STATS_ID_PARAM, "v1") .append(ParamConstants.SAMPLE_VARIANT_STATS_COUNT_PARAM, 15); - assertEquals(1, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(1, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); query = new Query() .append(ParamConstants.SAMPLE_VARIANT_STATS_ID_PARAM, "v1") .append(ParamConstants.SAMPLE_VARIANT_STATS_TI_TV_RATIO_PARAM, 13.2); - assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); query = new Query() .append(ParamConstants.SAMPLE_VARIANT_STATS_ID_PARAM, "v1") .append(ParamConstants.SAMPLE_VARIANT_STATS_TI_TV_RATIO_PARAM, 15.2); - assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); query = new Query() .append(ParamConstants.SAMPLE_VARIANT_STATS_ID_PARAM, "v1") .append(ParamConstants.SAMPLE_VARIANT_STATS_TI_TV_RATIO_PARAM, 3.5); - assertEquals(1, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(1, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); // Remove SampleQcVariantStats values qualityControl = new SampleQualityControl(Arrays.asList("file1", "file2"), null, null); // And update sample result = catalogManager.getSampleManager().update(studyFqn, "sample", new SampleUpdateParams().setQualityControl(qualityControl), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumUpdated()); // None of the previous queries should give any result query = new Query() .append(ParamConstants.SAMPLE_VARIANT_STATS_ID_PARAM, "v1") .append(ParamConstants.SAMPLE_VARIANT_STATS_COUNT_PARAM, 20); - assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); query = new Query() .append(ParamConstants.SAMPLE_VARIANT_STATS_ID_PARAM, "v2") .append(ParamConstants.SAMPLE_VARIANT_STATS_COUNT_PARAM, 10); - assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); query = new Query() .append(ParamConstants.SAMPLE_VARIANT_STATS_ID_PARAM, "v1") .append(ParamConstants.SAMPLE_VARIANT_STATS_COUNT_PARAM, 15); - assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); query = new Query() .append(ParamConstants.SAMPLE_VARIANT_STATS_ID_PARAM, "v1") .append(ParamConstants.SAMPLE_VARIANT_STATS_TI_TV_RATIO_PARAM, 13.2); - assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); query = new Query() .append(ParamConstants.SAMPLE_VARIANT_STATS_ID_PARAM, "v1") .append(ParamConstants.SAMPLE_VARIANT_STATS_TI_TV_RATIO_PARAM, 15.2); - assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); query = new Query() .append(ParamConstants.SAMPLE_VARIANT_STATS_ID_PARAM, "v1") .append(ParamConstants.SAMPLE_VARIANT_STATS_TI_TV_RATIO_PARAM, 3.5); - assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); } @Test public void updateQualityControlTest2() throws CatalogException { Sample sample = new Sample().setId("sample"); - catalogManager.getSampleManager().create(studyFqn, sample, QueryOptions.empty(), token); + catalogManager.getSampleManager().create(studyFqn, sample, QueryOptions.empty(), ownerToken); List sampleQcVariantStats = new ArrayList<>(); SampleVariantStats sampleVariantStats = new SampleVariantStats(); @@ -630,29 +621,29 @@ public void updateQualityControlTest2() throws CatalogException { SampleQualityControl qualityControl = new SampleQualityControl(null, null, metrics); OpenCGAResult result = catalogManager.getSampleManager().update(studyFqn, "sample", - new SampleUpdateParams().setQualityControl(qualityControl), QueryOptions.empty(), token); + new SampleUpdateParams().setQualityControl(qualityControl), QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumUpdated()); Query query = new Query(Constants.ANNOTATION, "opencga_sample_variant_stats__v1@opencga_sample_variant_stats:variantCount=20"); - assertEquals(1, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(1, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); query = new Query(Constants.ANNOTATION, "opencga_sample_variant_stats__v2@opencga_sample_variant_stats:variantCount=20"); - assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); query = new Query(Constants.ANNOTATION, "opencga_sample_variant_stats__v2@opencga_sample_variant_stats:variantCount=10"); - assertEquals(1, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(1, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); query = new Query(Constants.ANNOTATION, "opencga_sample_variant_stats:variantCount=15"); - assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); query = new Query(Constants.ANNOTATION, "opencga_sample_variant_stats:tiTvRatio=13.2"); - assertEquals(1, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(1, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); query = new Query(Constants.ANNOTATION, "opencga_sample_variant_stats:tiTvRatio=15.2"); - assertEquals(1, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(1, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); query = new Query(Constants.ANNOTATION, "opencga_sample_variant_stats:tiTvRatio=3.5"); - assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); // Change values sampleQcVariantStats = new ArrayList<>(); @@ -665,48 +656,48 @@ public void updateQualityControlTest2() throws CatalogException { // And update sample result = catalogManager.getSampleManager().update(studyFqn, "sample", new SampleUpdateParams().setQualityControl(qualityControl), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumUpdated()); // Check same values as before but the results should be now different query = new Query(Constants.ANNOTATION, "opencga_sample_variant_stats:variantCount=20"); - assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); query = new Query(Constants.ANNOTATION, "opencga_sample_variant_stats:variantCount=10"); - assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); query = new Query(Constants.ANNOTATION, "opencga_sample_variant_stats:variantCount=15"); - assertEquals(1, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(1, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); query = new Query(Constants.ANNOTATION, "opencga_sample_variant_stats:tiTvRatio=13.2"); - assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); query = new Query(Constants.ANNOTATION, "opencga_sample_variant_stats:tiTvRatio=15.2"); - assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); query = new Query(Constants.ANNOTATION, "opencga_sample_variant_stats:tiTvRatio=3.5"); - assertEquals(1, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(1, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); } @Test public void distinctTest() throws CatalogException { - OpenCGAResult distinct = catalogManager.getSampleManager().distinct(studyFqn, SampleDBAdaptor.QueryParams.ID.key(), null, token); + OpenCGAResult distinct = catalogManager.getSampleManager().distinct(organizationId, studyFqn, SampleDBAdaptor.QueryParams.ID.key(), null, ownerToken); assertEquals(String.class.getName(), distinct.getResultType()); assertEquals(9, distinct.getNumResults()); assertEquals(9, distinct.getResults().size()); distinct = catalogManager.getSampleManager().distinct(studyFqn, Arrays.asList(SampleDBAdaptor.QueryParams.ID.key(), - SampleDBAdaptor.QueryParams.UID.key()), null, token); + SampleDBAdaptor.QueryParams.UID.key()), null, ownerToken); assertEquals(String.class.getName(), distinct.getResultType()); assertEquals(18, distinct.getNumResults()); assertEquals(18, distinct.getResults().size()); - distinct = catalogManager.getSampleManager().distinct(studyFqn, SampleDBAdaptor.QueryParams.UID.key(), null, token); + distinct = catalogManager.getSampleManager().distinct(organizationId, studyFqn, SampleDBAdaptor.QueryParams.UID.key(), null, ownerToken); assertEquals(Long.class.getName(), distinct.getResultType()); assertEquals(9, distinct.getNumResults()); assertEquals(9, distinct.getResults().size()); - distinct = catalogManager.getSampleManager().distinct(studyFqn, SampleDBAdaptor.QueryParams.SOMATIC.key(), null, token); + distinct = catalogManager.getSampleManager().distinct(organizationId, studyFqn, SampleDBAdaptor.QueryParams.SOMATIC.key(), null, ownerToken); assertEquals(Boolean.class.getName(), distinct.getResultType()); assertEquals(1, distinct.getNumResults()); assertEquals(1, distinct.getResults().size()); @@ -715,12 +706,12 @@ public void distinctTest() throws CatalogException { @Test public void updateAndReturnResultTest() throws CatalogException { catalogManager.getSampleManager().create(studyFqn, - new Sample().setId("testSample").setDescription("description"), null, token); + new Sample().setId("testSample").setDescription("description"), null, ownerToken); SampleProcessing processing = new SampleProcessing(null, "preparationMethod", "extractionMethod", "labSampleId", "quantity", "date", Collections.emptyMap()); OpenCGAResult result = catalogManager.getSampleManager().update(studyFqn, "testSample", - new SampleUpdateParams().setProcessing(processing), new QueryOptions(), token); + new SampleUpdateParams().setProcessing(processing), new QueryOptions(), ownerToken); assertEquals(1, result.getNumUpdated()); assertEquals(1, result.getNumMatches()); assertEquals(0, result.getNumResults()); @@ -729,7 +720,7 @@ public void updateAndReturnResultTest() throws CatalogException { result = catalogManager.getSampleManager().update(studyFqn, "testSample", new SampleUpdateParams().setDescription("my new description"), new QueryOptions() .append(ParamConstants.INCLUDE_RESULT_PARAM, true) - , token); + , ownerToken); assertEquals(1, result.getNumUpdated()); assertEquals(1, result.getNumMatches()); assertEquals(1, result.getNumResults()); @@ -741,14 +732,14 @@ public void updateAndReturnResultTest() throws CatalogException { @Test public void updateProcessingField() throws CatalogException { catalogManager.getSampleManager().create(studyFqn, - new Sample().setId("testSample").setDescription("description"), null, token); + new Sample().setId("testSample").setDescription("description"), null, ownerToken); SampleProcessing processing = new SampleProcessing(new OntologyTermAnnotation().setId("product"), "preparationMethod", "extractionMethod", "labSampleId", "quantity", "date", Collections.emptyMap()); catalogManager.getSampleManager().update(studyFqn, "testSample", - new SampleUpdateParams().setProcessing(processing), new QueryOptions(), token); + new SampleUpdateParams().setProcessing(processing), new QueryOptions(), ownerToken); - DataResult testSample = catalogManager.getSampleManager().get(studyFqn, "testSample", new QueryOptions(), token); + DataResult testSample = catalogManager.getSampleManager().get(studyFqn, "testSample", new QueryOptions(), ownerToken); assertEquals("product", testSample.first().getProcessing().getProduct().getId()); assertEquals("preparationMethod", testSample.first().getProcessing().getPreparationMethod()); assertEquals("extractionMethod", testSample.first().getProcessing().getExtractionMethod()); @@ -761,16 +752,16 @@ public void updateProcessingField() throws CatalogException { @Test public void updateCollectionField() throws CatalogException { catalogManager.getSampleManager().create(studyFqn, - new Sample().setId("testSample").setDescription("description"), null, token); + new Sample().setId("testSample").setDescription("description"), null, ownerToken); SampleCollection collection = new SampleCollection(Collections.singletonList(new OntologyTermAnnotation("id", "name", "desc", "source", "", Collections.emptyMap())), "type", "quantity", "method", "date", Collections.emptyMap()); StatusParams statusParams = new StatusParams("status1", "status1", "my description"); catalogManager.getSampleManager().update(studyFqn, "testSample", new SampleUpdateParams().setCollection(collection).setStatus(statusParams), - new QueryOptions(), token); + new QueryOptions(), ownerToken); - DataResult testSample = catalogManager.getSampleManager().get(studyFqn, "testSample", new QueryOptions(), token); + DataResult testSample = catalogManager.getSampleManager().get(studyFqn, "testSample", new QueryOptions(), ownerToken); assertEquals(1, testSample.first().getCollection().getFrom().size()); assertEquals("id", testSample.first().getCollection().getFrom().get(0).getId()); assertEquals("name", testSample.first().getCollection().getFrom().get(0).getName()); @@ -794,7 +785,7 @@ public void testCreateSample() throws CatalogException { new Sample() .setId("HG007") .setStatus(new Status("stat1", "stat1", "my description", time)), - INCLUDE_RESULT, token); + INCLUDE_RESULT, ownerToken); assertEquals(1, sampleDataResult.getNumResults()); assertEquals("stat1", sampleDataResult.first().getStatus().getName()); assertEquals(time, sampleDataResult.first().getStatus().getDate()); @@ -818,29 +809,29 @@ public void testCreateSample() throws CatalogException { @Test public void testUpdateWithLockedClinicalAnalysis() throws CatalogException { Sample sample = new Sample().setId("sample1"); - catalogManager.getSampleManager().create(studyFqn, sample, QueryOptions.empty(), token); + catalogManager.getSampleManager().create(studyFqn, sample, QueryOptions.empty(), ownerToken); sample = new Sample().setId("sample2"); - catalogManager.getSampleManager().create(studyFqn, sample, QueryOptions.empty(), token); + catalogManager.getSampleManager().create(studyFqn, sample, QueryOptions.empty(), ownerToken); sample = new Sample().setId("sample3"); - catalogManager.getSampleManager().create(studyFqn, sample, QueryOptions.empty(), token); + catalogManager.getSampleManager().create(studyFqn, sample, QueryOptions.empty(), ownerToken); sample = new Sample().setId("sample4"); - catalogManager.getSampleManager().create(studyFqn, sample, QueryOptions.empty(), token); + catalogManager.getSampleManager().create(studyFqn, sample, QueryOptions.empty(), ownerToken); Individual individual = new Individual() .setId("proband") .setDisorders(Collections.singletonList(new Disorder().setId("disorder"))); catalogManager.getIndividualManager().create(studyFqn, individual, Arrays.asList("sample1", "sample2"), QueryOptions.empty(), - token); + ownerToken); individual = new Individual().setId("father"); catalogManager.getIndividualManager().create(studyFqn, individual, Arrays.asList("sample3", "sample4"), QueryOptions.empty(), - token); + ownerToken); Family family = new Family().setId("family"); - catalogManager.getFamilyManager().create(studyFqn, family, Arrays.asList("proband", "father"), QueryOptions.empty(), token); + catalogManager.getFamilyManager().create(studyFqn, family, Arrays.asList("proband", "father"), QueryOptions.empty(), ownerToken); family.setMembers(Arrays.asList( new Individual().setId("proband").setSamples(Collections.singletonList(new Sample().setId("sample2"))), @@ -853,7 +844,7 @@ public void testUpdateWithLockedClinicalAnalysis() throws CatalogException { .setFamily(family) .setLocked(true) .setType(ClinicalAnalysis.Type.FAMILY); - catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, QueryOptions.empty(), token); + catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, QueryOptions.empty(), ownerToken); // We will create another clinical analysis with the same information. In this test, we will not lock clinical2 clinicalAnalysis = new ClinicalAnalysis() @@ -861,36 +852,36 @@ public void testUpdateWithLockedClinicalAnalysis() throws CatalogException { .setProband(new Individual().setId("proband")) .setFamily(family) .setType(ClinicalAnalysis.Type.FAMILY); - catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, new QueryOptions(), token); + catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis, new QueryOptions(), ownerToken); // Update sample1 from proband not used in Clinical Analysis catalogManager.getSampleManager().update(studyFqn, "sample1", new SampleUpdateParams().setDescription("a"), new QueryOptions(), - token); + ownerToken); - Sample sampleResult = catalogManager.getSampleManager().get(studyFqn, "sample1", QueryOptions.empty(), token).first(); + Sample sampleResult = catalogManager.getSampleManager().get(studyFqn, "sample1", QueryOptions.empty(), ownerToken).first(); assertEquals(3, sampleResult.getVersion()); - Individual individualResult = catalogManager.getIndividualManager().get(studyFqn, "proband", QueryOptions.empty(), token).first(); + Individual individualResult = catalogManager.getIndividualManager().get(studyFqn, "proband", QueryOptions.empty(), ownerToken).first(); assertEquals(3, individualResult.getVersion()); assertEquals(2, individualResult.getSamples().size()); assertEquals(3, individualResult.getSamples().get(0).getVersion()); assertEquals(2, individualResult.getSamples().get(1).getVersion()); - Family familyResult = catalogManager.getFamilyManager().get(studyFqn, "family", QueryOptions.empty(), token).first(); + Family familyResult = catalogManager.getFamilyManager().get(studyFqn, "family", QueryOptions.empty(), ownerToken).first(); assertEquals(2, familyResult.getVersion()); assertEquals(2, familyResult.getMembers().size()); assertEquals(3, familyResult.getMembers().get(0).getVersion()); assertEquals(2, familyResult.getMembers().get(1).getVersion()); ClinicalAnalysis clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, "clinical", QueryOptions.empty(), - token).first(); + ownerToken).first(); assertEquals(2, clinicalResult.getProband().getVersion()); assertEquals(2, clinicalResult.getProband().getSamples().get(0).getVersion()); // sample2 version assertEquals(1, clinicalResult.getFamily().getVersion()); assertEquals(2, clinicalResult.getFamily().getMembers().get(0).getVersion()); // proband version assertEquals(2, clinicalResult.getFamily().getMembers().get(1).getVersion()); // father version - clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, "clinical2", QueryOptions.empty(), token).first(); + clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, "clinical2", QueryOptions.empty(), ownerToken).first(); assertEquals(3, clinicalResult.getProband().getVersion()); assertEquals(2, clinicalResult.getProband().getSamples().get(0).getVersion()); // sample2 version assertEquals(2, clinicalResult.getFamily().getVersion()); @@ -899,24 +890,24 @@ public void testUpdateWithLockedClinicalAnalysis() throws CatalogException { // Update sample used in Clinical Analysis from father catalogManager.getSampleManager().update(studyFqn, "sample3", new SampleUpdateParams().setDescription("asd"), - new QueryOptions(), token); + new QueryOptions(), ownerToken); - sampleResult = catalogManager.getSampleManager().get(studyFqn, "sample3", QueryOptions.empty(), token).first(); + sampleResult = catalogManager.getSampleManager().get(studyFqn, "sample3", QueryOptions.empty(), ownerToken).first(); assertEquals(3, sampleResult.getVersion()); - individualResult = catalogManager.getIndividualManager().get(studyFqn, "father", QueryOptions.empty(), token).first(); + individualResult = catalogManager.getIndividualManager().get(studyFqn, "father", QueryOptions.empty(), ownerToken).first(); assertEquals(3, individualResult.getVersion()); assertEquals(2, individualResult.getSamples().size()); assertEquals(3, individualResult.getSamples().get(0).getVersion()); assertEquals(2, individualResult.getSamples().get(1).getVersion()); - familyResult = catalogManager.getFamilyManager().get(studyFqn, "family", QueryOptions.empty(), token).first(); + familyResult = catalogManager.getFamilyManager().get(studyFqn, "family", QueryOptions.empty(), ownerToken).first(); assertEquals(3, familyResult.getVersion()); assertEquals(2, familyResult.getMembers().size()); assertEquals(3, familyResult.getMembers().get(0).getVersion()); assertEquals(3, familyResult.getMembers().get(1).getVersion()); - clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, "clinical", QueryOptions.empty(), token).first(); + clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, "clinical", QueryOptions.empty(), ownerToken).first(); assertEquals(2, clinicalResult.getProband().getVersion()); assertEquals(2, clinicalResult.getProband().getSamples().get(0).getVersion()); // sample2 version assertEquals(1, clinicalResult.getFamily().getVersion()); @@ -924,7 +915,7 @@ public void testUpdateWithLockedClinicalAnalysis() throws CatalogException { assertEquals(2, clinicalResult.getFamily().getMembers().get(1).getVersion()); // father version assertEquals(2, clinicalResult.getFamily().getMembers().get(1).getSamples().get(0).getVersion()); // father sample3 version - clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, "clinical2", QueryOptions.empty(), token).first(); + clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, "clinical2", QueryOptions.empty(), ownerToken).first(); assertEquals(3, clinicalResult.getProband().getVersion()); assertEquals(2, clinicalResult.getProband().getSamples().get(0).getVersion()); // sample2 version assertEquals(3, clinicalResult.getFamily().getVersion()); @@ -933,33 +924,33 @@ public void testUpdateWithLockedClinicalAnalysis() throws CatalogException { assertEquals(3, clinicalResult.getFamily().getMembers().get(1).getSamples().get(0).getVersion()); // father sample3 version catalogManager.getSampleManager().updateSampleInternalVariantSecondarySampleIndex( - catalogManager.getSampleManager().get(studyFqn, "sample3", null, token).first(), + studyFqn, catalogManager.getSampleManager().get(studyFqn, "sample3", null, ownerToken).first(), new SampleInternalVariantSecondarySampleIndex( new IndexStatus(IndexStatus.READY, "This should be doable!"), new IndexStatus(IndexStatus.READY, "This should be doable!"), 12 - ), token); + ), ownerToken); // Update sample 2 from proband catalogManager.getSampleManager().update(studyFqn, "sample2", new SampleUpdateParams().setDescription("asda"), - new QueryOptions(), token); + new QueryOptions(), ownerToken); - sampleResult = catalogManager.getSampleManager().get(studyFqn, "sample2", QueryOptions.empty(), token).first(); + sampleResult = catalogManager.getSampleManager().get(studyFqn, "sample2", QueryOptions.empty(), ownerToken).first(); assertEquals(3, sampleResult.getVersion()); - individualResult = catalogManager.getIndividualManager().get(studyFqn, "proband", QueryOptions.empty(), token).first(); + individualResult = catalogManager.getIndividualManager().get(studyFqn, "proband", QueryOptions.empty(), ownerToken).first(); assertEquals(4, individualResult.getVersion()); assertEquals(2, individualResult.getSamples().size()); assertEquals(3, individualResult.getSamples().get(0).getVersion()); assertEquals(3, individualResult.getSamples().get(1).getVersion()); - familyResult = catalogManager.getFamilyManager().get(studyFqn, "family", QueryOptions.empty(), token).first(); + familyResult = catalogManager.getFamilyManager().get(studyFqn, "family", QueryOptions.empty(), ownerToken).first(); assertEquals(5, familyResult.getVersion()); assertEquals(2, familyResult.getMembers().size()); assertEquals(4, familyResult.getMembers().get(0).getVersion()); assertEquals(4, familyResult.getMembers().get(1).getVersion()); - clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, "clinical", QueryOptions.empty(), token).first(); + clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, "clinical", QueryOptions.empty(), ownerToken).first(); assertEquals(2, clinicalResult.getProband().getVersion()); assertEquals(2, clinicalResult.getProband().getSamples().get(0).getVersion()); // sample2 version assertEquals(1, clinicalResult.getFamily().getVersion()); @@ -967,7 +958,7 @@ public void testUpdateWithLockedClinicalAnalysis() throws CatalogException { assertEquals(2, clinicalResult.getFamily().getMembers().get(1).getVersion()); // father version assertEquals(2, clinicalResult.getFamily().getMembers().get(1).getSamples().get(0).getVersion()); // father sample3 version - clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, "clinical2", QueryOptions.empty(), token).first(); + clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, "clinical2", QueryOptions.empty(), ownerToken).first(); assertEquals(4, clinicalResult.getProband().getVersion()); assertEquals(3, clinicalResult.getProband().getSamples().get(0).getVersion()); // sample2 version assertEquals(5, clinicalResult.getFamily().getVersion()); @@ -981,7 +972,7 @@ public void testUpdateWithLockedClinicalAnalysis() throws CatalogException { public void testCreateSampleWithDotInName() throws CatalogException { String name = "HG007.sample"; DataResult sampleDataResult = catalogManager.getSampleManager().create(studyFqn, new Sample().setId(name), INCLUDE_RESULT, - token); + ownerToken); assertEquals(name, sampleDataResult.first().getId()); } @@ -997,7 +988,7 @@ public void testAnnotate() throws CatalogException { variables.add(new Variable("MAP", "", Variable.VariableType.OBJECT, new HashMap<>(), false, false, Collections.emptyList(), null, 0, "", "", null, Collections.emptyMap())); VariableSet vs1 = catalogManager.getStudyManager().createVariableSet(studyFqn, "vs1", "vs1", false, false, "", null, variables, - Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), token).first(); + Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), ownerToken).first(); HashMap annotations = new HashMap<>(); annotations.put("NAME", "Joe"); @@ -1005,12 +996,12 @@ public void testAnnotate() throws CatalogException { annotations.put("HEIGHT", 180); annotations.put("MAP", new ObjectMap("unknownKey1", "value1").append("unknownKey2", 42)); - catalogManager.getSampleManager().update(studyFqn, s_1, new SampleUpdateParams() + catalogManager.getSampleManager().update(studyFqn, s_1Id, new SampleUpdateParams() .setAnnotationSets(Collections.singletonList(new AnnotationSet("annotation1", vs1.getId(), annotations))), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); - DataResult sampleDataResult = catalogManager.getSampleManager().get(studyFqn, s_1, - new QueryOptions(QueryOptions.INCLUDE, Constants.ANNOTATION_SET_NAME + ".annotation1"), token); + DataResult sampleDataResult = catalogManager.getSampleManager().get(studyFqn, s_1Id, + new QueryOptions(QueryOptions.INCLUDE, Constants.ANNOTATION_SET_NAME + ".annotation1"), ownerToken); assertEquals(1, sampleDataResult.getNumResults()); assertEquals(1, sampleDataResult.first().getAnnotationSets().size()); @@ -1055,18 +1046,18 @@ public void testDynamicAnnotationsCreation() throws CatalogException, IOExceptio Collections.emptyMap()))), Collections.emptyMap())); VariableSet vs1 = catalogManager.getStudyManager().createVariableSet(studyFqn, "vs1", "vs1", false, false, "", null, variables, - Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), token).first(); + Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), ownerToken).first(); InputStream inputStream = this.getClass().getClassLoader().getResource("annotation_sets/complete_annotation.json").openStream(); ObjectMapper objectMapper = new ObjectMapper(); ObjectMap annotations = objectMapper.readValue(inputStream, ObjectMap.class); - catalogManager.getSampleManager().update(studyFqn, s_1, new SampleUpdateParams() + catalogManager.getSampleManager().update(studyFqn, s_1Id, new SampleUpdateParams() .setAnnotationSets(Collections.singletonList(new AnnotationSet("annotation1", vs1.getId(), annotations))), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); - DataResult sampleDataResult = catalogManager.getSampleManager().get(studyFqn, s_1, - new QueryOptions(QueryOptions.INCLUDE, Constants.ANNOTATION_SET_NAME + ".annotation1"), token); + DataResult sampleDataResult = catalogManager.getSampleManager().get(studyFqn, s_1Id, + new QueryOptions(QueryOptions.INCLUDE, Constants.ANNOTATION_SET_NAME + ".annotation1"), ownerToken); assertEquals(objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(annotations), objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(sampleDataResult.first().getAnnotationSets().get(0) @@ -1078,9 +1069,9 @@ public void testDynamicAnnotationsCreation() throws CatalogException, IOExceptio thrown.expect(CatalogException.class); thrown.expectMessage("Missing required variable"); - catalogManager.getSampleManager().update(studyFqn, s_2, new SampleUpdateParams() + catalogManager.getSampleManager().update(studyFqn, s_2Id, new SampleUpdateParams() .setAnnotationSets(Collections.singletonList(new AnnotationSet("annotation1", vs1.getId(), annotations))), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); } @Test @@ -1091,16 +1082,16 @@ public void testWrongAnnotation() throws CatalogException { variables.add(new Variable("b", "b", "", Variable.VariableType.STRING, null, true, false, null, null, 0, "", "", Collections.emptySet(), Collections.emptyMap())); VariableSet vs1 = catalogManager.getStudyManager().createVariableSet(studyFqn, "vs1", "vs1", false, false, "", null, variables, - Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), token).first(); + Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), ownerToken).first(); ObjectMap annotation = new ObjectMap() .append("a", 5) .append("b", "my_string"); try { - catalogManager.getSampleManager().update(studyFqn, s_1, new SampleUpdateParams() + catalogManager.getSampleManager().update(studyFqn, s_1Id, new SampleUpdateParams() .setAnnotationSets(Collections.singletonList(new AnnotationSet("annotation1", vs1.getId(), annotation))), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); fail("Annotation '5' should not be valid for variable MAP_INTEGER"); } catch (CatalogException e) { assertTrue(e.getMessage().contains("does not seem appropriate for variable")); @@ -1108,9 +1099,9 @@ public void testWrongAnnotation() throws CatalogException { annotation.put("a", new ObjectMap("b", "test")); try { - catalogManager.getSampleManager().update(studyFqn, s_1, new SampleUpdateParams() + catalogManager.getSampleManager().update(studyFqn, s_1Id, new SampleUpdateParams() .setAnnotationSets(Collections.singletonList(new AnnotationSet("annotation1", vs1.getId(), annotation))), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); fail("Annotation 'a.b' should not be INTEGER instead of STRING"); } catch (CatalogException e) { assertTrue(e.getMessage().contains("Expected an integer map")); @@ -1118,18 +1109,18 @@ public void testWrongAnnotation() throws CatalogException { annotation.put("a", new ObjectMap("b", "46")); try { - catalogManager.getSampleManager().update(studyFqn, s_1, new SampleUpdateParams() + catalogManager.getSampleManager().update(studyFqn, s_1Id, new SampleUpdateParams() .setAnnotationSets(Collections.singletonList(new AnnotationSet("annotation1", vs1.getId(), annotation))), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); fail("Annotation 'a.b' should not be INTEGER instead of STRING"); } catch (CatalogException e) { assertTrue(e.getMessage().contains("Expected an integer map")); } annotation.put("a", new ObjectMap("b", 46)); - catalogManager.getSampleManager().update(studyFqn, s_1, new SampleUpdateParams() + catalogManager.getSampleManager().update(studyFqn, s_1Id, new SampleUpdateParams() .setAnnotationSets(Collections.singletonList(new AnnotationSet("annotation1", vs1.getId(), annotation))), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); } @Test @@ -1161,15 +1152,15 @@ public void testVariableAllowedKeys() throws CatalogException, IOException { Collections.emptyMap()))), Collections.emptyMap())); VariableSet vs1 = catalogManager.getStudyManager().createVariableSet(studyFqn, "vs1", "vs1", false, false, "", null, variables, - Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), token).first(); + Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), ownerToken).first(); InputStream inputStream = this.getClass().getClassLoader().getResource("annotation_sets/complete_annotation.json").openStream(); ObjectMapper objectMapper = new ObjectMapper(); ObjectMap annotations = objectMapper.readValue(inputStream, ObjectMap.class); - catalogManager.getSampleManager().update(studyFqn, s_1, new SampleUpdateParams() + catalogManager.getSampleManager().update(studyFqn, s_1Id, new SampleUpdateParams() .setAnnotationSets(Collections.singletonList(new AnnotationSet("annotation1", vs1.getId(), annotations))), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); // The only keys covered in the example annotationset are x, y, z; so we will change the allowedKeys so the annotation should be @@ -1201,13 +1192,13 @@ public void testVariableAllowedKeys() throws CatalogException, IOException { Collections.emptyMap()))), Collections.emptyMap())); VariableSet vs2 = catalogManager.getStudyManager().createVariableSet(studyFqn, "vs2", "vs2", false, false, "", null, variables, - Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), token).first(); + Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), ownerToken).first(); thrown.expect(CatalogException.class); thrown.expectMessage("not an accepted key"); - catalogManager.getSampleManager().update(studyFqn, s_1, new SampleUpdateParams() + catalogManager.getSampleManager().update(studyFqn, s_1Id, new SampleUpdateParams() .setAnnotationSets(Collections.singletonList(new AnnotationSet("annotation2", vs2.getId(), annotations))), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); } @Test @@ -1239,7 +1230,7 @@ public void testDynamicAnnotationsSearch() throws CatalogException, IOException variables.add(new Variable("a4", "a4", "", Variable.VariableType.MAP_INTEGER, null, true, false, null, null, 0, "", "", Collections.emptySet(), Collections.emptyMap())); VariableSet vs1 = catalogManager.getStudyManager().createVariableSet(studyFqn, "vs1", "vs1", false, false, "", null, variables, - Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), token).first(); + Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), ownerToken).first(); InputStream inputStream = this.getClass().getClassLoader().getResource("annotation_sets/complete_annotation.json").openStream(); ObjectMapper objectMapper = new ObjectMapper(); @@ -1255,46 +1246,46 @@ public void testDynamicAnnotationsSearch() throws CatalogException, IOException .append(".asd", 4) ); - catalogManager.getSampleManager().update(studyFqn, s_1, new SampleUpdateParams() + catalogManager.getSampleManager().update(studyFqn, s_1Id, new SampleUpdateParams() .setAnnotationSets(Collections.singletonList(new AnnotationSet("annotation1", vs1.getId(), annotations))), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); Query query = new Query(Constants.ANNOTATION, "a3.b.c.z=z2;a2.b.c.z=z3"); QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, "annotationSet.annotation1"); - assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, token).getNumMatches()); + assertEquals(0, catalogManager.getSampleManager().count(studyFqn, query, ownerToken).getNumMatches()); query = new Query(Constants.ANNOTATION, "a3.b.c.z=z2;a2.b.c.z=z"); - OpenCGAResult result = catalogManager.getSampleManager().search(studyFqn, query, options, token); + OpenCGAResult result = catalogManager.getSampleManager().search(studyFqn, query, options, ownerToken); assertEquals(1, result.getNumResults()); - assertEquals(s_1, result.first().getId()); + assertEquals(s_1Id, result.first().getId()); assertEquals(7, ((Map) result.first().getAnnotationSets().get(0).getAnnotations().get("a4")).size()); options = new QueryOptions(ParamConstants.FLATTEN_ANNOTATIONS, true); - result = catalogManager.getSampleManager().search(studyFqn, query, options, token); + result = catalogManager.getSampleManager().search(studyFqn, query, options, ownerToken); assertEquals(1, result.getNumResults()); - assertEquals(s_1, result.first().getId()); + assertEquals(s_1Id, result.first().getId()); query = new Query(Constants.ANNOTATION, "a4..=3"); - result = catalogManager.getSampleManager().search(studyFqn, query, options, token); + result = catalogManager.getSampleManager().search(studyFqn, query, options, ownerToken); assertEquals(1, result.getNumResults()); query = new Query(Constants.ANNOTATION, "a4..=2"); - result = catalogManager.getSampleManager().search(studyFqn, query, options, token); + result = catalogManager.getSampleManager().search(studyFqn, query, options, ownerToken); assertEquals(0, result.getNumResults()); } @Test public void searchSamples() throws CatalogException { - catalogManager.getStudyManager().createGroup(studyFqn, "myGroup", Arrays.asList("user2", "user3"), token); - catalogManager.getStudyManager().createGroup(studyFqn, "myGroup2", Arrays.asList("user2", "user3"), token); - catalogManager.getStudyManager().updateAcl(Arrays.asList(studyFqn), "@myGroup", - new StudyAclParams("", null), SET, token); + catalogManager.getStudyManager().createGroup(studyFqn2, "myGroup", Arrays.asList(normalUserId1, normalUserId3), ownerToken); + catalogManager.getStudyManager().createGroup(studyFqn2, "myGroup2", Arrays.asList(normalUserId1, normalUserId3), ownerToken); + catalogManager.getStudyManager().updateAcl(studyFqn2, "@myGroup", + new StudyAclParams("", null), SET, ownerToken); - catalogManager.getSampleManager().updateAcl(studyFqn, Arrays.asList("s_1"), "@myGroup", - new SampleAclParams(null, null, null, null, "VIEW"), SET, token); + catalogManager.getSampleManager().create(studyFqn2, new Sample().setId("s_1"), QueryOptions.empty(), ownerToken); + catalogManager.getSampleManager().updateAcl(studyFqn2, Arrays.asList("s_1"), "@myGroup", + new SampleAclParams(null, null, null, null, "VIEW"), SET, ownerToken); - DataResult search = catalogManager.getSampleManager().search(studyFqn, new Query(), new QueryOptions(), - sessionIdUser2); + DataResult search = catalogManager.getSampleManager().search(studyFqn2, new Query(), new QueryOptions(), normalToken1); assertEquals(1, search.getNumResults()); } @@ -1308,7 +1299,7 @@ public void testDeleteAnnotationset() throws CatalogException, JsonProcessingExc variables.add(new Variable("HEIGHT", "", "", Variable.VariableType.DOUBLE, "", false, false, Collections.emptyList(), null, 0, "", "", null, Collections.emptyMap())); VariableSet vs1 = catalogManager.getStudyManager().createVariableSet(studyFqn, "vs1", "vs1", false, false, "", null, variables, - Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), token).first(); + Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), ownerToken).first(); ObjectMap annotations = new ObjectMap() .append("var_name", "Joe") @@ -1317,24 +1308,24 @@ public void testDeleteAnnotationset() throws CatalogException, JsonProcessingExc AnnotationSet annotationSet = new AnnotationSet("annotation1", vs1.getId(), annotations); AnnotationSet annotationSet1 = new AnnotationSet("annotation2", vs1.getId(), annotations); - DataResult update = catalogManager.getSampleManager().update(studyFqn, s_1, new SampleUpdateParams() - .setAnnotationSets(Arrays.asList(annotationSet, annotationSet1)), QueryOptions.empty(), token); + DataResult update = catalogManager.getSampleManager().update(studyFqn, s_1Id, new SampleUpdateParams() + .setAnnotationSets(Arrays.asList(annotationSet, annotationSet1)), QueryOptions.empty(), ownerToken); assertEquals(1, update.getNumUpdated()); - Sample sample = catalogManager.getSampleManager().get(studyFqn, s_1, QueryOptions.empty(), token).first(); + Sample sample = catalogManager.getSampleManager().get(studyFqn, s_1Id, QueryOptions.empty(), ownerToken).first(); assertEquals(3, sample.getAnnotationSets().size()); - catalogManager.getSampleManager().removeAnnotationSet(studyFqn, s_1, "annotation1", QueryOptions.empty(), token); + catalogManager.getSampleManager().removeAnnotationSet(studyFqn, s_1Id, "annotation1", QueryOptions.empty(), ownerToken); update = catalogManager.getSampleManager() - .removeAnnotationSet(studyFqn, s_1, "annotation2", QueryOptions.empty(), token); + .removeAnnotationSet(studyFqn, s_1Id, "annotation2", QueryOptions.empty(), ownerToken); assertEquals(1, update.getNumUpdated()); - sample = catalogManager.getSampleManager().get(studyFqn, s_1, QueryOptions.empty(), token).first(); + sample = catalogManager.getSampleManager().get(studyFqn, s_1Id, QueryOptions.empty(), ownerToken).first(); assertEquals(1, sample.getAnnotationSets().size()); thrown.expect(CatalogDBException.class); thrown.expectMessage("not found"); - catalogManager.getSampleManager().removeAnnotationSet(studyFqn, s_1, "non_existing", QueryOptions.empty(), token); + catalogManager.getSampleManager().removeAnnotationSet(studyFqn, s_1Id, "non_existing", QueryOptions.empty(), ownerToken); } @Test @@ -1349,7 +1340,7 @@ public void testSearchAnnotation() throws CatalogException, JsonProcessingExcept variables.add(new Variable("OTHER", "", "", Variable.VariableType.OBJECT, null, false, false, null, null, 1, "", "", null, Collections.emptyMap())); VariableSet vs1 = catalogManager.getStudyManager().createVariableSet(studyFqn, "vs1", "vs1", false, false, "", null, variables, - Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), token).first(); + Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), ownerToken).first(); ObjectMap annotations = new ObjectMap() .append("var_name", "Joe") @@ -1357,50 +1348,50 @@ public void testSearchAnnotation() throws CatalogException, JsonProcessingExcept .append("HEIGHT", 180); AnnotationSet annotationSet = new AnnotationSet("annotation1", vs1.getId(), annotations); - catalogManager.getSampleManager().update(studyFqn, s_1, new SampleUpdateParams() - .setAnnotationSets(Collections.singletonList(annotationSet)), QueryOptions.empty(), token); + catalogManager.getSampleManager().update(studyFqn, s_1Id, new SampleUpdateParams() + .setAnnotationSets(Collections.singletonList(annotationSet)), QueryOptions.empty(), ownerToken); Query query = new Query(Constants.ANNOTATION, "var_name=Joe;" + vs1.getId() + ":AGE=25"); DataResult annotDataResult = catalogManager.getSampleManager().search(studyFqn, query, QueryOptions.empty(), - token); + ownerToken); assertEquals(1, annotDataResult.getNumResults()); query.put(Constants.ANNOTATION, "var_name=Joe;" + vs1.getId() + ":AGE=23"); - annotDataResult = catalogManager.getSampleManager().search(studyFqn, query, QueryOptions.empty(), token); + annotDataResult = catalogManager.getSampleManager().search(studyFqn, query, QueryOptions.empty(), ownerToken); assertEquals(0, annotDataResult.getNumResults()); query.put(Constants.ANNOTATION, "var_name=Joe;" + vs1.getId() + ":AGE=25;variableSet!=" + vs1.getId()); - annotDataResult = catalogManager.getSampleManager().search(studyFqn, query, QueryOptions.empty(), token); + annotDataResult = catalogManager.getSampleManager().search(studyFqn, query, QueryOptions.empty(), ownerToken); assertEquals(1, annotDataResult.getNumResults()); query.put(Constants.ANNOTATION, "var_name=Joe;" + vs1.getId() + ":AGE=25;variableSet!==" + vs1.getId()); - annotDataResult = catalogManager.getSampleManager().search(studyFqn, query, QueryOptions.empty(), token); + annotDataResult = catalogManager.getSampleManager().search(studyFqn, query, QueryOptions.empty(), ownerToken); assertEquals(0, annotDataResult.getNumResults()); query.put(Constants.ANNOTATION, "var_name=Joe;" + vs1.getId() + ":AGE=25;variableSet==" + vs1.getId()); - annotDataResult = catalogManager.getSampleManager().search(studyFqn, query, QueryOptions.empty(), token); + annotDataResult = catalogManager.getSampleManager().search(studyFqn, query, QueryOptions.empty(), ownerToken); assertEquals(1, annotDataResult.getNumResults()); query.put(Constants.ANNOTATION, "var_name=Joe;" + vs1.getId() + ":AGE=25;variableSet===" + vs1.getId()); - annotDataResult = catalogManager.getSampleManager().search(studyFqn, query, QueryOptions.empty(), token); + annotDataResult = catalogManager.getSampleManager().search(studyFqn, query, QueryOptions.empty(), ownerToken); assertEquals(0, annotDataResult.getNumResults()); - VariableSet vs = catalogManager.getStudyManager().getVariableSet(studyFqn, "vs", null, token).first(); + VariableSet vs = catalogManager.getStudyManager().getVariableSet(studyFqn, "vs", null, ownerToken).first(); query.put(Constants.ANNOTATION, "variableSet===" + vs.getId()); - annotDataResult = catalogManager.getSampleManager().search(studyFqn, query, QueryOptions.empty(), token); + annotDataResult = catalogManager.getSampleManager().search(studyFqn, query, QueryOptions.empty(), ownerToken); assertEquals(7, annotDataResult.getNumResults()); query.put(Constants.ANNOTATION, "variableSet!=" + vs1.getId()); - annotDataResult = catalogManager.getSampleManager().search(studyFqn, query, QueryOptions.empty(), token); + annotDataResult = catalogManager.getSampleManager().search(studyFqn, query, QueryOptions.empty(), ownerToken); assertEquals(9, annotDataResult.getNumResults()); query.put(Constants.ANNOTATION, "variableSet!==" + vs1.getId()); - annotDataResult = catalogManager.getSampleManager().search(studyFqn, query, QueryOptions.empty(), token); + annotDataResult = catalogManager.getSampleManager().search(studyFqn, query, QueryOptions.empty(), ownerToken); assertEquals(8, annotDataResult.getNumResults()); query.put(Constants.ANNOTATION, "variableSet=" + vs1.getId()); annotDataResult = catalogManager.getSampleManager().search(studyFqn, query, - new QueryOptions(QueryOptions.INCLUDE, Constants.VARIABLE_SET + "." + vs1.getId()), token); + new QueryOptions(QueryOptions.INCLUDE, Constants.VARIABLE_SET + "." + vs1.getId()), ownerToken); assertEquals(1, annotDataResult.getNumResults()); assertEquals(1, annotDataResult.first().getAnnotationSets().size()); assertEquals(vs1.getId(), annotDataResult.first().getAnnotationSets().get(0).getVariableSetId()); @@ -1408,12 +1399,12 @@ public void testSearchAnnotation() throws CatalogException, JsonProcessingExcept @Test public void testProjections() throws CatalogException { - VariableSet variableSet = catalogManager.getStudyManager().getVariableSet("1000G:phase1", "vs", null, token).first(); + VariableSet variableSet = catalogManager.getStudyManager().getVariableSet("1000G:phase1", "vs", null, ownerToken).first(); Query query = new Query(Constants.ANNOTATION, "variableSet===" + variableSet.getId()); QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, "annotationSets"); DataResult annotDataResult = catalogManager.getSampleManager().search(studyFqn, query, options, - token); + ownerToken); assertEquals(8, annotDataResult.getNumResults()); for (Sample sample : annotDataResult.getResults()) { @@ -1425,13 +1416,13 @@ public void testProjections() throws CatalogException { @Test public void testAnnotateMulti() throws CatalogException { String sampleId = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_1"), INCLUDE_RESULT, - token).first().getId(); + ownerToken).first().getId(); List variables = new ArrayList<>(); variables.add(new Variable("NAME", "NAME", "", Variable.VariableType.STRING, "", true, false, Collections.emptyList(), null, 0, "", "", null, Collections.emptyMap())); VariableSet vs1 = catalogManager.getStudyManager().createVariableSet(studyFqn, "vs1", "vs1", false, false, "", null, variables, - Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), token).first(); + Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), ownerToken).first(); HashMap annotations = new HashMap<>(); @@ -1439,18 +1430,18 @@ public void testAnnotateMulti() throws CatalogException { catalogManager.getSampleManager().update(studyFqn, sampleId, new SampleUpdateParams() .setAnnotationSets(Collections.singletonList(new AnnotationSet("annotation1", vs1.getId(), annotations))), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); DataResult sampleDataResult = catalogManager.getSampleManager().get(studyFqn, sampleId, - new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key()), token); + new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key()), ownerToken); assertEquals(1, sampleDataResult.first().getAnnotationSets().size()); annotations = new HashMap<>(); annotations.put("NAME", "Lucas"); catalogManager.getSampleManager().update(studyFqn, sampleId, new SampleUpdateParams() .setAnnotationSets(Collections.singletonList(new AnnotationSet("annotation2", vs1.getId(), annotations))), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); sampleDataResult = catalogManager.getSampleManager().get(studyFqn, sampleId, - new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key()), token); + new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key()), ownerToken); assertEquals(2, sampleDataResult.first().getAnnotationSets().size()); assertTrue(Arrays.asList("annotation1", "annotation2") @@ -1460,13 +1451,13 @@ public void testAnnotateMulti() throws CatalogException { @Test public void testAnnotateUnique() throws CatalogException { String sampleId = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_1"), INCLUDE_RESULT, - token).first().getId(); + ownerToken).first().getId(); List variables = new ArrayList<>(); variables.add(new Variable("NAME", "NAME", "", Variable.VariableType.STRING, "", true, false, Collections.emptyList(), null, 0, "", "", null, Collections.emptyMap())); VariableSet vs1 = catalogManager.getStudyManager().createVariableSet(studyFqn, "vs1", "vs1", true, false, "", null, variables, - Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), token).first(); + Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), ownerToken).first(); HashMap annotations = new HashMap<>(); @@ -1474,9 +1465,9 @@ public void testAnnotateUnique() throws CatalogException { catalogManager.getSampleManager().update(studyFqn, sampleId, new SampleUpdateParams() .setAnnotationSets(Collections.singletonList(new AnnotationSet("annotation1", vs1.getId(), annotations))), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); DataResult sampleDataResult = catalogManager.getSampleManager().get(studyFqn, sampleId, - new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key()), token); + new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key()), ownerToken); assertEquals(1, sampleDataResult.first().getAnnotationSets().size()); annotations.put("NAME", "Lucas"); @@ -1484,28 +1475,28 @@ public void testAnnotateUnique() throws CatalogException { thrown.expectMessage("unique"); catalogManager.getSampleManager().update(studyFqn, sampleId, new SampleUpdateParams() .setAnnotationSets(Collections.singletonList(new AnnotationSet("annotation2", vs1.getId(), annotations))), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); } @Test public void testAnnotateIndividualUnique() throws CatalogException { String individualId = catalogManager.getIndividualManager().create(studyFqn, new Individual().setId("INDIVIDUAL_1"), - INCLUDE_RESULT, token).first().getId(); + INCLUDE_RESULT, ownerToken).first().getId(); List variables = new ArrayList<>(); variables.add(new Variable("NAME", "NAME", "", Variable.VariableType.STRING, "", true, false, Collections.emptyList(), null, 0, "", "", null, Collections.emptyMap())); VariableSet vs1 = catalogManager.getStudyManager().createVariableSet(studyFqn, "vs1", "vs1", true, false, "", null, variables, - Collections.singletonList(VariableSet.AnnotableDataModels.INDIVIDUAL), token).first(); + Collections.singletonList(VariableSet.AnnotableDataModels.INDIVIDUAL), ownerToken).first(); HashMap annotations = new HashMap<>(); annotations.put("NAME", "Luke"); catalogManager.getIndividualManager().update(studyFqn, individualId, new IndividualUpdateParams() .setAnnotationSets(Collections.singletonList(new AnnotationSet("annotation1", vs1.getId(), annotations))), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); DataResult individualDataResult = catalogManager.getIndividualManager().get(studyFqn, individualId, - new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key()), token); + new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key()), ownerToken); assertEquals(1, individualDataResult.first().getAnnotationSets().size()); annotations.put("NAME", "Lucas"); @@ -1513,106 +1504,106 @@ public void testAnnotateIndividualUnique() throws CatalogException { thrown.expectMessage("unique"); catalogManager.getIndividualManager().update(studyFqn, individualId, new IndividualUpdateParams() .setAnnotationSets(Collections.singletonList(new AnnotationSet("annotation2", vs1.getId(), annotations))), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); } @Test public void testAnnotateIncorrectType() throws CatalogException { String sampleId = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_1"), INCLUDE_RESULT, - token).first().getId(); + ownerToken).first().getId(); List variables = new ArrayList<>(); variables.add(new Variable("NUM", "NUM", "", Variable.VariableType.DOUBLE, "", true, false, null, null, 0, "", "", null, Collections.emptyMap())); VariableSet vs1 = catalogManager.getStudyManager().createVariableSet(studyFqn, "vs1", "vs1", false, false, "", null, variables, - Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), token).first(); + Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), ownerToken).first(); HashMap annotations = new HashMap<>(); annotations.put("NUM", "5"); catalogManager.getSampleManager().update(studyFqn, sampleId, new SampleUpdateParams() .setAnnotationSets(Collections.singletonList(new AnnotationSet("annotation1", vs1.getId(), annotations))), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); DataResult sampleDataResult = catalogManager.getSampleManager().get(studyFqn, sampleId, - new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key()), token); + new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key()), ownerToken); assertEquals(1, sampleDataResult.first().getAnnotationSets().size()); annotations.put("NUM", "6.8"); catalogManager.getSampleManager().update(studyFqn, sampleId, new SampleUpdateParams() .setAnnotationSets(Collections.singletonList(new AnnotationSet("annotation2", vs1.getId(), annotations))), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); sampleDataResult = catalogManager.getSampleManager().get(studyFqn, sampleId, - new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key()), token); + new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key()), ownerToken); assertEquals(2, sampleDataResult.first().getAnnotationSets().size()); annotations.put("NUM", "five polong five"); thrown.expect(CatalogException.class); catalogManager.getSampleManager().update(studyFqn, sampleId, new SampleUpdateParams() .setAnnotationSets(Collections.singletonList(new AnnotationSet("annotation3", vs1.getId(), annotations))), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); } @Test public void testAnnotateRange() throws CatalogException { String sampleId = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_1"), INCLUDE_RESULT, - token).first().getId(); + ownerToken).first().getId(); List variables = new ArrayList<>(); variables.add(new Variable("RANGE_NUM", "RANGE_NUM", "", Variable.VariableType.DOUBLE, "", true, false, Arrays.asList("1:14", "16:22", "50:"), null, 0, "", "", null, Collections.emptyMap())); VariableSet vs1 = catalogManager.getStudyManager().createVariableSet(studyFqn, "vs1", "vs1", false, false, "", null, variables, - Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), token).first(); + Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), ownerToken).first(); HashMap annotations = new HashMap<>(); annotations.put("RANGE_NUM", "1"); // 1:14 catalogManager.getSampleManager().update(studyFqn, sampleId, new SampleUpdateParams() .setAnnotationSets(Collections.singletonList(new AnnotationSet("annotation1", vs1.getId(), annotations))), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); DataResult sampleDataResult = catalogManager.getSampleManager().get(studyFqn, sampleId, - new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key()), token); + new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key()), ownerToken); assertEquals(1, sampleDataResult.first().getAnnotationSets().size()); annotations.put("RANGE_NUM", "14"); // 1:14 catalogManager.getSampleManager().update(studyFqn, sampleId, new SampleUpdateParams() .setAnnotationSets(Collections.singletonList(new AnnotationSet("annotation2", vs1.getId(), annotations))), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); sampleDataResult = catalogManager.getSampleManager().get(studyFqn, sampleId, - new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key()), token); + new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key()), ownerToken); assertEquals(2, sampleDataResult.first().getAnnotationSets().size()); annotations.put("RANGE_NUM", "20"); // 16:20 catalogManager.getSampleManager().update(studyFqn, sampleId, new SampleUpdateParams() .setAnnotationSets(Collections.singletonList(new AnnotationSet("annotation3", vs1.getId(), annotations))), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); sampleDataResult = catalogManager.getSampleManager().get(studyFqn, sampleId, - new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key()), token); + new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key()), ownerToken); assertEquals(3, sampleDataResult.first().getAnnotationSets().size()); annotations.put("RANGE_NUM", "100000"); // 50: catalogManager.getSampleManager().update(studyFqn, sampleId, new SampleUpdateParams() .setAnnotationSets(Collections.singletonList(new AnnotationSet("annotation4", vs1.getId(), annotations))), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); sampleDataResult = catalogManager.getSampleManager().get(studyFqn, sampleId, - new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key()), token); + new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key()), ownerToken); assertEquals(4, sampleDataResult.first().getAnnotationSets().size()); annotations.put("RANGE_NUM", "14.1"); thrown.expect(CatalogException.class); catalogManager.getSampleManager().update(studyFqn, sampleId, new SampleUpdateParams() .setAnnotationSets(Collections.singletonList(new AnnotationSet("annotation5", vs1.getId(), annotations))), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); } @Test public void testAnnotateCategorical() throws CatalogException { String sampleId = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_1"), INCLUDE_RESULT, - token).first().getId(); + ownerToken).first().getId(); List variables = new ArrayList<>(); variables.add(new Variable("COOL_NAME", "COOL_NAME", "", Variable.VariableType.CATEGORICAL, "", true, false, Arrays.asList("LUKE", "LEIA", "VADER", "YODA"), null, 0, "", "", null, Collections.emptyMap())); VariableSet vs1 = catalogManager.getStudyManager().createVariableSet(studyFqn, "vs1", "vs1", false, false, "", null, variables, - Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), token).first(); + Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), ownerToken).first(); Map actionMap = new HashMap<>(); actionMap.put(AnnotationSetManager.ANNOTATION_SETS, ParamUtils.BasicUpdateAction.ADD); @@ -1622,52 +1613,52 @@ public void testAnnotateCategorical() throws CatalogException { annotations.put("COOL_NAME", "LUKE"); catalogManager.getSampleManager().update(studyFqn, sampleId, new SampleUpdateParams() .setAnnotationSets(Collections.singletonList(new AnnotationSet("annotation1", vs1.getId(), annotations))), - options, token); + options, ownerToken); DataResult sampleDataResult = catalogManager.getSampleManager().get(studyFqn, sampleId, - new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key()), token); + new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key()), ownerToken); assertEquals(1, sampleDataResult.first().getAnnotationSets().size()); annotations.put("COOL_NAME", "LEIA"); catalogManager.getSampleManager().update(studyFqn, sampleId, new SampleUpdateParams() .setAnnotationSets(Collections.singletonList(new AnnotationSet("annotation2", vs1.getId(), annotations))), - options, token); + options, ownerToken); sampleDataResult = catalogManager.getSampleManager().get(studyFqn, sampleId, - new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key()), token); + new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key()), ownerToken); assertEquals(2, sampleDataResult.first().getAnnotationSets().size()); annotations.put("COOL_NAME", "VADER"); catalogManager.getSampleManager().update(studyFqn, sampleId, new SampleUpdateParams() .setAnnotationSets(Collections.singletonList(new AnnotationSet("annotation3", vs1.getId(), annotations))), - options, token); + options, ownerToken); sampleDataResult = catalogManager.getSampleManager().get(studyFqn, sampleId, - new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key()), token); + new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key()), ownerToken); assertEquals(3, sampleDataResult.first().getAnnotationSets().size()); annotations.put("COOL_NAME", "YODA"); catalogManager.getSampleManager().update(studyFqn, sampleId, new SampleUpdateParams() .setAnnotationSets(Collections.singletonList(new AnnotationSet("annotation4", vs1.getId(), annotations))), - options, token); + options, ownerToken); sampleDataResult = catalogManager.getSampleManager().get(studyFqn, sampleId, - new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key()), token); + new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key()), ownerToken); assertEquals(4, sampleDataResult.first().getAnnotationSets().size()); annotations.put("COOL_NAME", "SPOCK"); thrown.expect(CatalogException.class); catalogManager.getSampleManager().update(studyFqn, sampleId, new SampleUpdateParams() .setAnnotationSets(Collections.singletonList(new AnnotationSet("annotation5", vs1.getId(), annotations))), - options, token); + options, ownerToken); } @Test public void testAnnotateNested() throws CatalogException { String sampleId1 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_1"), - INCLUDE_RESULT, token).first().getId(); + INCLUDE_RESULT, ownerToken).first().getId(); String sampleId2 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_2"), - INCLUDE_RESULT, token).first().getId(); + INCLUDE_RESULT, ownerToken).first().getId(); VariableSet vs1 = catalogManager.getStudyManager().createVariableSet(studyFqn, "vs1", "vs1", false, false, "", null, Collections.singletonList(CatalogAnnotationsValidatorTest.nestedObject), - Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), token).first(); + Collections.singletonList(VariableSet.AnnotableDataModels.SAMPLE), ownerToken).first(); HashMap annotations = new HashMap<>(); annotations.put("nestedObject", new ObjectMap() @@ -1677,9 +1668,9 @@ public void testAnnotateNested() throws CatalogException { .append("numberList", Arrays.asList(2, 3, 4)))); catalogManager.getSampleManager().update(studyFqn, sampleId1, new SampleUpdateParams() .setAnnotationSets(Collections.singletonList(new AnnotationSet("annotation1", vs1.getId(), annotations))), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); DataResult sampleDataResult = catalogManager.getSampleManager().get(studyFqn, sampleId1, - new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key()), token); + new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key()), ownerToken); assertEquals(1, sampleDataResult.first().getAnnotationSets().size()); annotations.put("nestedObject", new ObjectMap() @@ -1689,67 +1680,67 @@ public void testAnnotateNested() throws CatalogException { .append("numberList", Arrays.asList(3, 4, 5)))); catalogManager.getSampleManager().update(studyFqn, sampleId2, new SampleUpdateParams() .setAnnotationSets(Collections.singletonList(new AnnotationSet("annotation1", vs1.getId(), annotations))), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); sampleDataResult = catalogManager.getSampleManager().get(studyFqn, sampleId2, - new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key()), token); + new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key()), ownerToken); assertEquals(1, sampleDataResult.first().getAnnotationSets().size()); List samples; Query query = new Query(SampleDBAdaptor.QueryParams.ANNOTATION.key(), vs1.getId() + ":nestedObject.stringList=li"); - samples = catalogManager.getSampleManager().search(studyFqn, query, null, token).getResults(); + samples = catalogManager.getSampleManager().search(studyFqn, query, null, ownerToken).getResults(); assertEquals(1, samples.size()); query.put(SampleDBAdaptor.QueryParams.ANNOTATION.key(), vs1.getId() + ":nestedObject.stringList=lo"); - samples = catalogManager.getSampleManager().search(studyFqn, query, null, token).getResults(); + samples = catalogManager.getSampleManager().search(studyFqn, query, null, ownerToken).getResults(); assertEquals(1, samples.size()); query.put(SampleDBAdaptor.QueryParams.ANNOTATION.key(), vs1.getId() + ":nestedObject.stringList=LL"); - samples = catalogManager.getSampleManager().search(studyFqn, query, null, token).getResults(); + samples = catalogManager.getSampleManager().search(studyFqn, query, null, ownerToken).getResults(); assertEquals(0, samples.size()); query.put(SampleDBAdaptor.QueryParams.ANNOTATION.key(), vs1.getId() + ":nestedObject.stringList=lo,li,LL"); - samples = catalogManager.getSampleManager().search(studyFqn, query, null, token).getResults(); + samples = catalogManager.getSampleManager().search(studyFqn, query, null, ownerToken).getResults(); assertEquals(2, samples.size()); query.put(SampleDBAdaptor.QueryParams.ANNOTATION.key(), vs1.getId() + ":nestedObject.object.string=my value"); - samples = catalogManager.getSampleManager().search(studyFqn, query, null, token).getResults(); + samples = catalogManager.getSampleManager().search(studyFqn, query, null, ownerToken).getResults(); assertEquals(1, samples.size()); query.put(SampleDBAdaptor.QueryParams.ANNOTATION.key(), vs1.getId() + ":nestedObject.stringList=lo,lu,LL;" + vs1.getId() + ":nestedObject.object.string=my value"); - samples = catalogManager.getSampleManager().search(studyFqn, query, null, token).getResults(); + samples = catalogManager.getSampleManager().search(studyFqn, query, null, ownerToken).getResults(); assertEquals(1, samples.size()); query.put(SampleDBAdaptor.QueryParams.ANNOTATION.key(), vs1.getId() + ":nestedObject.stringList=lo,lu,LL;" + vs1.getId() + ":nestedObject.object.numberList=7"); - samples = catalogManager.getSampleManager().search(studyFqn, query, null, token).getResults(); + samples = catalogManager.getSampleManager().search(studyFqn, query, null, ownerToken).getResults(); assertEquals(0, samples.size()); query.put(SampleDBAdaptor.QueryParams.ANNOTATION.key(), vs1.getId() + ":nestedObject.stringList=lo,lu,LL;" + vs1.getId() + ":nestedObject.object.numberList=3"); - samples = catalogManager.getSampleManager().search(studyFqn, query, null, token).getResults(); + samples = catalogManager.getSampleManager().search(studyFqn, query, null, ownerToken).getResults(); assertEquals(2, samples.size()); query.put(SampleDBAdaptor.QueryParams.ANNOTATION.key(), vs1.getId() + ":nestedObject.stringList=lo,lu,LL;" + vs1.getId() + ":nestedObject.object.numberList=5;" + vs1.getId() + ":nestedObject.object.string=stringValue"); - samples = catalogManager.getSampleManager().search(studyFqn, query, null, token).getResults(); + samples = catalogManager.getSampleManager().search(studyFqn, query, null, ownerToken).getResults(); assertEquals(1, samples.size()); query.put(SampleDBAdaptor.QueryParams.ANNOTATION.key(), vs1.getId() + ":nestedObject.stringList=lo,lu,LL;" + vs1.getId() + ":nestedObject.object.numberList=2,5"); - samples = catalogManager.getSampleManager().search(studyFqn, query, null, token).getResults(); + samples = catalogManager.getSampleManager().search(studyFqn, query, null, ownerToken).getResults(); assertEquals(2, samples.size()); query.put(SampleDBAdaptor.QueryParams.ANNOTATION.key(), vs1.getId() + ":nestedObject.stringList=lo,lu,LL;" + vs1.getId() + ":nestedObject.object.numberList=0"); - samples = catalogManager.getSampleManager().search(studyFqn, query, null, token).getResults(); + samples = catalogManager.getSampleManager().search(studyFqn, query, null, ownerToken).getResults(); assertEquals(0, samples.size()); query.put(SampleDBAdaptor.QueryParams.ANNOTATION.key(), vs1.getId() + ":unexisting=lo,lu,LL"); thrown.expect(CatalogException.class); thrown.expectMessage("does not exist"); - catalogManager.getSampleManager().search(studyFqn, query, null, token).getResults(); + catalogManager.getSampleManager().search(studyFqn, query, null, ownerToken).getResults(); } // @Test @@ -1775,11 +1766,10 @@ public void testAnnotateNested() throws CatalogException { @Test public void testGroupByAnnotations() throws Exception { - AbstractManager.MyResourceId vs1 = catalogManager.getStudyManager().getVariableSetId("vs", studyFqn, token); - + VariableSet vs = catalogManager.getStudyManager().getVariableSet(studyFqn, "vs", QueryOptions.empty(), ownerToken).first(); DataResult queryResult = catalogManager.getSampleManager().groupBy(studyFqn, new Query(), - Collections.singletonList(Constants.ANNOTATION + ":" + vs1.getResourceId() + ":annot1:PHEN"), QueryOptions.empty(), - token); + Collections.singletonList(Constants.ANNOTATION + ":" + vs.getUid() + ":annot1:PHEN"), QueryOptions.empty(), + ownerToken); assertEquals(3, queryResult.getNumResults()); for (Document document : (List) queryResult.getResults()) { @@ -1807,7 +1797,7 @@ public void testGroupByAnnotations() throws Exception { public void testIteratorSamples() throws CatalogException { Query query = new Query(); - DBIterator iterator = catalogManager.getSampleManager().iterator(studyFqn, query, null, token); + DBIterator iterator = catalogManager.getSampleManager().iterator(studyFqn, query, null, ownerToken); int count = 0; while (iterator.hasNext()) { iterator.next(); @@ -1818,41 +1808,41 @@ public void testIteratorSamples() throws CatalogException { @Test public void testQuerySamples() throws CatalogException { - VariableSet variableSet = catalogManager.getStudyManager().getVariableSet(studyFqn, "vs", QueryOptions.empty(), token).first(); + VariableSet variableSet = catalogManager.getStudyManager().getVariableSet(studyFqn, "vs", QueryOptions.empty(), ownerToken).first(); List samples; Query query = new Query(); - samples = catalogManager.getSampleManager().search(studyFqn, query, null, token).getResults(); + samples = catalogManager.getSampleManager().search(studyFqn, query, null, ownerToken).getResults(); assertEquals(9, samples.size()); query = new Query(ANNOTATION.key(), Constants.VARIABLE_SET + "=" + variableSet.getId()); - samples = catalogManager.getSampleManager().search(studyFqn, query, null, token).getResults(); + samples = catalogManager.getSampleManager().search(studyFqn, query, null, ownerToken).getResults(); assertEquals(8, samples.size()); query = new Query(ANNOTATION.key(), Constants.ANNOTATION_SET_NAME + "=annot2"); - samples = catalogManager.getSampleManager().search(studyFqn, query, null, token).getResults(); + samples = catalogManager.getSampleManager().search(studyFqn, query, null, ownerToken).getResults(); assertEquals(3, samples.size()); query = new Query(ANNOTATION.key(), Constants.ANNOTATION_SET_NAME + "=noExist"); - samples = catalogManager.getSampleManager().search(studyFqn, query, null, token).getResults(); + samples = catalogManager.getSampleManager().search(studyFqn, query, null, ownerToken).getResults(); assertEquals(0, samples.size()); query = new Query(ANNOTATION.key(), variableSet.getId() + ":NAME=s_1,s_2,s_3"); - samples = catalogManager.getSampleManager().search(studyFqn, query, null, token).getResults(); + samples = catalogManager.getSampleManager().search(studyFqn, query, null, ownerToken).getResults(); assertEquals(3, samples.size()); query = new Query(ANNOTATION.key(), variableSet.getId() + ":AGE>30;" + Constants.VARIABLE_SET + "=" + variableSet.getId()); - samples = catalogManager.getSampleManager().search(studyFqn, query, null, token).getResults(); + samples = catalogManager.getSampleManager().search(studyFqn, query, null, ownerToken).getResults(); assertEquals(3, samples.size()); query = new Query(ANNOTATION.key(), variableSet.getId() + ":AGE>30;" + Constants.VARIABLE_SET + "=" + variableSet.getId()); - samples = catalogManager.getSampleManager().search(studyFqn, query, null, token).getResults(); + samples = catalogManager.getSampleManager().search(studyFqn, query, null, ownerToken).getResults(); assertEquals(3, samples.size()); query = new Query(ANNOTATION.key(), variableSet.getId() + ":AGE>30;" + variableSet.getId() + ":ALIVE=true;" + Constants.VARIABLE_SET + "=" + variableSet.getId()); - samples = catalogManager.getSampleManager().search(studyFqn, query, null, token).getResults(); + samples = catalogManager.getSampleManager().search(studyFqn, query, null, ownerToken).getResults(); assertEquals(2, samples.size()); } @@ -1860,19 +1850,19 @@ public void testQuerySamples() throws CatalogException { public void testUpdateSampleId() throws CatalogException { thrown.expect(CatalogException.class); thrown.expectMessage("valid id"); - catalogManager.getSampleManager().update(studyFqn, s_1, new SampleUpdateParams().setId(""), QueryOptions.empty(), token); + catalogManager.getSampleManager().update(studyFqn, s_1Id, new SampleUpdateParams().setId(""), QueryOptions.empty(), ownerToken); } @Test public void testUpdateAnnotation() throws CatalogException { - Sample sample = catalogManager.getSampleManager().get(studyFqn, s_1, null, token).first(); + Sample sample = catalogManager.getSampleManager().get(studyFqn, s_1Id, null, ownerToken).first(); AnnotationSet annotationSet = sample.getAnnotationSets().get(0); Individual ind = new Individual() .setId("INDIVIDUAL_1") .setSex(SexOntologyTermAnnotation.initUnknown()); ind.setAnnotationSets(Collections.singletonList(annotationSet)); - ind = catalogManager.getIndividualManager().create(studyFqn, ind, INCLUDE_RESULT, token).first(); + ind = catalogManager.getIndividualManager().create(studyFqn, ind, INCLUDE_RESULT, ownerToken).first(); // First update annotationSet.getAnnotations().put("NAME", "SAMPLE1"); @@ -1883,9 +1873,9 @@ public void testUpdateAnnotation() throws CatalogException { // Update annotation set catalogManager.getIndividualManager().updateAnnotations(studyFqn, ind.getId(), annotationSet.getId(), annotationSet.getAnnotations(), ParamUtils.CompleteUpdateAction.SET, new QueryOptions(), - token); - catalogManager.getSampleManager().updateAnnotations(studyFqn, s_1, annotationSet.getId(), annotationSet.getAnnotations(), - ParamUtils.CompleteUpdateAction.SET, new QueryOptions(), token); + ownerToken); + catalogManager.getSampleManager().updateAnnotations(studyFqn, s_1Id, annotationSet.getId(), annotationSet.getAnnotations(), + ParamUtils.CompleteUpdateAction.SET, new QueryOptions(), ownerToken); Consumer check = as -> { Map auxAnnotations = as.getAnnotations(); @@ -1896,15 +1886,15 @@ public void testUpdateAnnotation() throws CatalogException { assertEquals("extra", auxAnnotations.get("EXTRA")); }; - sample = catalogManager.getSampleManager().get(studyFqn, s_1, null, token).first(); - ind = catalogManager.getIndividualManager().get(studyFqn, ind.getId(), null, token).first(); + sample = catalogManager.getSampleManager().get(studyFqn, s_1Id, null, ownerToken).first(); + ind = catalogManager.getIndividualManager().get(studyFqn, ind.getId(), null, ownerToken).first(); check.accept(sample.getAnnotationSets().get(0)); check.accept(ind.getAnnotationSets().get(0)); // Call again to the update to check that nothing changed catalogManager.getIndividualManager().updateAnnotations(studyFqn, ind.getId(), annotationSet.getId(), annotationSet.getAnnotations(), ParamUtils.CompleteUpdateAction.SET, new QueryOptions(), - token); + ownerToken); check.accept(ind.getAnnotationSets().get(0)); // Update mandatory annotation @@ -1913,9 +1903,9 @@ public void testUpdateAnnotation() throws CatalogException { catalogManager.getIndividualManager().updateAnnotations(studyFqn, ind.getId(), annotationSet.getId(), annotationSet.getAnnotations(), ParamUtils.CompleteUpdateAction.SET, new QueryOptions(), - token); - catalogManager.getSampleManager().updateAnnotations(studyFqn, s_1, annotationSet.getId(), annotationSet.getAnnotations(), - ParamUtils.CompleteUpdateAction.SET, new QueryOptions(), token); + ownerToken); + catalogManager.getSampleManager().updateAnnotations(studyFqn, s_1Id, annotationSet.getId(), annotationSet.getAnnotations(), + ParamUtils.CompleteUpdateAction.SET, new QueryOptions(), ownerToken); check = as -> { Map auxAnnotations = as.getAnnotations(); @@ -1925,8 +1915,8 @@ public void testUpdateAnnotation() throws CatalogException { assertEquals(false, auxAnnotations.containsKey("EXTRA")); }; - sample = catalogManager.getSampleManager().get(studyFqn, s_1, null, token).first(); - ind = catalogManager.getIndividualManager().get(studyFqn, ind.getId(), null, token).first(); + sample = catalogManager.getSampleManager().get(studyFqn, s_1Id, null, ownerToken).first(); + ind = catalogManager.getIndividualManager().get(studyFqn, ind.getId(), null, ownerToken).first(); check.accept(sample.getAnnotationSets().get(0)); check.accept(ind.getAnnotationSets().get(0)); @@ -1934,9 +1924,9 @@ public void testUpdateAnnotation() throws CatalogException { annotationSet.getAnnotations().put("EXTRA", "extra"); catalogManager.getIndividualManager().updateAnnotations(studyFqn, ind.getId(), annotationSet.getId(), annotationSet.getAnnotations(), ParamUtils.CompleteUpdateAction.SET, new QueryOptions(), - token); - catalogManager.getSampleManager().updateAnnotations(studyFqn, s_1, annotationSet.getId(), annotationSet.getAnnotations(), - ParamUtils.CompleteUpdateAction.SET, new QueryOptions(), token); + ownerToken); + catalogManager.getSampleManager().updateAnnotations(studyFqn, s_1Id, annotationSet.getId(), annotationSet.getAnnotations(), + ParamUtils.CompleteUpdateAction.SET, new QueryOptions(), ownerToken); check = as -> { Map auxAnnotations = as.getAnnotations(); @@ -1946,8 +1936,8 @@ public void testUpdateAnnotation() throws CatalogException { assertEquals("extra", auxAnnotations.get("EXTRA")); }; - sample = catalogManager.getSampleManager().get(studyFqn, s_1, null, token).first(); - ind = catalogManager.getIndividualManager().get(studyFqn, ind.getId(), null, token).first(); + sample = catalogManager.getSampleManager().get(studyFqn, s_1Id, null, ownerToken).first(); + ind = catalogManager.getIndividualManager().get(studyFqn, ind.getId(), null, ownerToken).first(); check.accept(sample.getAnnotationSets().get(0)); check.accept(ind.getAnnotationSets().get(0)); @@ -1955,9 +1945,9 @@ public void testUpdateAnnotation() throws CatalogException { Map annotationUpdate = new ObjectMap("EXTRA", "extraa"); // Action now is ADD, we only want to change that annotation catalogManager.getIndividualManager().updateAnnotations(studyFqn, ind.getId(), annotationSet.getId(), annotationUpdate, - ParamUtils.CompleteUpdateAction.ADD, new QueryOptions(), token); - catalogManager.getSampleManager().updateAnnotations(studyFqn, s_1, annotationSet.getId(), annotationUpdate, - ParamUtils.CompleteUpdateAction.ADD, new QueryOptions(), token); + ParamUtils.CompleteUpdateAction.ADD, new QueryOptions(), ownerToken); + catalogManager.getSampleManager().updateAnnotations(studyFqn, s_1Id, annotationSet.getId(), annotationUpdate, + ParamUtils.CompleteUpdateAction.ADD, new QueryOptions(), ownerToken); check = as -> { Map auxAnnotations = as.getAnnotations(); @@ -1967,20 +1957,20 @@ public void testUpdateAnnotation() throws CatalogException { assertEquals("extraa", auxAnnotations.get("EXTRA")); }; - sample = catalogManager.getSampleManager().get(studyFqn, s_1, null, token).first(); - ind = catalogManager.getIndividualManager().get(studyFqn, ind.getId(), null, token).first(); + sample = catalogManager.getSampleManager().get(studyFqn, s_1Id, null, ownerToken).first(); + ind = catalogManager.getIndividualManager().get(studyFqn, ind.getId(), null, ownerToken).first(); check.accept(sample.getAnnotationSets().get(0)); check.accept(ind.getAnnotationSets().get(0)); thrown.expect(CatalogException.class); thrown.expectMessage("not found"); catalogManager.getIndividualManager().updateAnnotations(studyFqn, ind.getId(), "blabla", annotationUpdate, - ParamUtils.CompleteUpdateAction.ADD, new QueryOptions(), token); + ParamUtils.CompleteUpdateAction.ADD, new QueryOptions(), ownerToken); } @Test public void testUpdatePhenotypes() throws CatalogException { - Sample sample = catalogManager.getSampleManager().get(studyFqn, s_1, null, token).first(); + Sample sample = catalogManager.getSampleManager().get(studyFqn, s_1Id, null, ownerToken).first(); List phenotypeList = Arrays.asList( new Phenotype("phenotype0", "phenotypeName0", "SOURCE"), @@ -1989,8 +1979,8 @@ public void testUpdatePhenotypes() throws CatalogException { ); SampleUpdateParams updateParams = new SampleUpdateParams().setPhenotypes(phenotypeList); - catalogManager.getSampleManager().update(studyFqn, sample.getId(), updateParams, QueryOptions.empty(), token); - sample = catalogManager.getSampleManager().get(studyFqn, sample.getId(), QueryOptions.empty(), token).first(); + catalogManager.getSampleManager().update(studyFqn, sample.getId(), updateParams, QueryOptions.empty(), ownerToken); + sample = catalogManager.getSampleManager().get(studyFqn, sample.getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(3, sample.getPhenotypes().size()); for (int i = 0; i < sample.getPhenotypes().size(); i++) { assertEquals("phenotype" + i, sample.getPhenotypes().get(i).getId()); @@ -2002,8 +1992,8 @@ public void testUpdatePhenotypes() throws CatalogException { QueryOptions options = new QueryOptions(Constants.ACTIONS, actionMap); updateParams = new SampleUpdateParams().setPhenotypes(Arrays.asList( new Phenotype("phenotype0", "phenotypeName0", "SOURCE"), new Phenotype("phenotype2", "phenotypeName2", "SOURCE"))); - catalogManager.getSampleManager().update(studyFqn, sample.getId(), updateParams, options, token); - sample = catalogManager.getSampleManager().get(studyFqn, sample.getId(), QueryOptions.empty(), token).first(); + catalogManager.getSampleManager().update(studyFqn, sample.getId(), updateParams, options, ownerToken); + sample = catalogManager.getSampleManager().get(studyFqn, sample.getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(1, sample.getPhenotypes().size()); assertEquals("phenotype1", sample.getPhenotypes().get(0).getId()); @@ -2012,8 +2002,8 @@ public void testUpdatePhenotypes() throws CatalogException { options = new QueryOptions(Constants.ACTIONS, actionMap); updateParams = new SampleUpdateParams().setPhenotypes(Arrays.asList( new Phenotype("phenotype1", "phenotypeName1", "SOURCE"), new Phenotype("phenotype2", "phenotypeName2", "SOURCE"))); - catalogManager.getSampleManager().update(studyFqn, sample.getId(), updateParams, options, token); - sample = catalogManager.getSampleManager().get(studyFqn, sample.getId(), QueryOptions.empty(), token).first(); + catalogManager.getSampleManager().update(studyFqn, sample.getId(), updateParams, options, ownerToken); + sample = catalogManager.getSampleManager().get(studyFqn, sample.getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(2, sample.getPhenotypes().size()); for (int i = 0; i < sample.getPhenotypes().size(); i++) { assertEquals("phenotype" + (i + 1), sample.getPhenotypes().get(i).getId()); @@ -2027,8 +2017,8 @@ public void testUpdatePhenotypes() throws CatalogException { new Phenotype("phenotype3", "phenotypeName3", "SOURCE") ); updateParams = new SampleUpdateParams().setPhenotypes(phenotypeList); - catalogManager.getSampleManager().update(studyFqn, sample.getId(), updateParams, options, token); - sample = catalogManager.getSampleManager().get(studyFqn, sample.getId(), QueryOptions.empty(), token).first(); + catalogManager.getSampleManager().update(studyFqn, sample.getId(), updateParams, options, ownerToken); + sample = catalogManager.getSampleManager().get(studyFqn, sample.getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(2, sample.getPhenotypes().size()); for (int i = 0; i < sample.getPhenotypes().size(); i++) { assertEquals("phenotype" + (i + 2), sample.getPhenotypes().get(i).getId()); @@ -2037,13 +2027,13 @@ public void testUpdatePhenotypes() throws CatalogException { @Test public void testUpdateAnnotationFail() throws CatalogException { - Sample sample = catalogManager.getSampleManager().get(studyFqn, s_1, null, token).first(); + Sample sample = catalogManager.getSampleManager().get(studyFqn, s_1Id, null, ownerToken).first(); AnnotationSet annotationSet = sample.getAnnotationSets().get(0); thrown.expect(CatalogException.class); //Can not delete required fields thrown.expectMessage("required variable"); - catalogManager.getSampleManager().removeAnnotations(studyFqn, s_1, annotationSet.getId(), Collections.singletonList("NAME"), - QueryOptions.empty(), token); + catalogManager.getSampleManager().removeAnnotations(studyFqn, s_1Id, annotationSet.getId(), Collections.singletonList("NAME"), + QueryOptions.empty(), ownerToken); } @Test @@ -2051,43 +2041,43 @@ public void testDeleteAnnotation() throws CatalogException { // We add one of the non mandatory annotations // First update - catalogManager.getSampleManager().updateAnnotations(studyFqn, s_1, "annot1", new ObjectMap("EXTRA", "extra"), - ParamUtils.CompleteUpdateAction.ADD, QueryOptions.empty(), token); + catalogManager.getSampleManager().updateAnnotations(studyFqn, s_1Id, "annot1", new ObjectMap("EXTRA", "extra"), + ParamUtils.CompleteUpdateAction.ADD, QueryOptions.empty(), ownerToken); - Sample sample = catalogManager.getSampleManager().get(studyFqn, s_1, null, token).first(); + Sample sample = catalogManager.getSampleManager().get(studyFqn, s_1Id, null, ownerToken).first(); AnnotationSet annotationSet = sample.getAnnotationSets().get(0); assertEquals("extra", annotationSet.getAnnotations().get("EXTRA")); // Now we remove that non mandatory annotation - catalogManager.getSampleManager().removeAnnotations(studyFqn, s_1, annotationSet.getId(), Collections.singletonList("EXTRA"), - QueryOptions.empty(), token); + catalogManager.getSampleManager().removeAnnotations(studyFqn, s_1Id, annotationSet.getId(), Collections.singletonList("EXTRA"), + QueryOptions.empty(), ownerToken); - sample = catalogManager.getSampleManager().get(studyFqn, s_1, null, token).first(); + sample = catalogManager.getSampleManager().get(studyFqn, s_1Id, null, ownerToken).first(); annotationSet = sample.getAnnotationSets().get(0); assertTrue(!annotationSet.getAnnotations().containsKey("EXTRA")); // Now we attempt to remove one mandatory annotation thrown.expect(CatalogException.class); //Can not delete required fields thrown.expectMessage("required variable"); - catalogManager.getSampleManager().removeAnnotations(studyFqn, s_1, annotationSet.getId(), Collections.singletonList("AGE"), - QueryOptions.empty(), token); + catalogManager.getSampleManager().removeAnnotations(studyFqn, s_1Id, annotationSet.getId(), Collections.singletonList("AGE"), + QueryOptions.empty(), ownerToken); } @Test public void testDeleteAnnotationSet() throws CatalogException { - catalogManager.getSampleManager().removeAnnotationSet(studyFqn, s_1, "annot1", QueryOptions.empty(), token); + catalogManager.getSampleManager().removeAnnotationSet(studyFqn, s_1Id, "annot1", QueryOptions.empty(), ownerToken); - DataResult sampleDataResult = catalogManager.getSampleManager().get(studyFqn, s_1, - new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key()), token); + DataResult sampleDataResult = catalogManager.getSampleManager().get(studyFqn, s_1Id, + new QueryOptions(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key()), ownerToken); assertEquals(0, sampleDataResult.first().getAnnotationSets().size()); } @Test public void getVariableSetSummary() throws CatalogException { - VariableSet variableSet = catalogManager.getStudyManager().getVariableSet(studyFqn, "vs", null, token).first(); + VariableSet variableSet = catalogManager.getStudyManager().getVariableSet(studyFqn, "vs", null, ownerToken).first(); DataResult variableSetSummary = catalogManager.getStudyManager() - .getVariableSetSummary(studyFqn, variableSet.getId(), token); + .getVariableSetSummary(studyFqn, variableSet.getId(), ownerToken); assertEquals(1, variableSetSummary.getNumResults()); VariableSetSummary summary = variableSetSummary.first(); @@ -2126,24 +2116,24 @@ public void getVariableSetSummary() throws CatalogException { @Test public void testModifySample() throws CatalogException { String sampleId1 = catalogManager.getSampleManager() - .create(studyFqn, new Sample().setId("SAMPLE_1"), INCLUDE_RESULT, token).first().getId(); + .create(studyFqn, new Sample().setId("SAMPLE_1"), INCLUDE_RESULT, ownerToken).first().getId(); String individualId = catalogManager.getIndividualManager().create(studyFqn, new Individual().setId("Individual1"), - INCLUDE_RESULT, token).first().getId(); + INCLUDE_RESULT, ownerToken).first().getId(); DataResult updateResult = catalogManager.getSampleManager() - .update(studyFqn, sampleId1, new SampleUpdateParams().setIndividualId(individualId), null, token); + .update(studyFqn, sampleId1, new SampleUpdateParams().setIndividualId(individualId), null, ownerToken); assertEquals(1, updateResult.getNumUpdated()); - Sample sample = catalogManager.getSampleManager().get(studyFqn, sampleId1, QueryOptions.empty(), token).first(); + Sample sample = catalogManager.getSampleManager().get(studyFqn, sampleId1, QueryOptions.empty(), ownerToken).first(); assertEquals(individualId, sample.getIndividualId()); sample = catalogManager.getSampleManager().get(studyFqn, sampleId1, new QueryOptions(QueryOptions.INCLUDE, - SampleDBAdaptor.QueryParams.INDIVIDUAL_ID.key()), token).first(); + SampleDBAdaptor.QueryParams.INDIVIDUAL_ID.key()), ownerToken).first(); assertNull(sample.getAttributes()); sample = catalogManager.getSampleManager().get(studyFqn, sampleId1, new QueryOptions() .append(QueryOptions.INCLUDE, SampleDBAdaptor.QueryParams.INDIVIDUAL_ID.key()) - .append(SAMPLE_INCLUDE_INDIVIDUAL_PARAM, true), token).first(); + .append(SAMPLE_INCLUDE_INDIVIDUAL_PARAM, true), ownerToken).first(); assertNotNull(sample.getAttributes()); } @@ -2152,20 +2142,20 @@ public void associateSameSampleAndIndividualInDifferentStudies() throws CatalogE String id = "myUniqueId"; for (String studyId : Arrays.asList("st1", "st2")) { System.out.println("Study " + studyId); - catalogManager.getStudyManager().create(project1, new Study().setId(studyId), QueryOptions.empty(), token); + catalogManager.getStudyManager().create(project1, new Study().setId(studyId), QueryOptions.empty(), ownerToken); - catalogManager.getSampleManager().create(studyId, new Sample().setId(id), QueryOptions.empty(), token); - catalogManager.getIndividualManager().create(studyId, new Individual().setId(id), QueryOptions.empty(), token); + catalogManager.getSampleManager().create(studyId, new Sample().setId(id), QueryOptions.empty(), ownerToken); + catalogManager.getIndividualManager().create(studyId, new Individual().setId(id), QueryOptions.empty(), ownerToken); catalogManager.getSampleManager().update(studyId, id, new SampleUpdateParams().setIndividualId(id), QueryOptions.empty(), - token); + ownerToken); - OpenCGAResult sampleResult = catalogManager.getSampleManager().get(studyId, id, QueryOptions.empty(), token); + OpenCGAResult sampleResult = catalogManager.getSampleManager().get(studyId, id, QueryOptions.empty(), ownerToken); assertEquals(1, sampleResult.getNumResults()); assertEquals(id, sampleResult.first().getIndividualId()); assertEquals(2, sampleResult.first().getVersion()); OpenCGAResult individualResult = catalogManager.getIndividualManager().get(studyId, id, QueryOptions.empty(), - token); + ownerToken); assertEquals(1, individualResult.getNumResults()); assertEquals(2, individualResult.first().getVersion()); assertEquals(1, individualResult.first().getSamples().size()); @@ -2178,33 +2168,33 @@ public void associateSameSampleAndIndividualInDifferentStudies() throws CatalogE @Test public void testGetSampleAndIndividualWithPermissionsChecked() throws CatalogException { String sampleId1 = catalogManager.getSampleManager() - .create(studyFqn, new Sample().setId("SAMPLE_1"), INCLUDE_RESULT, token).first().getId(); + .create(studyFqn, new Sample().setId("SAMPLE_1"), INCLUDE_RESULT, ownerToken).first().getId(); String individualId = catalogManager.getIndividualManager().create(studyFqn, new Individual().setId("Individual1"), - INCLUDE_RESULT, token).first().getId(); + INCLUDE_RESULT, ownerToken).first().getId(); DataResult updateResult = catalogManager.getSampleManager() - .update(studyFqn, sampleId1, new SampleUpdateParams().setIndividualId(individualId), null, token); + .update(studyFqn, sampleId1, new SampleUpdateParams().setIndividualId(individualId), null, ownerToken); assertEquals(1, updateResult.getNumUpdated()); - Sample sample = catalogManager.getSampleManager().get(studyFqn, sampleId1, QueryOptions.empty(), token).first(); + Sample sample = catalogManager.getSampleManager().get(studyFqn, sampleId1, QueryOptions.empty(), ownerToken).first(); assertEquals(individualId, sample.getIndividualId()); - catalogManager.getSampleManager().updateAcl(studyFqn, Collections.singletonList("SAMPLE_1"), "user2", - new SampleAclParams(null, null, null, null, SamplePermissions.VIEW.name()), SET, token); + catalogManager.getSampleManager().updateAcl(studyFqn, Collections.singletonList("SAMPLE_1"), normalUserId2, + new SampleAclParams(null, null, null, null, SamplePermissions.VIEW.name()), SET, ownerToken); sample = catalogManager.getSampleManager().get(studyFqn, "SAMPLE_1", new QueryOptions(SAMPLE_INCLUDE_INDIVIDUAL_PARAM, true), - sessionIdUser2).first(); + normalToken2).first(); assertEquals(null, sample.getAttributes().get("OPENCGA_INDIVIDUAL")); - catalogManager.getIndividualManager().updateAcl(studyFqn, Collections.emptyList(), "user2", - new IndividualAclParams(sampleId1, IndividualPermissions.VIEW.name()), SET, false, token); + catalogManager.getIndividualManager().updateAcl(studyFqn, Collections.emptyList(), normalUserId2, + new IndividualAclParams(sampleId1, IndividualPermissions.VIEW.name()), SET, false, ownerToken); sample = catalogManager.getSampleManager().get(studyFqn, "SAMPLE_1", new QueryOptions(SAMPLE_INCLUDE_INDIVIDUAL_PARAM, true), - sessionIdUser2).first(); + normalToken2).first(); assertEquals(individualId, ((Individual) sample.getAttributes().get("OPENCGA_INDIVIDUAL")).getId()); assertEquals(sampleId1, sample.getId()); sample = catalogManager.getSampleManager().search(studyFqn, new Query(SampleDBAdaptor.QueryParams.INDIVIDUAL_ID.key(), - "Individual1"), new QueryOptions(SAMPLE_INCLUDE_INDIVIDUAL_PARAM, true), sessionIdUser2).first(); + "Individual1"), new QueryOptions(SAMPLE_INCLUDE_INDIVIDUAL_PARAM, true), normalToken2).first(); assertEquals(individualId, ((Individual) sample.getAttributes().get("OPENCGA_INDIVIDUAL")).getId()); assertEquals(sampleId1, sample.getId()); } @@ -2212,72 +2202,72 @@ public void testGetSampleAndIndividualWithPermissionsChecked() throws CatalogExc @Test public void searchSamplesByIndividual() throws CatalogException { catalogManager.getIndividualManager().create(studyFqn, new Individual().setId("Individual1") - .setSamples(Arrays.asList(new Sample().setId("sample1"), new Sample().setId("sample2"))), new QueryOptions(), token); + .setSamples(Arrays.asList(new Sample().setId("sample1"), new Sample().setId("sample2"))), new QueryOptions(), ownerToken); DataResult sampleDataResult = catalogManager.getSampleManager().search(studyFqn, - new Query(SampleDBAdaptor.QueryParams.INDIVIDUAL_ID.key(), "Individual1"), QueryOptions.empty(), token); + new Query(SampleDBAdaptor.QueryParams.INDIVIDUAL_ID.key(), "Individual1"), QueryOptions.empty(), ownerToken); assertEquals(2, sampleDataResult.getNumResults()); sampleDataResult = catalogManager.getSampleManager().search(studyFqn, new Query().append(SampleDBAdaptor.QueryParams.INDIVIDUAL_ID.key(), "Individual1") - .append(SampleDBAdaptor.QueryParams.ID.key(), "sample1"), QueryOptions.empty(), token); + .append(SampleDBAdaptor.QueryParams.ID.key(), "sample1"), QueryOptions.empty(), ownerToken); assertEquals(1, sampleDataResult.getNumResults()); - catalogManager.getIndividualManager().create(studyFqn, new Individual().setId("Individual2"), new QueryOptions(), token); + catalogManager.getIndividualManager().create(studyFqn, new Individual().setId("Individual2"), new QueryOptions(), ownerToken); sampleDataResult = catalogManager.getSampleManager().search(studyFqn, - new Query().append(SampleDBAdaptor.QueryParams.INDIVIDUAL_ID.key(), "Individual2"), QueryOptions.empty(), token); + new Query().append(SampleDBAdaptor.QueryParams.INDIVIDUAL_ID.key(), "Individual2"), QueryOptions.empty(), ownerToken); assertEquals(0, sampleDataResult.getNumResults()); } @Test public void searchIndividualsBySample() throws CatalogException { catalogManager.getIndividualManager().create(studyFqn, new Individual().setId("Individual1") - .setSamples(Arrays.asList(new Sample().setId("sample1"), new Sample().setId("sample2"))), new QueryOptions(), token); + .setSamples(Arrays.asList(new Sample().setId("sample1"), new Sample().setId("sample2"))), new QueryOptions(), ownerToken); Individual individual = catalogManager.getIndividualManager().search(studyFqn, - new Query(IndividualDBAdaptor.QueryParams.SAMPLES.key(), "sample1"), QueryOptions.empty(), token).first(); + new Query(IndividualDBAdaptor.QueryParams.SAMPLES.key(), "sample1"), QueryOptions.empty(), ownerToken).first(); assertEquals("Individual1", individual.getId()); } @Test public void searchIndividualsBySampleWithNoSamples() throws CatalogException { catalogManager.getIndividualManager().create(studyFqn, new Individual().setId("Individual1") - .setSamples(Arrays.asList(new Sample().setId("sample1"), new Sample().setId("sample2"))), new QueryOptions(), token); + .setSamples(Arrays.asList(new Sample().setId("sample1"), new Sample().setId("sample2"))), new QueryOptions(), ownerToken); OpenCGAResult search = catalogManager.getIndividualManager().search(studyFqn, - new Query(IndividualDBAdaptor.QueryParams.SAMPLES.key(), ""), QueryOptions.empty(), token); - assertEquals(1, search.getNumResults()); + new Query(IndividualDBAdaptor.QueryParams.SAMPLES.key(), ""), QueryOptions.empty(), ownerToken); + assertEquals(3, search.getNumResults()); } @Test public void searchSamplesDifferentVersions() throws CatalogException { - catalogManager.getSampleManager().create(studyFqn, new Sample().setId("sample1"), QueryOptions.empty(), token); - catalogManager.getSampleManager().create(studyFqn, new Sample().setId("sample2"), QueryOptions.empty(), token); - catalogManager.getSampleManager().create(studyFqn, new Sample().setId("sample3"), QueryOptions.empty(), token); + catalogManager.getSampleManager().create(studyFqn, new Sample().setId("sample1"), QueryOptions.empty(), ownerToken); + catalogManager.getSampleManager().create(studyFqn, new Sample().setId("sample2"), QueryOptions.empty(), ownerToken); + catalogManager.getSampleManager().create(studyFqn, new Sample().setId("sample3"), QueryOptions.empty(), ownerToken); // Generate 4 versions of sample1 catalogManager.getSampleManager().update(studyFqn, "sample1", new SampleUpdateParams().setDescription(RandomStringUtils.random(5)), - new QueryOptions(), token); + new QueryOptions(), ownerToken); catalogManager.getSampleManager().update(studyFqn, "sample1", new SampleUpdateParams().setDescription(RandomStringUtils.random(5)), - new QueryOptions(), token); + new QueryOptions(), ownerToken); catalogManager.getSampleManager().update(studyFqn, "sample1", new SampleUpdateParams().setDescription(RandomStringUtils.random(5)), - new QueryOptions(), token); + new QueryOptions(), ownerToken); // Generate 3 versions of sample2 catalogManager.getSampleManager().update(studyFqn, "sample2", new SampleUpdateParams().setDescription(RandomStringUtils.random(5)), - new QueryOptions(), token); + new QueryOptions(), ownerToken); catalogManager.getSampleManager().update(studyFqn, "sample2", new SampleUpdateParams().setDescription(RandomStringUtils.random(5)), - new QueryOptions(), token); + new QueryOptions(), ownerToken); // Generate 1 versions of sample3 catalogManager.getSampleManager().update(studyFqn, "sample3", new SampleUpdateParams().setDescription(RandomStringUtils.random(5)), - new QueryOptions(), token); + new QueryOptions(), ownerToken); Query query = new Query() .append(SampleDBAdaptor.QueryParams.ID.key(), "sample1,sample2,sample3") .append(SampleDBAdaptor.QueryParams.VERSION.key(), "3,2,1"); - DataResult sampleDataResult = catalogManager.getSampleManager().search(studyFqn, query, QueryOptions.empty(), token); + DataResult sampleDataResult = catalogManager.getSampleManager().search(studyFqn, query, QueryOptions.empty(), ownerToken); assertEquals(3, sampleDataResult.getNumResults()); for (Sample sample : sampleDataResult.getResults()) { switch (sample.getId()) { @@ -2296,7 +2286,7 @@ public void searchSamplesDifferentVersions() throws CatalogException { } query.put(SampleDBAdaptor.QueryParams.VERSION.key(), "2"); - sampleDataResult = catalogManager.getSampleManager().search(studyFqn, query, QueryOptions.empty(), token); + sampleDataResult = catalogManager.getSampleManager().search(studyFqn, query, QueryOptions.empty(), ownerToken); assertEquals(3, sampleDataResult.getNumResults()); sampleDataResult.getResults().forEach( s -> assertEquals(2, s.getVersion()) @@ -2305,93 +2295,78 @@ public void searchSamplesDifferentVersions() throws CatalogException { query.put(SampleDBAdaptor.QueryParams.VERSION.key(), "1,2"); thrown.expect(CatalogException.class); thrown.expectMessage("size of the array"); - catalogManager.getSampleManager().search(studyFqn, query, QueryOptions.empty(), token); + catalogManager.getSampleManager().search(studyFqn, query, QueryOptions.empty(), ownerToken); } @Test public void updateIndividualFromSample() throws CatalogException { - catalogManager.getIndividualManager().create(studyFqn, new Individual().setId("Individual1"), new QueryOptions(), token); - catalogManager.getIndividualManager().create(studyFqn, new Individual().setId("Individual2"), new QueryOptions(), token); + catalogManager.getIndividualManager().create(studyFqn, new Individual().setId("Individual1"), new QueryOptions(), ownerToken); + catalogManager.getIndividualManager().create(studyFqn, new Individual().setId("Individual2"), new QueryOptions(), ownerToken); catalogManager.getSampleManager().update(studyFqn, "s_1", new SampleUpdateParams().setIndividualId("Individual1"), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); DataResult sampleDataResult = catalogManager.getSampleManager().search(studyFqn, - new Query(SampleDBAdaptor.QueryParams.INDIVIDUAL_ID.key(), "Individual1"), QueryOptions.empty(), token); + new Query(SampleDBAdaptor.QueryParams.INDIVIDUAL_ID.key(), "Individual1"), QueryOptions.empty(), ownerToken); assertEquals(1, sampleDataResult.getNumResults()); assertEquals("s_1", sampleDataResult.first().getId()); catalogManager.getSampleManager().update(studyFqn, "s_1", new SampleUpdateParams().setIndividualId("Individual2"), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); sampleDataResult = catalogManager.getSampleManager().search(studyFqn, - new Query(SampleDBAdaptor.QueryParams.INDIVIDUAL_ID.key(), "Individual1"), QueryOptions.empty(), token); + new Query(SampleDBAdaptor.QueryParams.INDIVIDUAL_ID.key(), "Individual1"), QueryOptions.empty(), ownerToken); assertEquals(0, sampleDataResult.getNumResults()); sampleDataResult = catalogManager.getSampleManager().search(studyFqn, - new Query(SampleDBAdaptor.QueryParams.INDIVIDUAL_ID.key(), "Individual2"), QueryOptions.empty(), token); + new Query(SampleDBAdaptor.QueryParams.INDIVIDUAL_ID.key(), "Individual2"), QueryOptions.empty(), ownerToken); assertEquals(1, sampleDataResult.getNumResults()); assertEquals("s_1", sampleDataResult.first().getId()); - catalogManager.getIndividualManager().delete(studyFqn, Arrays.asList("Individual1", "Individual2"), QueryOptions.empty(), token); - catalogManager.getIndividualManager().create(studyFqn, new Individual().setId("Individual1"), new QueryOptions(), token); - catalogManager.getIndividualManager().create(studyFqn, new Individual().setId("Individual2"), new QueryOptions(), token); + catalogManager.getIndividualManager().delete(studyFqn, Arrays.asList("Individual1", "Individual2"), QueryOptions.empty(), ownerToken); + catalogManager.getIndividualManager().create(studyFqn, new Individual().setId("Individual1"), new QueryOptions(), ownerToken); + catalogManager.getIndividualManager().create(studyFqn, new Individual().setId("Individual2"), new QueryOptions(), ownerToken); catalogManager.getSampleManager().update(studyFqn, "s_1", new SampleUpdateParams().setIndividualId("Individual2"), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); sampleDataResult = catalogManager.getSampleManager().search(studyFqn, - new Query(SampleDBAdaptor.QueryParams.INDIVIDUAL_ID.key(), "Individual2"), QueryOptions.empty(), token); + new Query(SampleDBAdaptor.QueryParams.INDIVIDUAL_ID.key(), "Individual2"), QueryOptions.empty(), ownerToken); assertEquals(1, sampleDataResult.getNumResults()); assertEquals("s_1", sampleDataResult.first().getId()); } - @Test - public void getSharedProject() throws CatalogException, IOException { - catalogManager.getUserManager().create("dummy", "dummy", "asd@asd.asd", TestParamConstants.PASSWORD, "", 50000L, - Account.AccountType.GUEST, opencgaToken); - catalogManager.getStudyManager().updateGroup(studyFqn, "@members", ParamUtils.BasicUpdateAction.ADD, - new GroupUpdateParams(Collections.singletonList("dummy")), token); - - String token = catalogManager.getUserManager().login("dummy", TestParamConstants.PASSWORD).getToken(); - DataResult queryResult = catalogManager.getProjectManager().getSharedProjects("dummy", QueryOptions.empty(), token); - assertEquals(1, queryResult.getNumResults()); - - catalogManager.getStudyManager().updateGroup(studyFqn, "@members", ParamUtils.BasicUpdateAction.ADD, - new GroupUpdateParams(Collections.singletonList("*")), this.token); - queryResult = catalogManager.getProjectManager().getSharedProjects("*", QueryOptions.empty(), null); - assertEquals(1, queryResult.getNumResults()); - } - @Test public void smartResolutorStudyAliasFromAnonymousUser() throws CatalogException { catalogManager.getStudyManager().updateGroup(studyFqn, "@members", ParamUtils.BasicUpdateAction.ADD, - new GroupUpdateParams(Collections.singletonList("*")), token); - Study study = catalogManager.getStudyManager().resolveId(studyFqn, "*"); + new GroupUpdateParams(Collections.singletonList("*")), ownerToken); + Study study = catalogManager.getStudyManager().resolveId(studyFqn, "*", organizationId); assertTrue(study != null); } @Test public void checkRegisteredUserPermissions() throws CatalogException { - catalogManager.getUserManager().create("dummy", "dummy", "asd@asd.asd", TestParamConstants.PASSWORD, "", 50000L, - Account.AccountType.GUEST, opencgaToken); - String token = catalogManager.getUserManager().login("dummy", TestParamConstants.PASSWORD).getToken(); + catalogManager.getUserManager().create("dummy", "dummy", "asd@asd.asd", TestParamConstants.PASSWORD, organizationId, 50000L, + opencgaToken); + String token = catalogManager.getUserManager().login(organizationId, "dummy", TestParamConstants.PASSWORD).getToken(); - catalogManager.getStudyManager().updateGroup(studyFqn, "@members", ParamUtils.BasicUpdateAction.ADD, - new GroupUpdateParams(Collections.singletonList(ParamConstants.REGISTERED_USERS)), this.token); + catalogManager.getStudyManager().updateGroup(studyFqn2, "@members", ParamUtils.BasicUpdateAction.ADD, + new GroupUpdateParams(Collections.singletonList(ParamConstants.REGISTERED_USERS)), this.ownerToken); - OpenCGAResult studyResult = catalogManager.getStudyManager().get(studyFqn, QueryOptions.empty(), token); + OpenCGAResult studyResult = catalogManager.getStudyManager().get(studyFqn2, QueryOptions.empty(), token); assertEquals(1, studyResult.getNumResults()); - assertEquals(0, catalogManager.getSampleManager().search(studyFqn, new Query(), new QueryOptions(), token).getNumResults()); + + catalogManager.getSampleManager().create(studyFqn2, new Sample().setId("s1"), QueryOptions.empty(), ownerToken); + assertEquals(0, catalogManager.getSampleManager().search(studyFqn2, new Query(), new QueryOptions(), token).getNumResults()); } @Test public void checkRegisteredUserPermissions2() throws CatalogException { - catalogManager.getUserManager().create("dummy", "dummy", "asd@asd.asd", TestParamConstants.PASSWORD, "", 50000L, - Account.AccountType.GUEST, opencgaToken); - String token = catalogManager.getUserManager().login("dummy", TestParamConstants.PASSWORD).getToken(); + catalogManager.getUserManager().create("dummy", "dummy", "asd@asd.asd", TestParamConstants.PASSWORD, organizationId, 50000L, + opencgaToken); + String token = catalogManager.getUserManager().login(organizationId, "dummy", TestParamConstants.PASSWORD).getToken(); catalogManager.getStudyManager().updateGroup(studyFqn, "@members", ParamUtils.BasicUpdateAction.ADD, - new GroupUpdateParams(Collections.singletonList(ParamConstants.REGISTERED_USERS)), this.token); + new GroupUpdateParams(Collections.singletonList(ParamConstants.REGISTERED_USERS)), this.ownerToken); catalogManager.getStudyManager().updateAcl(studyFqn, "@members", new StudyAclParams("", AuthorizationManager.ROLE_VIEW_ONLY), - ParamUtils.AclAction.ADD, this.token); + ParamUtils.AclAction.ADD, this.ownerToken); OpenCGAResult studyResult = catalogManager.getStudyManager().get(studyFqn, QueryOptions.empty(), token); assertEquals(1, studyResult.getNumResults()); @@ -2401,41 +2376,42 @@ public void checkRegisteredUserPermissions2() throws CatalogException { @Test public void checkAnonymousUserWithNoPermissions() throws CatalogException { - catalogManager.getStudyManager().updateGroup(studyFqn, "@members", ParamUtils.BasicUpdateAction.ADD, - new GroupUpdateParams(Collections.singletonList(ParamConstants.REGISTERED_USERS)), this.token); - catalogManager.getStudyManager().updateAcl(studyFqn, "@members", new StudyAclParams("", AuthorizationManager.ROLE_VIEW_ONLY), - ParamUtils.AclAction.ADD, this.token); + catalogManager.getStudyManager().updateGroup(studyFqn2, "@members", ParamUtils.BasicUpdateAction.ADD, + new GroupUpdateParams(Collections.singletonList(ParamConstants.REGISTERED_USERS)), this.ownerToken); + catalogManager.getStudyManager().updateAcl(studyFqn2, "@members", new StudyAclParams("", AuthorizationManager.ROLE_VIEW_ONLY), + ParamUtils.AclAction.ADD, this.ownerToken); + String token = catalogManager.getUserManager().loginAnonymous(organizationId).getToken(); thrown.expect(CatalogException.class); thrown.expectMessage("view any study"); - catalogManager.getStudyManager().get(studyFqn, QueryOptions.empty(), ""); + catalogManager.getStudyManager().get(studyFqn2, QueryOptions.empty(), token); } @Test public void checkAnonymousUserPermissions() throws CatalogException { - catalogManager.getStudyManager().updateGroup(studyFqn, "@members", ParamUtils.BasicUpdateAction.ADD, - new GroupUpdateParams(Collections.singletonList(ParamConstants.ANONYMOUS_USER_ID)), this.token); + String token = catalogManager.getUserManager().loginAnonymous(organizationId).getToken(); - OpenCGAResult studyResult = catalogManager.getStudyManager().get(studyFqn, QueryOptions.empty(), ""); + OpenCGAResult studyResult = catalogManager.getStudyManager().get(studyFqn, QueryOptions.empty(), token); assertEquals(1, studyResult.getNumResults()); - assertEquals(0, catalogManager.getSampleManager().search(studyFqn, new Query(), new QueryOptions(), "").getNumResults()); + assertEquals(0, catalogManager.getSampleManager().search(studyFqn, new Query(), new QueryOptions(), token).getNumResults()); } @Test public void checkAnonymousUserPermissions2() throws CatalogException { catalogManager.getStudyManager().updateGroup(studyFqn, "@members", ParamUtils.BasicUpdateAction.ADD, - new GroupUpdateParams(Collections.singletonList(ParamConstants.ANONYMOUS_USER_ID)), this.token); + new GroupUpdateParams(Collections.singletonList(ParamConstants.ANONYMOUS_USER_ID)), this.ownerToken); catalogManager.getStudyManager().updateAcl(studyFqn, "@members", new StudyAclParams("", AuthorizationManager.ROLE_VIEW_ONLY), - ParamUtils.AclAction.ADD, this.token); + ParamUtils.AclAction.ADD, this.ownerToken); - OpenCGAResult studyResult = catalogManager.getStudyManager().get(studyFqn, QueryOptions.empty(), ""); + String token = catalogManager.getUserManager().loginAnonymous(organizationId).getToken(); + OpenCGAResult studyResult = catalogManager.getStudyManager().get(studyFqn, QueryOptions.empty(), token); assertEquals(1, studyResult.getNumResults()); - assertTrue(catalogManager.getSampleManager().search(studyFqn, new Query(), new QueryOptions(), "").getNumResults() > 0); - assertTrue(catalogManager.getFileManager().search(studyFqn, new Query(), new QueryOptions(), "").getNumResults() > 0); + assertTrue(catalogManager.getSampleManager().search(studyFqn, new Query(), new QueryOptions(), token).getNumResults() > 0); + assertTrue(catalogManager.getFileManager().search(studyFqn, new Query(), new QueryOptions(), token).getNumResults() > 0); - catalogManager.getUserManager().create("dummy", "dummy", "asd@asd.asd", TestParamConstants.PASSWORD, "", 50000L, - Account.AccountType.GUEST, opencgaToken); - String token = catalogManager.getUserManager().login("dummy", TestParamConstants.PASSWORD).getToken(); + catalogManager.getUserManager().create("dummy", "dummy", "asd@asd.asd", TestParamConstants.PASSWORD, organizationId, 50000L, + opencgaToken); + token = catalogManager.getUserManager().login(organizationId, "dummy", TestParamConstants.PASSWORD).getToken(); studyResult = catalogManager.getStudyManager().get(studyFqn, QueryOptions.empty(), token); assertEquals(1, studyResult.getNumResults()); assertTrue(catalogManager.getSampleManager().search(studyFqn, new Query(), new QueryOptions(), token).getNumResults() > 0); @@ -2445,23 +2421,23 @@ public void checkAnonymousUserPermissions2() throws CatalogException { @Test public void testCreateSampleWithIndividual() throws CatalogException { String individualId = catalogManager.getIndividualManager().create(studyFqn, new Individual().setId("Individual1"), - INCLUDE_RESULT, token).first().getId(); + INCLUDE_RESULT, ownerToken).first().getId(); String sampleId1 = catalogManager.getSampleManager().create(studyFqn, new Sample() .setId("SAMPLE_1") .setIndividualId(individualId), - INCLUDE_RESULT, token).first().getId(); + INCLUDE_RESULT, ownerToken).first().getId(); DataResult individualDataResult = catalogManager.getIndividualManager().get(studyFqn, individualId, - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); assertEquals(sampleId1, individualDataResult.first().getSamples().get(0).getId()); // Create sample linking to individual based on the individual name String sampleId2 = catalogManager.getSampleManager().create(studyFqn, new Sample() .setId("SAMPLE_2") .setIndividualId("Individual1"), - INCLUDE_RESULT, token).first().getId(); + INCLUDE_RESULT, ownerToken).first().getId(); - individualDataResult = catalogManager.getIndividualManager().get(studyFqn, individualId, QueryOptions.empty(), token); + individualDataResult = catalogManager.getIndividualManager().get(studyFqn, individualId, QueryOptions.empty(), ownerToken); assertEquals(2, individualDataResult.first().getSamples().size()); assertTrue(individualDataResult.first().getSamples().stream().map(Sample::getId).collect(Collectors.toSet()).containsAll( Arrays.asList(sampleId1, sampleId2) @@ -2471,38 +2447,38 @@ public void testCreateSampleWithIndividual() throws CatalogException { @Test public void testModifySampleBadIndividual() throws CatalogException { String sampleId1 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_1"), INCLUDE_RESULT, - token).first().getId(); + ownerToken).first().getId(); thrown.expect(CatalogException.class); thrown.expectMessage("not found"); - catalogManager.getSampleManager().update(studyFqn, sampleId1, new SampleUpdateParams().setIndividualId("ind"), null, token); + catalogManager.getSampleManager().update(studyFqn, sampleId1, new SampleUpdateParams().setIndividualId("ind"), null, ownerToken); } @Test public void testDeleteSample() throws CatalogException { long sampleUid = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_1"), INCLUDE_RESULT, - token).first().getUid(); + ownerToken).first().getUid(); Query query = new Query(SampleDBAdaptor.QueryParams.ID.key(), "SAMPLE_1"); - DataResult sampleDataResult = catalogManager.getSampleManager().search("1000G:phase1", query, new QueryOptions(), token); + DataResult sampleDataResult = catalogManager.getSampleManager().search("1000G:phase1", query, new QueryOptions(), ownerToken); assertEquals(1, sampleDataResult.getNumResults()); assertEquals("SAMPLE_1", sampleDataResult.first().getId()); query = new Query() .append(SampleDBAdaptor.QueryParams.ID.key(), "SAMPLE_1") .append(SampleDBAdaptor.QueryParams.DELETED.key(), false); - sampleDataResult = catalogManager.getSampleManager().search("1000G:phase1", query, new QueryOptions(), token); + sampleDataResult = catalogManager.getSampleManager().search("1000G:phase1", query, new QueryOptions(), ownerToken); assertEquals(1, sampleDataResult.getNumResults()); assertEquals("SAMPLE_1", sampleDataResult.first().getId()); - DataResult delete = catalogManager.getSampleManager().delete("1000G:phase1", query, null, token); + DataResult delete = catalogManager.getSampleManager().delete("1000G:phase1", query, null, ownerToken); assertEquals(1, delete.getNumDeleted()); query = new Query() .append(SampleDBAdaptor.QueryParams.UID.key(), sampleUid) .append(SampleDBAdaptor.QueryParams.DELETED.key(), true); - sampleDataResult = catalogManager.getSampleManager().search("1000G:phase1", query, new QueryOptions(), token); + sampleDataResult = catalogManager.getSampleManager().search("1000G:phase1", query, new QueryOptions(), ownerToken); assertEquals(1, sampleDataResult.getNumResults()); assertEquals(InternalStatus.DELETED, sampleDataResult.first().getInternal().getStatus().getId()); } @@ -2510,57 +2486,57 @@ public void testDeleteSample() throws CatalogException { @Test public void testAssignPermissions() throws CatalogException { Sample sample = new Sample().setId("sample"); - catalogManager.getSampleManager().create(studyFqn, sample, QueryOptions.empty(), token); + catalogManager.getSampleManager().create(studyFqn, sample, QueryOptions.empty(), ownerToken); OpenCGAResult> dataResult = catalogManager.getSampleManager().updateAcl(studyFqn, - Collections.singletonList("sample"), "user2", new SampleAclParams(null, null, null, null, "VIEW"), SET, token); + Collections.singletonList("sample"), normalUserId2, new SampleAclParams(null, null, null, null, "VIEW"), SET, ownerToken); assertEquals(1, dataResult.getNumResults()); assertEquals(1, dataResult.first().getAcl().size()); - assertEquals("user2", dataResult.first().getAcl().get(0).getMember()); + assertEquals(normalUserId2, dataResult.first().getAcl().get(0).getMember()); assertTrue(dataResult.first().getAcl().get(0).getPermissions().contains(SamplePermissions.VIEW)); } @Test public void testAssignPermissionsByFamily() throws CatalogException { Sample sample = new Sample().setId("sample1"); - catalogManager.getSampleManager().create(studyFqn, sample, QueryOptions.empty(), token); + catalogManager.getSampleManager().create(studyFqn2, sample, QueryOptions.empty(), ownerToken); Sample sample2 = new Sample().setId("sample2"); - catalogManager.getSampleManager().create(studyFqn, sample2, QueryOptions.empty(), token); + catalogManager.getSampleManager().create(studyFqn2, sample2, QueryOptions.empty(), ownerToken); Individual individual = new Individual().setId("individual"); - catalogManager.getIndividualManager().create(studyFqn, individual, Collections.singletonList(sample.getId()), - QueryOptions.empty(), token); + catalogManager.getIndividualManager().create(studyFqn2, individual, Collections.singletonList(sample.getId()), + QueryOptions.empty(), ownerToken); Individual individual2 = new Individual().setId("individual2"); - catalogManager.getIndividualManager().create(studyFqn, individual2, Collections.singletonList(sample2.getId()), - QueryOptions.empty(), token); + catalogManager.getIndividualManager().create(studyFqn2, individual2, Collections.singletonList(sample2.getId()), + QueryOptions.empty(), ownerToken); Family family = new Family().setId("family1"); - catalogManager.getFamilyManager().create(studyFqn, family, Collections.singletonList(individual.getId()), QueryOptions.empty(), - token); + catalogManager.getFamilyManager().create(studyFqn2, family, Collections.singletonList(individual.getId()), QueryOptions.empty(), + ownerToken); Family family2 = new Family().setId("family2"); - catalogManager.getFamilyManager().create(studyFqn, family2, Collections.singletonList(individual2.getId()), QueryOptions.empty(), - token); + catalogManager.getFamilyManager().create(studyFqn2, family2, Collections.singletonList(individual2.getId()), QueryOptions.empty(), + ownerToken); - OpenCGAResult> permissions = catalogManager.getSampleManager().getAcls(studyFqn, - Arrays.asList(sample.getId(), sample2.getId()), "user2", false, token); + OpenCGAResult> permissions = catalogManager.getSampleManager().getAcls(studyFqn2, + Arrays.asList(sample.getId(), sample2.getId()), normalUserId2, false, ownerToken); assertEquals(2, permissions.getNumResults()); for (AclEntryList result : permissions.getResults()) { assertEquals(1, result.getAcl().size()); - assertEquals("user2", result.getAcl().get(0).getMember()); + assertEquals(normalUserId2, result.getAcl().get(0).getMember()); assertNull(result.getAcl().get(0).getPermissions()); } // Assign permissions to both families - catalogManager.getSampleManager().updateAcl(studyFqn, Collections.emptyList(), "user2", + catalogManager.getSampleManager().updateAcl(studyFqn2, Collections.emptyList(), normalUserId2, new SampleAclParams() .setFamily(family.getId() + "," + family2.getId()) - .setPermissions(SamplePermissions.VIEW.name()), ADD, token); + .setPermissions(SamplePermissions.VIEW.name()), ADD, ownerToken); - permissions = catalogManager.getSampleManager().getAcls(studyFqn, - Arrays.asList(sample.getId(), sample2.getId()), "user2", false, token); + permissions = catalogManager.getSampleManager().getAcls(studyFqn2, + Arrays.asList(sample.getId(), sample2.getId()), normalUserId2, false, ownerToken); assertEquals(2, permissions.getNumResults()); for (AclEntryList result : permissions.getResults()) { assertEquals(1, result.getAcl().size()); @@ -2577,8 +2553,8 @@ public void incrementVersionTest() throws CatalogException { Sample dummySample1 = DummyModelUtils.getDummySample(); Sample dummySample2 = DummyModelUtils.getDummySample(); - dummySample1 = catalogManager.getSampleManager().create(studyFqn, dummySample1, options, token).first(); - dummySample2 = catalogManager.getSampleManager().create(studyFqn, dummySample2, options, token).first(); + dummySample1 = catalogManager.getSampleManager().create(studyFqn, dummySample1, options, ownerToken).first(); + dummySample2 = catalogManager.getSampleManager().create(studyFqn, dummySample2, options, ownerToken).first(); String oldDescription = dummySample1.getDescription(); @@ -2586,18 +2562,18 @@ public void incrementVersionTest() throws CatalogException { assertEquals(1, dummySample2.getVersion()); dummySample1 = catalogManager.getSampleManager().update(studyFqn, dummySample1.getId(), - new SampleUpdateParams().setDescription("name"), options, token).first(); + new SampleUpdateParams().setDescription("name"), options, ownerToken).first(); assertEquals(2, dummySample1.getVersion()); assertEquals("name", dummySample1.getDescription()); - dummySample2 = catalogManager.getSampleManager().get(studyFqn, dummySample2.getId(), QueryOptions.empty(), token).first(); + dummySample2 = catalogManager.getSampleManager().get(studyFqn, dummySample2.getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(1, dummySample2.getVersion()); Query query = new Query() .append(SampleDBAdaptor.QueryParams.ID.key(), dummySample1.getId()) .append(Constants.ALL_VERSIONS, true); - OpenCGAResult result = catalogManager.getSampleManager().search(studyFqn, query, QueryOptions.empty(), token); + OpenCGAResult result = catalogManager.getSampleManager().search(studyFqn, query, QueryOptions.empty(), ownerToken); assertEquals(2, result.getNumResults()); assertEquals(oldDescription, result.getResults().get(0).getDescription()); assertEquals(1, result.getResults().get(0).getVersion()); @@ -2626,28 +2602,28 @@ public void memberReferenceTest() throws CatalogException { family.setMembers(null); QueryOptions options = new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true); - individual1 = catalogManager.getIndividualManager().create(studyFqn, individual1, options, token).first(); + individual1 = catalogManager.getIndividualManager().create(studyFqn, individual1, options, ownerToken).first(); assertEquals(1, individual1.getVersion()); assertEquals(2, individual1.getSamples().size()); assertEquals(2, individual1.getSamples().stream().map(Sample::getVersion).filter(v -> v == 1).count()); assertEquals(2, individual1.getSamples().stream().map(Sample::getIndividualId).filter(i -> i.equals(individualId1)).count()); - individual2 = catalogManager.getIndividualManager().create(studyFqn, individual2, options, token).first(); + individual2 = catalogManager.getIndividualManager().create(studyFqn, individual2, options, ownerToken).first(); assertEquals(1, individual2.getVersion()); assertEquals(2, individual2.getSamples().size()); assertEquals(2, individual2.getSamples().stream().map(Sample::getVersion).filter(v -> v == 1).count()); assertEquals(2, individual2.getSamples().stream().map(Sample::getIndividualId).filter(i -> i.equals(individualId2)).count()); family = catalogManager.getFamilyManager().create(studyFqn, family, Arrays.asList(individual1.getId(), individual2.getId()), - options, token).first(); + options, ownerToken).first(); assertEquals(2, family.getMembers().size()); assertEquals(1, family.getVersion()); assertEquals(2, family.getMembers().stream().map(Individual::getVersion).filter(v -> v == 2).count()); // Update sample1 sample1 = catalogManager.getSampleManager().update(studyFqn, sample1.getId(), new SampleUpdateParams().setDescription("blabla"), - options, token).first(); - individual1 = catalogManager.getIndividualManager().get(studyFqn, individualId1, QueryOptions.empty(), token).first(); + options, ownerToken).first(); + individual1 = catalogManager.getIndividualManager().get(studyFqn, individualId1, QueryOptions.empty(), ownerToken).first(); assertEquals(2, sample1.getVersion()); assertEquals(2, individual1.getSamples().size()); @@ -2659,7 +2635,7 @@ public void memberReferenceTest() throws CatalogException { assertEquals(1, individual1.getFamilyIds().size()); assertEquals(family.getId(), individual1.getFamilyIds().get(0)); - family = catalogManager.getFamilyManager().get(studyFqn, family.getId(), QueryOptions.empty(), token).first(); + family = catalogManager.getFamilyManager().get(studyFqn, family.getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(2, family.getMembers().size()); for (Individual member : family.getMembers()) { if (member.getId().equals(individualId1)) { @@ -2679,12 +2655,12 @@ public void updateInUseInCATest() throws CatalogException { Family family = DummyModelUtils.getDummyCaseFamily("family1"); for (int i = family.getMembers().size() - 1; i >= 0; i--) { - catalogManager.getIndividualManager().create(studyFqn, family.getMembers().get(i), QueryOptions.empty(), token); + catalogManager.getIndividualManager().create(studyFqn, family.getMembers().get(i), QueryOptions.empty(), ownerToken); } List members = family.getMembers().stream().map(Individual::getId).collect(Collectors.toList()); family.setMembers(null); - family = catalogManager.getFamilyManager().create(studyFqn, family, members, options, token).first(); + family = catalogManager.getFamilyManager().create(studyFqn, family, members, options, ownerToken).first(); // Unlocked cases ClinicalAnalysis case1 = DummyModelUtils.getDummyClinicalAnalysis(family.getMembers().get(0), family, null); @@ -2694,11 +2670,11 @@ public void updateInUseInCATest() throws CatalogException { ClinicalAnalysis case3 = DummyModelUtils.getDummyClinicalAnalysis(family.getMembers().get(0), family, null) .setLocked(true); - case1 = catalogManager.getClinicalAnalysisManager().create(studyFqn, case1, options, token).first(); + case1 = catalogManager.getClinicalAnalysisManager().create(studyFqn, case1, options, ownerToken).first(); assertFalse(case1.isLocked()); - case2 = catalogManager.getClinicalAnalysisManager().create(studyFqn, case2, options, token).first(); + case2 = catalogManager.getClinicalAnalysisManager().create(studyFqn, case2, options, ownerToken).first(); assertFalse(case2.isLocked()); - case3 = catalogManager.getClinicalAnalysisManager().create(studyFqn, case3, options, token).first(); + case3 = catalogManager.getClinicalAnalysisManager().create(studyFqn, case3, options, ownerToken).first(); assertTrue(case3.isLocked()); String probandId = case3.getProband().getId(); @@ -2706,21 +2682,21 @@ public void updateInUseInCATest() throws CatalogException { // Update proband sample information Sample sample = catalogManager.getSampleManager().update(studyFqn, sampleProbandId, - new SampleUpdateParams().setDescription("blabla"), options, token).first(); + new SampleUpdateParams().setDescription("blabla"), options, ownerToken).first(); assertEquals(2, sample.getVersion()); - Individual proband = catalogManager.getIndividualManager().get(studyFqn, probandId, QueryOptions.empty(), token).first(); + Individual proband = catalogManager.getIndividualManager().get(studyFqn, probandId, QueryOptions.empty(), ownerToken).first(); assertEquals(3, proband.getVersion()); assertEquals(2, proband.getSamples().get(0).getVersion()); assertEquals("blabla", proband.getSamples().get(0).getDescription()); - family = catalogManager.getFamilyManager().get(studyFqn, family.getId(), QueryOptions.empty(), token).first(); + family = catalogManager.getFamilyManager().get(studyFqn, family.getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(2, family.getVersion()); assertEquals(1, family.getMembers().stream().filter(i -> i.getId().equals(probandId)).count()); assertEquals(1, family.getMembers().stream().filter(i -> i.getId().equals(probandId)).filter(i -> i.getVersion() == 3).count()); OpenCGAResult result = catalogManager.getClinicalAnalysisManager().get(studyFqn, - Arrays.asList(case1.getId(), case2.getId(), case3.getId()), QueryOptions.empty(), token); + Arrays.asList(case1.getId(), case2.getId(), case3.getId()), QueryOptions.empty(), ownerToken); case1 = result.getResults().get(0); case2 = result.getResults().get(1); case3 = result.getResults().get(2); @@ -2753,12 +2729,12 @@ public void deleteInUseInCATest() throws CatalogException { Family family = DummyModelUtils.getDummyCaseFamily("family1"); for (int i = family.getMembers().size() - 1; i >= 0; i--) { - catalogManager.getIndividualManager().create(studyFqn, family.getMembers().get(i), QueryOptions.empty(), token); + catalogManager.getIndividualManager().create(studyFqn, family.getMembers().get(i), QueryOptions.empty(), ownerToken); } List members = family.getMembers().stream().map(Individual::getId).collect(Collectors.toList()); family.setMembers(null); - family = catalogManager.getFamilyManager().create(studyFqn, family, members, options, token).first(); + family = catalogManager.getFamilyManager().create(studyFqn, family, members, options, ownerToken).first(); // Unlocked cases ClinicalAnalysis case1 = DummyModelUtils.getDummyClinicalAnalysis(family.getMembers().get(0), family, null); @@ -2768,17 +2744,17 @@ public void deleteInUseInCATest() throws CatalogException { ClinicalAnalysis case3 = DummyModelUtils.getDummyClinicalAnalysis(family.getMembers().get(0), family, null) .setLocked(true); - case1 = catalogManager.getClinicalAnalysisManager().create(studyFqn, case1, options, token).first(); + case1 = catalogManager.getClinicalAnalysisManager().create(studyFqn, case1, options, ownerToken).first(); assertFalse(case1.isLocked()); - case2 = catalogManager.getClinicalAnalysisManager().create(studyFqn, case2, options, token).first(); + case2 = catalogManager.getClinicalAnalysisManager().create(studyFqn, case2, options, ownerToken).first(); assertFalse(case2.isLocked()); - case3 = catalogManager.getClinicalAnalysisManager().create(studyFqn, case3, options, token).first(); + case3 = catalogManager.getClinicalAnalysisManager().create(studyFqn, case3, options, ownerToken).first(); assertTrue(case3.isLocked()); // Delete sample from proband try { catalogManager.getSampleManager().delete(studyFqn, Collections.singletonList(case1.getProband().getSamples().get(0).getId()), - new QueryOptions(ParamConstants.FORCE, true), token); + new QueryOptions(ParamConstants.FORCE, true), ownerToken); fail("Expected CatalogException with message containing 'in use in 3 cases'"); } catch (CatalogException e) { MatcherAssert.assertThat(e, ThrowableMessageMatcher.hasMessage(CoreMatchers.containsString("in use in 3 cases"))); @@ -2786,14 +2762,14 @@ public void deleteInUseInCATest() throws CatalogException { // unlock case3 catalogManager.getClinicalAnalysisManager().update(studyFqn, case3.getId(), new ClinicalAnalysisUpdateParams().setLocked(false), - QueryOptions.empty(), token); + QueryOptions.empty(), ownerToken); try { catalogManager.getSampleManager().delete(studyFqn, Collections.singletonList(case1.getProband().getSamples().get(0).getId()), - new QueryOptions(ParamConstants.FORCE, false), token); + new QueryOptions(ParamConstants.FORCE, true), ownerToken); fail("Expected CatalogException with message containing 'associated with individual'."); } catch (CatalogException e) { - MatcherAssert.assertThat(e, ThrowableMessageMatcher.hasMessage(CoreMatchers.containsString("associated with individual"))); + MatcherAssert.assertThat(e, ThrowableMessageMatcher.hasMessage(CoreMatchers.containsString("in 3 cases"))); } } diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/StudyManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/StudyManagerTest.java index 203dbf336ac..694f2af3f3f 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/StudyManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/StudyManagerTest.java @@ -16,7 +16,7 @@ package org.opencb.opencga.catalog.managers; -import org.apache.solr.common.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.junit.Test; import org.junit.experimental.categories.Category; import org.opencb.commons.datastore.core.ObjectMap; @@ -32,7 +32,6 @@ import org.opencb.opencga.core.models.study.*; import org.opencb.opencga.core.models.study.configuration.ClinicalAnalysisStudyConfiguration; import org.opencb.opencga.core.models.study.configuration.ClinicalPriorityValue; -import org.opencb.opencga.core.models.user.Account; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.MediumTests; import org.reflections.Reflections; @@ -52,9 +51,9 @@ public class StudyManagerTest extends AbstractManagerTest { @Test public void testDefaultVariableSets() throws Exception { String fqn = catalogManager.getStudyManager().create(project1, "newStudy", "newStudy", "newStudy", null, null, - null, null, null, new QueryOptions(), token).first().getFqn(); + null, null, null, new QueryOptions(), ownerToken).first().getFqn(); - Study study = catalogManager.getStudyManager().get(fqn, null, token).first(); + Study study = catalogManager.getStudyManager().get(fqn, null, ownerToken).first(); Set s = new Reflections(new ResourcesScanner(), "variablesets/").getResources(Pattern.compile(".*\\.json")); @@ -104,37 +103,37 @@ public void cleanVariable(Variable variable) { @Test public void testCreateDuplicatedVariableSets() throws Exception { - Study study = catalogManager.getStudyManager().get(studyFqn, null, token).first(); + Study study = catalogManager.getStudyManager().get(studyFqn, null, ownerToken).first(); // Create a new variable set changing the id study.getVariableSets().get(0).setId("newId"); - catalogManager.getStudyManager().createVariableSet(studyFqn, study.getVariableSets().get(0), token); - Study study2 = catalogManager.getStudyManager().get(studyFqn, null, token).first(); + catalogManager.getStudyManager().createVariableSet(studyFqn, study.getVariableSets().get(0), ownerToken); + Study study2 = catalogManager.getStudyManager().get(studyFqn, null, ownerToken).first(); assertEquals(study.getVariableSets().size() + 1, study2.getVariableSets().size()); // Replicate the first of the variable sets for creation thrown.expect(CatalogException.class); thrown.expectMessage("already exists"); - catalogManager.getStudyManager().createVariableSet(studyFqn, study.getVariableSets().get(0), token); + catalogManager.getStudyManager().createVariableSet(studyFqn, study.getVariableSets().get(0), ownerToken); } @Test public void internalVariableSetTest() throws CatalogException { Study study = catalogManager.getStudyManager().create(project1, "newStudy", "newStudy", "newStudy", null, null, - null, null, null, new QueryOptions(), token).first(); + null, null, null, new QueryOptions(), ownerToken).first(); Set variables = new HashSet<>(); variables.add(new Variable().setId("a").setType(Variable.VariableType.STRING)); variables.add(new Variable().setId("b").setType(Variable.VariableType.MAP_INTEGER).setAllowedKeys(Arrays.asList("b1", "b2"))); VariableSet variableSet = new VariableSet("myInternalVset", "", false, false, true, "", variables, null, 1, null); - OpenCGAResult result = catalogManager.getStudyManager().createVariableSet(study.getId(), variableSet, token); + OpenCGAResult result = catalogManager.getStudyManager().createVariableSet(study.getId(), variableSet, ownerToken); assertEquals(1, result.getNumUpdated()); assertEquals(1, result.getNumResults()); assertEquals(1, result.getResults().size()); // An internal variable set should never be returned - study = catalogManager.getStudyManager().get("newStudy", QueryOptions.empty(), token).first(); + study = catalogManager.getStudyManager().get("newStudy", QueryOptions.empty(), ownerToken).first(); for (VariableSet vset : study.getVariableSets()) { assertNotEquals(variableSet.getId(), vset.getId()); assertFalse(vset.isInternal()); @@ -143,36 +142,36 @@ public void internalVariableSetTest() throws CatalogException { // But if I try to create another one with the same id, it should fail thrown.expect(CatalogException.class); thrown.expectMessage("exists"); - catalogManager.getStudyManager().createVariableSet(study.getId(), variableSet, token); + catalogManager.getStudyManager().createVariableSet(study.getId(), variableSet, ownerToken); } @Test public void updateInternalRecessiveGene() throws CatalogException { Study study = catalogManager.getStudyManager().create(project1, "newStudy", "newStudy", "newStudy", null, null, - null, null, null, new QueryOptions(), token).first(); + null, null, null, new QueryOptions(), ownerToken).first(); assertEquals(RecessiveGeneSummaryIndex.Status.NOT_INDEXED, study.getInternal().getIndex().getRecessiveGene().getStatus()); String date = TimeUtils.getTime(); catalogManager.getStudyManager().updateSummaryIndex("newStudy", - new RecessiveGeneSummaryIndex(RecessiveGeneSummaryIndex.Status.INDEXED, date), token); - study = catalogManager.getStudyManager().get("newStudy", QueryOptions.empty(), token).first(); + new RecessiveGeneSummaryIndex(RecessiveGeneSummaryIndex.Status.INDEXED, date), ownerToken); + study = catalogManager.getStudyManager().get("newStudy", QueryOptions.empty(), ownerToken).first(); assertEquals(RecessiveGeneSummaryIndex.Status.INDEXED, study.getInternal().getIndex().getRecessiveGene().getStatus()); assertEquals(date, study.getInternal().getIndex().getRecessiveGene().getModificationDate()); catalogManager.getStudyManager().updateGroup("newStudy", "members", ParamUtils.BasicUpdateAction.ADD, - new GroupUpdateParams(Collections.singletonList("user2")), token); + new GroupUpdateParams(Collections.singletonList(normalUserId1)), ownerToken); thrown.expect(CatalogAuthorizationException.class); thrown.expectMessage("admin"); catalogManager.getStudyManager().updateSummaryIndex("newStudy", - new RecessiveGeneSummaryIndex(RecessiveGeneSummaryIndex.Status.INDEXED, date), sessionIdUser2); + new RecessiveGeneSummaryIndex(RecessiveGeneSummaryIndex.Status.INDEXED, date), normalToken1); } @Test public void updateClinicalConfiguration() throws CatalogException { Study study = catalogManager.getStudyManager().create(project1, "newStudy", "newStudy", "newStudy", null, null, - null, null, null, new QueryOptions(), token).first(); + null, null, null, new QueryOptions(), ownerToken).first(); assertNotNull(study.getInternal().getConfiguration()); assertNotNull(study.getInternal().getConfiguration().getClinical()); assertFalse(study.getInternal().getConfiguration().getClinical().getPriorities().isEmpty()); @@ -180,9 +179,9 @@ public void updateClinicalConfiguration() throws CatalogException { assertFalse(study.getInternal().getConfiguration().getClinical().getStatus().isEmpty()); study.getInternal().getConfiguration().getClinical().setPriorities(Collections.singletonList(new ClinicalPriorityValue("bla", "bla", 1, true))); - catalogManager.getClinicalAnalysisManager().configureStudy("newStudy", study.getInternal().getConfiguration().getClinical(), token); + catalogManager.getClinicalAnalysisManager().configureStudy("newStudy", study.getInternal().getConfiguration().getClinical(), ownerToken); - study = catalogManager.getStudyManager().get("newStudy", QueryOptions.empty(), token).first(); + study = catalogManager.getStudyManager().get("newStudy", QueryOptions.empty(), ownerToken).first(); assertNotNull(study.getInternal().getConfiguration()); assertNotNull(study.getInternal().getConfiguration().getClinical()); assertFalse(study.getInternal().getConfiguration().getClinical().getPriorities().isEmpty()); @@ -191,9 +190,9 @@ public void updateClinicalConfiguration() throws CatalogException { assertFalse(study.getInternal().getConfiguration().getClinical().getFlags().isEmpty()); assertFalse(study.getInternal().getConfiguration().getClinical().getStatus().isEmpty()); - catalogManager.getClinicalAnalysisManager().configureStudy("newStudy", ClinicalAnalysisStudyConfiguration.defaultConfiguration(), token); + catalogManager.getClinicalAnalysisManager().configureStudy("newStudy", ClinicalAnalysisStudyConfiguration.defaultConfiguration(), ownerToken); - study = catalogManager.getStudyManager().get("newStudy", QueryOptions.empty(), token).first(); + study = catalogManager.getStudyManager().get("newStudy", QueryOptions.empty(), ownerToken).first(); assertNotNull(study.getInternal().getConfiguration()); assertNotNull(study.getInternal().getConfiguration().getClinical()); assertFalse(study.getInternal().getConfiguration().getClinical().getPriorities().isEmpty()); @@ -205,26 +204,26 @@ public void updateClinicalConfiguration() throws CatalogException { @Test public void testSetVariantEngineConfiguration() throws CatalogException { - Study study = catalogManager.getStudyManager().get(studyFqn, null, token).first(); + Study study = catalogManager.getStudyManager().get(studyFqn, null, ownerToken).first(); System.out.println("getVariantEngineConfiguration() = " + study.getInternal().getConfiguration().getVariantEngine()); - catalogManager.getStudyManager().setVariantEngineConfigurationOptions(studyFqn, new ObjectMap("k1", "v1"), token); - study = catalogManager.getStudyManager().get(studyFqn, null, token).first(); + catalogManager.getStudyManager().setVariantEngineConfigurationOptions(studyFqn, new ObjectMap("k1", "v1"), ownerToken); + study = catalogManager.getStudyManager().get(studyFqn, null, ownerToken).first(); System.out.println("getVariantEngineConfiguration() = " + study.getInternal().getConfiguration().getVariantEngine()); assertEquals(new ObjectMap("k1", "v1"), study.getInternal().getConfiguration().getVariantEngine().getOptions()); - catalogManager.getStudyManager().setVariantEngineConfigurationOptions(studyFqn, new ObjectMap("k2", "v2"), token); - study = catalogManager.getStudyManager().get(studyFqn, null, token).first(); + catalogManager.getStudyManager().setVariantEngineConfigurationOptions(studyFqn, new ObjectMap("k2", "v2"), ownerToken); + study = catalogManager.getStudyManager().get(studyFqn, null, ownerToken).first(); System.out.println("getVariantEngineConfiguration() = " + study.getInternal().getConfiguration().getVariantEngine()); assertEquals(new ObjectMap("k2", "v2"), study.getInternal().getConfiguration().getVariantEngine().getOptions()); SampleIndexConfiguration sampleIndexConfiguration = SampleIndexConfiguration.defaultConfiguration(); catalogManager.getStudyManager() - .setVariantEngineConfigurationSampleIndex(studyFqn, sampleIndexConfiguration, token); - study = catalogManager.getStudyManager().get(studyFqn, null, token).first(); + .setVariantEngineConfigurationSampleIndex(studyFqn, sampleIndexConfiguration, ownerToken); + study = catalogManager.getStudyManager().get(studyFqn, null, ownerToken).first(); System.out.println("getVariantEngineConfiguration() = " + study.getInternal().getConfiguration().getVariantEngine()); assertEquals(sampleIndexConfiguration, study.getInternal().getConfiguration().getVariantEngine().getSampleIndex()); @@ -234,40 +233,40 @@ public void testSetVariantEngineConfiguration() throws CatalogException { @Test public void uploadTemplates() throws IOException, CatalogException { InputStream inputStream = getClass().getResource("/template.zip").openStream(); - OpenCGAResult result = catalogManager.getStudyManager().uploadTemplate(studyFqn, "template.zip", inputStream, token); + OpenCGAResult result = catalogManager.getStudyManager().uploadTemplate(studyFqn, "template.zip", inputStream, ownerToken); assertFalse(StringUtils.isEmpty(result.first())); System.out.println(result.first()); inputStream = getClass().getResource("/template.zip").openStream(); - result = catalogManager.getStudyManager().uploadTemplate(studyFqn, "template.zip", inputStream, token); + result = catalogManager.getStudyManager().uploadTemplate(studyFqn, "template.zip", inputStream, ownerToken); System.out.println(result.first()); } @Test public void deleteTemplates() throws IOException, CatalogException { InputStream inputStream = getClass().getResource("/template.zip").openStream(); - String templateId = catalogManager.getStudyManager().uploadTemplate(studyFqn, "template.zip", inputStream, token).first(); + String templateId = catalogManager.getStudyManager().uploadTemplate(studyFqn, "template.zip", inputStream, ownerToken).first(); - Boolean deleted = catalogManager.getStudyManager().deleteTemplate(studyFqn, templateId, token).first(); + Boolean deleted = catalogManager.getStudyManager().deleteTemplate(studyFqn, templateId, ownerToken).first(); assertTrue(deleted); thrown.expectMessage("doesn't exist"); thrown.expect(CatalogException.class); - catalogManager.getStudyManager().deleteTemplate(studyFqn, templateId, token); + catalogManager.getStudyManager().deleteTemplate(studyFqn, templateId, ownerToken); } @Test public void emptyGroupTest() throws CatalogException { // In the list of users we add it as null to test it properly - catalogManager.getStudyManager().createGroup(studyFqn, "@test", null, token); - Group first = catalogManager.getStudyManager().getGroup(studyFqn, "@test", token).first(); + catalogManager.getStudyManager().createGroup(studyFqn, "@test", null, ownerToken); + Group first = catalogManager.getStudyManager().getGroup(studyFqn, "@test", ownerToken).first(); assertNotNull(first.getUserIds()); - catalogManager.getUserManager().create("dummy", "dummy", "dummy@mail.com", TestParamConstants.PASSWORD, "", 0L, Account.AccountType.GUEST, opencgaToken); - catalogManager.getStudyManager().createGroup(studyFqn, "@test2", Collections.singletonList("dummy"), token); - catalogManager.getStudyManager().updateAcl(studyFqn, "@test2", new StudyAclParams("", "view_only"), ParamUtils.AclAction.ADD, token); + catalogManager.getUserManager().create("dummy", "dummy", "dummy@mail.com", TestParamConstants.PASSWORD, organizationId, 0L, opencgaToken); + catalogManager.getStudyManager().createGroup(studyFqn, "@test2", Collections.singletonList("dummy"), ownerToken); + catalogManager.getStudyManager().updateAcl(studyFqn, "@test2", new StudyAclParams("", "view_only"), ParamUtils.AclAction.ADD, ownerToken); - String dummyToken = catalogManager.getUserManager().login("dummy", TestParamConstants.PASSWORD).getToken(); + String dummyToken = catalogManager.getUserManager().login(organizationId, "dummy", TestParamConstants.PASSWORD).getToken(); OpenCGAResult search = catalogManager.getFileManager().search(studyFqn, new Query(), new QueryOptions(), dummyToken); assertTrue(search.getNumResults() > 0); } diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/UserManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/UserManagerTest.java new file mode 100644 index 00000000000..f23295fbd60 --- /dev/null +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/UserManagerTest.java @@ -0,0 +1,664 @@ +package org.opencb.opencga.catalog.managers; + +import com.fasterxml.jackson.core.JsonProcessingException; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.StringUtils; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; +import org.opencb.commons.datastore.core.DataResult; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.TestParamConstants; +import org.opencb.opencga.catalog.db.api.UserDBAdaptor; +import org.opencb.opencga.catalog.exceptions.*; +import org.opencb.opencga.catalog.utils.ParamUtils; +import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.common.PasswordUtils; +import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.models.JwtPayload; +import org.opencb.opencga.core.models.organizations.OrganizationCreateParams; +import org.opencb.opencga.core.models.organizations.OrganizationUpdateParams; +import org.opencb.opencga.core.models.project.Project; +import org.opencb.opencga.core.models.project.ProjectCreateParams; +import org.opencb.opencga.core.models.project.ProjectOrganism; +import org.opencb.opencga.core.models.study.Group; +import org.opencb.opencga.core.models.study.GroupUpdateParams; +import org.opencb.opencga.core.models.study.Study; +import org.opencb.opencga.core.models.study.StudyAclParams; +import org.opencb.opencga.core.models.user.*; +import org.opencb.opencga.core.response.OpenCGAResult; +import org.opencb.opencga.core.testclassification.duration.MediumTests; + +import javax.naming.NamingException; +import java.io.IOException; +import java.util.*; + +import static org.hamcrest.CoreMatchers.allOf; +import static org.hamcrest.CoreMatchers.containsString; +import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.opencb.opencga.core.common.JacksonUtils.getUpdateObjectMapper; + +@Category(MediumTests.class) +public class UserManagerTest extends AbstractManagerTest { + + @Test + public void createOpencgaUserTest() throws CatalogException { + thrown.expect(CatalogException.class); + thrown.expectMessage("forbidden"); + catalogManager.getUserManager().create(new User().setId(ParamConstants.OPENCGA_USER_ID).setName(orgOwnerUserId) + .setOrganization(organizationId), TestParamConstants.PASSWORD, opencgaToken); + } + + + @Test + public void testAdminUserExists() throws Exception { + String token = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); + JwtPayload payload = catalogManager.getUserManager().validateToken(token); + assertEquals(ParamConstants.OPENCGA_USER_ID, payload.getUserId()); + assertEquals(ParamConstants.ADMIN_ORGANIZATION, payload.getOrganization()); + } + + @Test + public void searchUsersTest() throws CatalogException { + OpenCGAResult search = catalogManager.getUserManager().search(organizationId, new Query(), QueryOptions.empty(), opencgaToken); + assertEquals(8, search.getNumResults()); + for (User user : search.getResults()) { + if (noAccessUserId1.equals(user.getId())) { + assertEquals(0, user.getProjects().size()); + } else if (user.getId().startsWith("normalUser")) { + assertEquals(1, user.getProjects().size()); + } else { + assertEquals(2, user.getProjects().size()); + } + } + + search = catalogManager.getUserManager().search(null, new Query(), QueryOptions.empty(), ownerToken); + assertEquals(8, search.getNumResults()); + + search = catalogManager.getUserManager().search(null, new Query(), QueryOptions.empty(), orgAdminToken2); + assertEquals(8, search.getNumResults()); + + search = catalogManager.getUserManager().search(null, new Query(), QueryOptions.empty(), orgAdminToken1); + assertEquals(8, search.getNumResults()); + + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().search(null, new Query(), + QueryOptions.empty(), studyAdminToken1)); + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().search(null, new Query(), + QueryOptions.empty(), normalToken1)); + } + + @Test + public void testGetToken() throws Exception { + String token = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); + Map claims = new HashMap<>(); + claims.put("a", "hola"); + claims.put("ab", "byw"); + // Create a token valid for 1 second + String expiringToken = catalogManager.getUserManager().getToken(ParamConstants.ADMIN_ORGANIZATION, "opencga", claims, 1L, token); + assertEquals("opencga", catalogManager.getUserManager().validateToken(expiringToken).getUserId()); + + String nonExpiringToken = catalogManager.getUserManager().getNonExpiringToken(ParamConstants.ADMIN_ORGANIZATION, "opencga", claims, token); + assertEquals("opencga", catalogManager.getUserManager().validateToken(nonExpiringToken).getUserId()); + + Thread.sleep(1000); + thrown.expect(CatalogAuthenticationException.class); + thrown.expectMessage("expired"); + assertEquals("opencga", catalogManager.getUserManager().validateToken(expiringToken).getUserId()); + } + + @Test + public void loginWithoutOrganizationId() throws CatalogException { + String token = catalogManager.getUserManager().login(null, ParamConstants.OPENCGA_USER_ID, TestParamConstants.ADMIN_PASSWORD).getToken(); + assertTrue(StringUtils.isNotEmpty(token)); + JwtPayload jwtPayload = new JwtPayload(token); + assertEquals(ParamConstants.ADMIN_ORGANIZATION, jwtPayload.getOrganization()); + + token = catalogManager.getUserManager().login(null, orgOwnerUserId, TestParamConstants.PASSWORD).getToken(); + assertTrue(StringUtils.isNotEmpty(token)); + jwtPayload = new JwtPayload(token); + assertEquals(organizationId, jwtPayload.getOrganization()); + + // Create a third organization + catalogManager.getOrganizationManager().create(new OrganizationCreateParams().setId("other").setName("Test"), QueryOptions.empty(), opencgaToken); + token = catalogManager.getUserManager().login(null, ParamConstants.OPENCGA_USER_ID, TestParamConstants.ADMIN_PASSWORD).getToken(); + assertTrue(StringUtils.isNotEmpty(token)); + jwtPayload = new JwtPayload(token); + assertEquals(ParamConstants.ADMIN_ORGANIZATION, jwtPayload.getOrganization()); + + thrown.expect(CatalogParameterException.class); + thrown.expectMessage("organization"); + catalogManager.getUserManager().login(null, orgOwnerUserId, TestParamConstants.PASSWORD); + } + + @Test + public void testCreateExistingUser() throws Exception { + thrown.expect(CatalogException.class); + thrown.expectMessage(containsString("already exists")); + catalogManager.getUserManager().create(orgOwnerUserId, "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, organizationId, + null, opencgaToken); + } + + @Test + public void testCreateAnonymousUser() throws Exception { + thrown.expect(CatalogParameterException.class); + thrown.expectMessage(containsString("reserved")); + catalogManager.getUserManager().create(ParamConstants.ANONYMOUS_USER_ID, "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, + organizationId, null, opencgaToken); + } + + @Test + public void testCreateRegisteredUser() throws Exception { + thrown.expect(CatalogParameterException.class); + thrown.expectMessage(containsString("reserved")); + catalogManager.getUserManager().create(ParamConstants.REGISTERED_USERS, "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, organizationId, null, + opencgaToken); + } + + @Test + public void testLogin() throws Exception { + catalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD); + + thrown.expect(CatalogAuthenticationException.class); + thrown.expectMessage(allOf(containsString("Incorrect"), containsString("password"))); + catalogManager.getUserManager().login(organizationId, normalUserId1, "fakePassword"); + } + + @Test + public void refreshTokenTest() throws Exception { + String refreshToken = catalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD).getRefreshToken(); + AuthenticationResponse authenticationResponse = catalogManager.getUserManager().refreshToken(refreshToken); + assertNotNull(authenticationResponse); + assertNotNull(authenticationResponse.getToken()); + } + + @Test + public void anonymousUserLoginTest() throws CatalogException { + AuthenticationResponse authResponse = catalogManager.getUserManager().loginAnonymous(organizationId); + assertNotNull(authResponse.getToken()); + + String org2 = "otherOrg"; + catalogManager.getOrganizationManager().create(new OrganizationCreateParams().setId(org2), QueryOptions.empty(), opencgaToken); + catalogManager.getUserManager().create(new User().setId("userFromOrg2").setName("name").setOrganization(org2), TestParamConstants.PASSWORD, opencgaToken); + catalogManager.getOrganizationManager().update(org2, new OrganizationUpdateParams().setOwner("userFromOrg2"), null, opencgaToken); + String owner2Token = catalogManager.getUserManager().login(org2, "userFromOrg2", TestParamConstants.PASSWORD).getToken(); + Project p = catalogManager.getProjectManager().create(new ProjectCreateParams() + .setId("project") + .setOrganism(new ProjectOrganism("Homo sapiens", "GRCh38")), + INCLUDE_RESULT, owner2Token).first(); + Study study = catalogManager.getStudyManager().create(p.getFqn(), new Study().setId("study"), INCLUDE_RESULT, owner2Token).first(); + + try { + catalogManager.getUserManager().loginAnonymous(org2); + fail("Anonymous user should not get a token for that organization as it has not been granted any kind of access"); + } catch (Exception e) { + assertEquals(CatalogAuthenticationException.class, e.getClass()); + assertTrue(e.getMessage().contains("not found")); + } + + catalogManager.getStudyManager().updateGroup(study.getFqn(), ParamConstants.MEMBERS_GROUP, ParamUtils.BasicUpdateAction.ADD, + new GroupUpdateParams(Collections.singletonList("*")), owner2Token); + authResponse = catalogManager.getUserManager().loginAnonymous(org2); + assertNotNull(authResponse.getToken()); + + + catalogManager.getStudyManager().updateGroup(study.getFqn(), ParamConstants.MEMBERS_GROUP, ParamUtils.BasicUpdateAction.REMOVE, + new GroupUpdateParams(Collections.singletonList("*")), owner2Token); + thrown.expect(CatalogAuthenticationException.class); + thrown.expectMessage("not found"); + catalogManager.getUserManager().loginAnonymous(org2); + } + + @Test + public void incrementLoginAttemptsTest() throws CatalogException { + assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, "incorrect")); + User user = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), ownerToken).first(); + UserInternal userInternal3 = user.getInternal(); + assertEquals(1, userInternal3.getAccount().getFailedAttempts()); + assertEquals(UserStatus.READY, user.getInternal().getStatus().getId()); + + for (int i = 2; i < 5; i++) { + assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, "incorrect")); + user = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), ownerToken).first(); + UserInternal userInternal = user.getInternal(); + assertEquals(i, userInternal.getAccount().getFailedAttempts()); + assertEquals(UserStatus.READY, user.getInternal().getStatus().getId()); + } + + assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, "incorrect")); + user = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), ownerToken).first(); + UserInternal userInternal2 = user.getInternal(); + assertEquals(5, userInternal2.getAccount().getFailedAttempts()); + assertEquals(UserStatus.BANNED, user.getInternal().getStatus().getId()); + + CatalogAuthenticationException incorrect = assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, "incorrect")); + assertTrue(incorrect.getMessage().contains("banned")); + user = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), ownerToken).first(); + UserInternal userInternal1 = user.getInternal(); + assertEquals(5, userInternal1.getAccount().getFailedAttempts()); + assertEquals(UserStatus.BANNED, user.getInternal().getStatus().getId()); + + CatalogAuthenticationException authException = assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD)); + assertTrue(authException.getMessage().contains("banned")); + + // Remove ban from user + catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.READY, QueryOptions.empty(), ownerToken); + user = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), ownerToken).first(); + UserInternal userInternal = user.getInternal(); + assertEquals(0, userInternal.getAccount().getFailedAttempts()); + assertEquals(UserStatus.READY, user.getInternal().getStatus().getId()); + + String token = catalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD).getToken(); + assertNotNull(token); + } + + @Test + public void changeUserStatusTest() throws CatalogException { + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.BANNED, QueryOptions.empty(), normalToken1)); + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.BANNED, QueryOptions.empty(), studyAdminToken1)); + assertThrows(CatalogParameterException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.BANNED, QueryOptions.empty(), ownerToken)); + catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.SUSPENDED, QueryOptions.empty(), ownerToken); + catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.SUSPENDED, QueryOptions.empty(), orgAdminToken1); + catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.SUSPENDED, QueryOptions.empty(), opencgaToken); + + catalogManager.getUserManager().changeStatus(organizationId, orgAdminUserId1, UserStatus.SUSPENDED, QueryOptions.empty(), ownerToken); + CatalogAuthorizationException authException = assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, orgOwnerUserId, UserStatus.SUSPENDED, QueryOptions.empty(), ownerToken)); + assertTrue(authException.getMessage().contains("own account")); + + authException = assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, orgAdminUserId1, UserStatus.SUSPENDED, QueryOptions.empty(), orgAdminToken2)); + assertTrue(authException.getMessage().contains("suspend administrators")); + + CatalogAuthenticationException incorrect = assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, orgAdminUserId1, TestParamConstants.PASSWORD)); + assertTrue(incorrect.getMessage().contains("suspended")); + + catalogManager.getUserManager().changeStatus(organizationId, orgAdminUserId1, UserStatus.READY, QueryOptions.empty(), orgAdminToken2); + String token = catalogManager.getUserManager().login(organizationId, orgAdminUserId1, TestParamConstants.PASSWORD).getToken(); + assertNotNull(token); + + CatalogParameterException paramException = assertThrows(CatalogParameterException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, orgAdminUserId1, "NOT_A_STATUS", QueryOptions.empty(), orgAdminToken2)); + assertTrue(paramException.getMessage().contains("Invalid status")); + + CatalogDBException dbException = assertThrows(CatalogDBException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, "notAUser", UserStatus.SUSPENDED, QueryOptions.empty(), orgAdminToken2)); + assertTrue(dbException.getMessage().contains("not exist")); + } + + @Test + public void loginExpiredAccountTest() throws CatalogException { + // Expire account of normalUserId1 + ObjectMap params = new ObjectMap(UserDBAdaptor.QueryParams.INTERNAL_ACCOUNT_EXPIRATION_DATE.key(), TimeUtils.getTime()); + catalogManager.getUserManager().getUserDBAdaptor(organizationId).update(normalUserId1, params); + + CatalogAuthenticationException authException = assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD)); + assertTrue(authException.getMessage().contains("expired")); + + // Ensure it doesn't matter whether opencga account is expired or not + catalogManager.getUserManager().getUserDBAdaptor(ParamConstants.ADMIN_ORGANIZATION).update(ParamConstants.OPENCGA_USER_ID, params); + String token = catalogManager.getUserManager().login(ParamConstants.ADMIN_ORGANIZATION, ParamConstants.OPENCGA_USER_ID, TestParamConstants.ADMIN_PASSWORD).getToken(); + assertNotNull(token); + } + + @Test + public void updateUserTest() throws JsonProcessingException, CatalogException { + UserUpdateParams userUpdateParams = new UserUpdateParams() + .setName("newName") + .setEmail("mail@mail.com"); + ObjectMap updateParams = new ObjectMap(getUpdateObjectMapper().writeValueAsString(userUpdateParams)); + User user = catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, normalToken1).first(); + assertEquals(userUpdateParams.getName(), user.getName()); + assertEquals(userUpdateParams.getEmail(), user.getEmail()); + + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, normalToken2)); + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, opencgaToken)); + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, ownerToken)); + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, orgAdminToken1)); + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, studyAdminToken1)); + + userUpdateParams = new UserUpdateParams() + .setEmail("notAnEmail"); + ObjectMap updateParams2 = new ObjectMap(getUpdateObjectMapper().writeValueAsString(userUpdateParams)); + assertThrows(CatalogParameterException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams2, INCLUDE_RESULT, normalToken1)); + } + + @Test + public void testGetUserInfo() throws CatalogException { + // OpenCGA administrator + DataResult user = catalogManager.getUserManager().get(organizationId, + Arrays.asList(normalUserId1, normalUserId2, normalUserId3), new QueryOptions(), opencgaToken); + assertEquals(3, user.getNumResults()); + assertEquals(normalUserId1, user.getResults().get(0).getId()); + assertEquals(normalUserId2, user.getResults().get(1).getId()); + assertEquals(normalUserId3, user.getResults().get(2).getId()); + + // Organization owner + user = catalogManager.getUserManager().get(organizationId, Arrays.asList(normalUserId1, normalUserId2, normalUserId3), + new QueryOptions(), ownerToken); + assertEquals(3, user.getNumResults()); + assertEquals(normalUserId1, user.getResults().get(0).getId()); + assertEquals(normalUserId2, user.getResults().get(1).getId()); + assertEquals(normalUserId3, user.getResults().get(2).getId()); + + // Organization administrator + user = catalogManager.getUserManager().get(organizationId, Arrays.asList(normalUserId1, normalUserId2, normalUserId3), + new QueryOptions(), orgAdminToken1); + assertEquals(3, user.getNumResults()); + assertEquals(normalUserId1, user.getResults().get(0).getId()); + assertEquals(normalUserId2, user.getResults().get(1).getId()); + assertEquals(normalUserId3, user.getResults().get(2).getId()); + + thrown.expect(CatalogAuthorizationException.class); + thrown.expectMessage("organization"); + catalogManager.getUserManager().get(organizationId, Arrays.asList(normalUserId1, normalUserId2, normalUserId3), new QueryOptions(), + studyAdminToken1); + } + + @Test + public void testGetProjectsFromUserInfo() throws CatalogException { + String userId = organizationId; + catalogManager.getUserManager().create(userId, "test", "mail@mail.com", TestParamConstants.PASSWORD, organizationId, null, + opencgaToken); + catalogManager.getStudyManager().updateGroup(studyFqn, ParamConstants.MEMBERS_GROUP, ParamUtils.BasicUpdateAction.ADD, + new GroupUpdateParams(Collections.singletonList("test")), ownerToken); + String token = catalogManager.getUserManager().login(organizationId, userId, TestParamConstants.PASSWORD).getToken(); + + DataResult user = catalogManager.getUserManager().get(organizationId, userId, new QueryOptions(), token); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + user = catalogManager.getUserManager().get(organizationId, normalUserId3, new QueryOptions(), normalToken3); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + user = catalogManager.getUserManager().get(organizationId, orgOwnerUserId, new QueryOptions(), ownerToken); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + user = catalogManager.getUserManager().get(organizationId, orgAdminUserId1, new QueryOptions(), orgAdminToken1); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + user = catalogManager.getUserManager().get(organizationId, studyAdminUserId1, new QueryOptions(), studyAdminToken1); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + user = catalogManager.getUserManager().get(organizationId, normalUserId1, new QueryOptions(), orgAdminToken1); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + + user = catalogManager.getUserManager().get(null, normalUserId1, new QueryOptions(), normalToken1); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + user = catalogManager.getUserManager().get(null, normalUserId3, new QueryOptions(), normalToken3); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + user = catalogManager.getUserManager().get(null, orgOwnerUserId, new QueryOptions(), ownerToken); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + user = catalogManager.getUserManager().get(null, orgAdminUserId1, new QueryOptions(), orgAdminToken1); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + user = catalogManager.getUserManager().get(null, studyAdminUserId1, new QueryOptions(), studyAdminToken1); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + user = catalogManager.getUserManager().get(null, normalUserId1, new QueryOptions(), orgAdminToken1); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + } + + @Test + public void testModifyUser() throws CatalogException, InterruptedException, IOException { + ObjectMap params = new ObjectMap(); + String newName = "Changed Name " + RandomStringUtils.randomAlphanumeric(10); + String newPassword = PasswordUtils.getStrongRandomPassword(); + String newEmail = "new@email.ac.uk"; + + params.put("name", newName); + + Thread.sleep(10); + + catalogManager.getUserManager().update(orgOwnerUserId, params, null, ownerToken); + catalogManager.getUserManager().update(orgOwnerUserId, new ObjectMap("email", newEmail), null, ownerToken); + catalogManager.getUserManager().changePassword(organizationId, orgOwnerUserId, TestParamConstants.PASSWORD, newPassword); + + List userList = catalogManager.getUserManager().get(organizationId, orgOwnerUserId, new QueryOptions(QueryOptions + .INCLUDE, Arrays.asList(UserDBAdaptor.QueryParams.NAME.key(), UserDBAdaptor.QueryParams.EMAIL.key(), + UserDBAdaptor.QueryParams.ATTRIBUTES.key())), ownerToken).getResults(); + User userPost = userList.get(0); + System.out.println("userPost = " + userPost); + assertEquals(userPost.getName(), newName); + assertEquals(userPost.getEmail(), newEmail); + + catalogManager.getUserManager().login(organizationId, orgOwnerUserId, newPassword); + CatalogAuthenticationException exception = assertThrows(CatalogAuthenticationException.class, + () -> catalogManager.getUserManager().changePassword(organizationId, orgOwnerUserId, newPassword, TestParamConstants.PASSWORD)); + assertTrue(exception.getMessage().contains("The new password has already been used")); + + String anotherPassword = PasswordUtils.getStrongRandomPassword(); + catalogManager.getUserManager().changePassword(organizationId, orgOwnerUserId, newPassword, anotherPassword); + catalogManager.getUserManager().login(organizationId, orgOwnerUserId, anotherPassword); + + try { + params = new ObjectMap(); + params.put("password", "1234321"); + catalogManager.getUserManager().update(orgOwnerUserId, params, null, ownerToken); + fail("Expected exception"); + } catch (CatalogDBException e) { + System.out.println(e); + } + + try { + catalogManager.getUserManager().update(orgOwnerUserId, params, null, orgAdminToken1); + fail("Expected exception"); + } catch (CatalogException e) { + System.out.println(e); + } + } + + @Test + public void automaticPasswordExpirationTest() throws CatalogException { + // Set 1 day of password expiration + catalogManager.getConfiguration().getAccount().setPasswordExpirationDays(1); + + String oneDay = TimeUtils.getTime(TimeUtils.addDaysToCurrentDate(1)); + String twoDays = TimeUtils.getTime(TimeUtils.addDaysToCurrentDate(2)); + + User user = new User().setId("tempUser"); + String password = PasswordUtils.getStrongRandomPassword(); + User storedUser = catalogManager.getUserManager().create(user, password, ownerToken).first(); + Account account2 = storedUser.getInternal().getAccount(); + assertTrue(Long.parseLong(oneDay) <= Long.parseLong(account2.getPassword().getExpirationDate())); + Account account1 = storedUser.getInternal().getAccount(); + assertTrue(Long.parseLong(twoDays) > Long.parseLong(account1.getPassword().getExpirationDate())); + + // Set 1 day of password expiration + catalogManager.getConfiguration().getAccount().setPasswordExpirationDays(-5); + user = new User().setId("tempUser2"); + storedUser = catalogManager.getUserManager().create(user, password, ownerToken).first(); + Account account = storedUser.getInternal().getAccount(); + assertNull(account.getPassword().getExpirationDate()); + } + + @Test + public void loginUserPasswordExpiredTest() throws CatalogException { + try (CatalogManager mockCatalogManager = mockCatalogManager()) { + UserDBAdaptor userDBAdaptor = mockCatalogManager.getUserManager().getUserDBAdaptor(organizationId); + + OpenCGAResult result = mockCatalogManager.getUserManager().get(organizationId, normalUserId1, new QueryOptions(), normalToken1); + + // Set password expired 2 days ago + Date date = TimeUtils.addDaysToCurrentDate(-2); + String beforeYesterday = TimeUtils.getTime(date); + User user = result.first(); + user.getInternal().getAccount().getPassword().setExpirationDate(beforeYesterday); + + Mockito.doReturn(result).when(userDBAdaptor).get(normalUserId1, UserManager.INCLUDE_INTERNAL); + CatalogAuthenticationException exception = assertThrows(CatalogAuthenticationException.class, + () -> mockCatalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD)); + assertTrue(exception.getMessage().contains("expired on " + beforeYesterday)); + } + } + + @Test + public void changePasswordTest() throws CatalogException { + String newPassword = PasswordUtils.getStrongRandomPassword(); + catalogManager.getUserManager().changePassword(organizationId, normalUserId1, TestParamConstants.PASSWORD, newPassword); + catalogManager.getUserManager().login(organizationId, normalUserId1, newPassword); + + CatalogAuthenticationException exception = assertThrows(CatalogAuthenticationException.class, + () -> catalogManager.getUserManager().changePassword(organizationId, normalUserId1, TestParamConstants.PASSWORD, newPassword)); + assertTrue(exception.getMessage().contains("verify that the current password is correct")); + + String anotherPassword = PasswordUtils.getStrongRandomPassword(); + catalogManager.getUserManager().changePassword(organizationId, normalUserId1, newPassword, anotherPassword); + catalogManager.getUserManager().login(organizationId, normalUserId1, anotherPassword); + } + + @Test + public void testUpdateUserConfig() throws CatalogException { + Map map = new HashMap<>(); + map.put("key1", "value1"); + map.put("key2", "value2"); + catalogManager.getUserManager().setConfig(normalUserId1, "a", map, normalToken1); + + Map config = (Map) catalogManager.getUserManager().getConfig(normalUserId1, "a", normalToken1).first(); + assertEquals(2, config.size()); + assertEquals("value1", config.get("key1")); + assertEquals("value2", config.get("key2")); + + map = new HashMap<>(); + map.put("key2", "value3"); + catalogManager.getUserManager().setConfig(normalUserId1, "a", map, normalToken1); + config = (Map) catalogManager.getUserManager().getConfig(normalUserId1, "a", normalToken1).first(); + assertEquals(1, config.size()); + assertEquals("value3", config.get("key2")); + + catalogManager.getUserManager().deleteConfig(normalUserId1, "a", normalToken1); + + thrown.expect(CatalogException.class); + thrown.expectMessage("not found"); + catalogManager.getUserManager().getConfig(normalUserId1, "a", normalToken1); + } + + private String getAdminToken() throws CatalogException, IOException { + return catalogManager.getUserManager().loginAsAdmin("admin").getToken(); + } + + @Test + public void createUserUsingMailAsId() throws CatalogException { + catalogManager.getUserManager().create(new User().setId("hello.mail@mymail.org").setName("Hello"), TestParamConstants.PASSWORD, ownerToken); + AuthenticationResponse login = catalogManager.getUserManager().login(organizationId, "hello.mail@mymail.org", TestParamConstants.PASSWORD); + assertNotNull(login); + User user = catalogManager.getUserManager().get(organizationId, "hello.mail@mymail.org", new QueryOptions(), login.getToken()).first(); + assertEquals("hello.mail@mymail.org", user.getId()); + } + + @Test + public void getUserInfoTest() throws CatalogException { + OpenCGAResult result = catalogManager.getUserManager().get(organizationId, orgOwnerUserId, QueryOptions.empty(), ownerToken); + assertEquals(1, result.getNumResults()); + assertNotNull(result.first().getProjects()); + assertEquals(2, result.first().getProjects().size()); + + result = catalogManager.getUserManager().get(organizationId, orgAdminUserId1, QueryOptions.empty(), orgAdminToken1); + assertEquals(1, result.getNumResults()); + assertNotNull(result.first().getProjects()); + assertEquals(2, result.first().getProjects().size()); + + result = catalogManager.getUserManager().get(organizationId, studyAdminUserId1, QueryOptions.empty(), studyAdminToken1); + assertEquals(1, result.getNumResults()); + assertNotNull(result.first().getProjects()); + assertEquals(2, result.first().getProjects().size()); + + result = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), normalToken1); + assertEquals(1, result.getNumResults()); + assertNotNull(result.first().getProjects()); + assertEquals(1, result.first().getProjects().size()); + } + + @Ignore + @Test + public void importLdapUsers() throws CatalogException, NamingException, IOException { + // Action only for admins + catalogManager.getUserManager().importRemoteEntities(organizationId, "ldap", Arrays.asList("pfurio", "imedina"), false, null, null, + getAdminToken()); + // TODO: Validate the users have been imported + } + + // To make this test work we will need to add a correct user and password to be able to login + @Ignore + @Test + public void loginNotRegisteredUsers() throws CatalogException { + // Action only for admins + Group group = new Group("ldap", Collections.emptyList()).setSyncedFrom(new Group.Sync("ldap", "bio")); + catalogManager.getStudyManager().createGroup(studyFqn, group, ownerToken); + catalogManager.getStudyManager().updateAcl(studyFqn, "@ldap", new StudyAclParams("", "view_only"), + ParamUtils.AclAction.SET, ownerToken); + String token = catalogManager.getUserManager().login(organizationId, orgOwnerUserId, "password").getToken(); + + assertEquals(9, catalogManager.getSampleManager().count(studyFqn, new Query(), token).getNumTotalResults()); + + // We remove the permissions for group ldap + catalogManager.getStudyManager().updateAcl(studyFqn, "@ldap", new StudyAclParams("", ""), + ParamUtils.AclAction.RESET, this.ownerToken); + + assertEquals(0, catalogManager.getSampleManager().count(studyFqn, new Query(), token).getNumTotalResults()); + } + + @Ignore + @Test + public void syncUsers() throws CatalogException { + // Action only for admins + String token = catalogManager.getUserManager().loginAsAdmin("admin").getToken(); + + catalogManager.getUserManager().importRemoteGroupOfUsers(organizationId, "ldap", "bio", "bio", studyFqn, true, token); + DataResult bio = catalogManager.getStudyManager().getGroup(studyFqn, "bio", this.ownerToken); + + assertEquals(1, bio.getNumResults()); + assertEquals(0, bio.first().getUserIds().size()); + + catalogManager.getUserManager().syncAllUsersOfExternalGroup(organizationId, studyFqn, "ldap", token); + bio = catalogManager.getStudyManager().getGroup(studyFqn, "bio", this.ownerToken); + + assertEquals(1, bio.getNumResults()); + assertTrue(!bio.first().getUserIds().isEmpty()); + } + + @Ignore + @Test + public void importLdapGroups() throws CatalogException, IOException { + // Action only for admins + String remoteGroup = "bio"; + String internalGroup = "test"; + catalogManager.getUserManager().importRemoteGroupOfUsers(organizationId, "ldap", remoteGroup, internalGroup, studyFqn, true, getAdminToken()); + + DataResult test = catalogManager.getStudyManager().getGroup(studyFqn, "test", ownerToken); + assertEquals(1, test.getNumResults()); + assertEquals("@test", test.first().getId()); + assertTrue(test.first().getUserIds().size() > 0); + +// internalGroup = "test1"; +// try { +// catalogManager.getUserManager().importRemoteGroupOfUsers("ldap", remoteGroup, internalGroup, study, getAdminToken()); +// fail("Should not be possible creating another group containing the same users that belong to a different group"); +// } catch (CatalogException e) { +// System.out.println(e.getMessage()); +// } + + remoteGroup = "bioo"; + internalGroup = "test2"; + thrown.expect(CatalogException.class); + thrown.expectMessage("not found"); + catalogManager.getUserManager().importRemoteGroupOfUsers(organizationId, "ldap", remoteGroup, internalGroup, studyFqn, true, getAdminToken()); + } + + +} diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/migration/MigrationManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/migration/MigrationManagerTest.java index 2f688a81fba..b7d6b72f62f 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/migration/MigrationManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/migration/MigrationManagerTest.java @@ -11,12 +11,14 @@ import org.opencb.opencga.TestParamConstants; import org.opencb.opencga.catalog.db.api.JobDBAdaptor; import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.AbstractManagerTest; import org.opencb.opencga.core.config.storage.StorageConfiguration; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.job.Job; import org.opencb.opencga.core.models.job.JobReferenceParam; +import org.opencb.opencga.core.models.migration.MigrationRun; import org.opencb.opencga.core.testclassification.duration.MediumTests; import java.io.FileOutputStream; @@ -120,10 +122,11 @@ protected void run() throws MigrationException { public static class MigrationWithJobs extends MigrationTool { @Override protected void run() throws Exception { - String fqn = catalogManager.getProjectManager().search(new Query(), new QueryOptions(), token).first().getFqn(); + String fqn = catalogManager.getProjectManager().search(organizationId, new Query(), new QueryOptions(), token).first().getFqn(); getMigrationRun().getJobs().clear(); - getMigrationRun().addJob(catalogManager.getJobManager().submitProject(fqn, "my-tool", null, Collections.emptyMap(), null, null, null, null, token).first()); + getMigrationRun().addJob(catalogManager.getJobManager().submitProject(fqn, "my-tool", null, Collections.emptyMap(), null, null, + null, null, token).first()); } } @@ -133,9 +136,11 @@ public void setUp() throws Exception { super.setUp(); try (MongoDBAdaptorFactory mongoDBAdaptorFactory = new MongoDBAdaptorFactory(catalogManager.getConfiguration(), catalogManager.getIoManagerFactory())) { - mongoDBAdaptorFactory.getMongoDataStore() - .getCollection(MongoDBAdaptorFactory.MIGRATION_COLLECTION) - .remove(new Document(), new QueryOptions(MongoDBCollection.MULTI, true)); + for (String organizationId : mongoDBAdaptorFactory.getOrganizationIds()) { + mongoDBAdaptorFactory.getMongoDataStore(organizationId) + .getCollection(OrganizationMongoDBAdaptorFactory.MIGRATION_COLLECTION) + .remove(new Document(), new QueryOptions(MongoDBCollection.MULTI, true)); + } } Files.createDirectories(catalogManagerResource.getOpencgaHome().resolve("conf")); try (OutputStream os = new FileOutputStream(catalogManagerResource.getOpencgaHome().resolve("conf").resolve("storage-configuration.yml").toFile())) { @@ -148,22 +153,23 @@ public void testMigration() throws Exception { MigrationManager migrationManager = catalogManager.getMigrationManager(); String token = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); - List> pendingMigrations = migrationManager.getPendingMigrations("0.0.1", token); + List> pendingMigrations = migrationManager.getPendingMigrations(organizationId, "0.0.1", token); assertEquals(0, pendingMigrations.size()); - pendingMigrations = migrationManager.getPendingMigrations("0.1.0", token); + pendingMigrations = migrationManager.getPendingMigrations(organizationId, "0.1.0", token); assertEquals(2, pendingMigrations.size()); for (Class pendingMigration : pendingMigrations) { Migration annotation = pendingMigration.getAnnotation(Migration.class); assertTrue(Arrays.asList("test-1", "test-2").contains(annotation.id())); } // Run migrations up to 0.0.1 - migrationManager.runMigration("0.0.1", Collections.emptySet(), Collections.emptySet(), false, catalogManagerResource.getOpencgaHome().toString(), token); + migrationManager.runMigration("0.0.1", Collections.emptySet(), Collections.emptySet(), false, + catalogManagerResource.getOpencgaHome().toString(), token); - pendingMigrations = migrationManager.getPendingMigrations("0.1.0", token); + pendingMigrations = migrationManager.getPendingMigrations(organizationId, "0.1.0", token); assertEquals(0, pendingMigrations.size()); - pendingMigrations = migrationManager.getPendingMigrations("0.2.0", token); + pendingMigrations = migrationManager.getPendingMigrations(organizationId, "0.2.0", token); assertEquals(2, pendingMigrations.size()); for (int i = 0; i < pendingMigrations.size(); i++) { Class pendingMigration = pendingMigrations.get(i); @@ -171,7 +177,7 @@ public void testMigration() throws Exception { assertTrue(Arrays.asList("test2-1", "test2-2").contains(annotation.id())); } - pendingMigrations = migrationManager.getPendingMigrations("0.2.1", token); + pendingMigrations = migrationManager.getPendingMigrations(organizationId, "0.2.1", token); assertEquals(4, pendingMigrations.size()); for (int i = 0; i < pendingMigrations.size(); i++) { Class pendingMigration = pendingMigrations.get(i); @@ -191,9 +197,10 @@ public void testMigration() throws Exception { fail(); } } - migrationManager.runMigration("0.2.0", Collections.emptySet(), Collections.emptySet(), false, catalogManagerResource.getOpencgaHome().toString(), token); + migrationManager.runMigration("0.2.0", Collections.emptySet(), Collections.emptySet(), false, + catalogManagerResource.getOpencgaHome().toString(), token); - pendingMigrations = migrationManager.getPendingMigrations("0.2.3", token); + pendingMigrations = migrationManager.getPendingMigrations(organizationId, "0.2.3", token); assertEquals(2, pendingMigrations.size()); for (int i = 0; i < pendingMigrations.size(); i++) { Class pendingMigration = pendingMigrations.get(i); @@ -212,17 +219,20 @@ public void testMigration() throws Exception { thrown.expectMessage("manual"); thrown.expect(MigrationException.class); - migrationManager.runMigration("0.2.2", Collections.emptySet(), Collections.emptySet(), false, catalogManagerResource.getOpencgaHome().toString(), token); + migrationManager.runMigration("0.2.2", Collections.emptySet(), Collections.emptySet(), false, + catalogManagerResource.getOpencgaHome().toString(), token); } @Test public void testManualMigrations() throws CatalogException, IOException { String token = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); - MigrationRun migrationRun = catalogManager.getMigrationManager().runManualMigration("0.2.1", "test4-1-manual", catalogManagerResource.getOpencgaHome(), new ObjectMap("key", "OtherValue"), token); + MigrationRun migrationRun = catalogManager.getMigrationManager().runManualMigration(organizationId, "0.2.1", "test4-1-manual", + catalogManagerResource.getOpencgaHome(), false, false, new ObjectMap("key", "OtherValue"), token); assertEquals(MigrationRun.MigrationStatus.ERROR, migrationRun.getStatus()); - migrationRun = catalogManager.getMigrationManager().runManualMigration("0.2.1", "test4-1-manual", catalogManagerResource.getOpencgaHome(), new ObjectMap("key", "value"), token); + migrationRun = catalogManager.getMigrationManager().runManualMigration(organizationId, "0.2.1", "test4-1-manual", + catalogManagerResource.getOpencgaHome(), false, false, new ObjectMap("key", "value"), token); assertEquals(MigrationRun.MigrationStatus.DONE, migrationRun.getStatus()); } @@ -230,19 +240,24 @@ public void testManualMigrations() throws CatalogException, IOException { public void testMigrationsWithJobs() throws CatalogException, IOException { String token = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); - catalogManager.getMigrationManager().runManualMigration("0.2.1", "test4-1-manual", catalogManagerResource.getOpencgaHome(), new ObjectMap("key", "value"), token); + catalogManager.getMigrationManager().runManualMigration("0.2.1", "test4-1-manual", + catalogManagerResource.getOpencgaHome(), new ObjectMap("key", "value"), token); // RUN. New status ON_HOLD - catalogManager.getMigrationManager().runMigration("1.0.0", Collections.emptySet(), Collections.emptySet(), false, catalogManagerResource.getOpencgaHome().toString(), token); + catalogManager.getMigrationManager().runMigration("1.0.0", Collections.emptySet(), Collections.emptySet(), false, + catalogManagerResource.getOpencgaHome().toString(), token); - MigrationRun migrationRun = catalogManager.getMigrationManager().getMigrationRuns(token).stream().filter(p1 -> p1.getKey().id().equals("test-with-jobs")).findFirst().get().getValue(); + MigrationRun migrationRun = catalogManager.getMigrationManager().getMigrationRuns(organizationId, token) + .stream().filter(p1 -> p1.getKey().id().equals("test-with-jobs")).findFirst().get().getValue(); assertEquals(MigrationRun.MigrationStatus.ON_HOLD, migrationRun.getStatus()); Date start = migrationRun.getStart(); JobReferenceParam j = migrationRun.getJobs().get(0); // RUN. Migration run does not get triggered - catalogManager.getMigrationManager().runMigration("1.0.0", Collections.emptySet(), Collections.emptySet(), false, catalogManagerResource.getOpencgaHome().toString(), token); - migrationRun = catalogManager.getMigrationManager().getMigrationRuns(token).stream().filter(p -> p.getKey().id().equals("test-with-jobs")).findFirst().get().getValue(); + catalogManager.getMigrationManager().runMigration("1.0.0", Collections.emptySet(), Collections.emptySet(), false, + catalogManagerResource.getOpencgaHome().toString(), token); + migrationRun = catalogManager.getMigrationManager().getMigrationRuns(organizationId, token) + .stream().filter(p -> p.getKey().id().equals("test-with-jobs")).findFirst().get().getValue(); assertEquals(MigrationRun.MigrationStatus.ON_HOLD, migrationRun.getStatus()); assertEquals(start, migrationRun.getStart()); assertEquals(j, migrationRun.getJobs().get(0)); @@ -253,13 +268,14 @@ public void testMigrationsWithJobs() throws CatalogException, IOException { catalogManager.getJobManager().update(job.getStudy().getId(), job.getId(), new ObjectMap(JobDBAdaptor.QueryParams.INTERNAL_STATUS.key(), status), new QueryOptions(), token); - migrationRun = catalogManager.getMigrationManager().getMigrationRuns(token) + migrationRun = catalogManager.getMigrationManager().getMigrationRuns(organizationId, token) .stream().filter(p -> p.getKey().id().equals("test-with-jobs")).findFirst().get().getValue(); assertEquals(MigrationRun.MigrationStatus.ERROR, migrationRun.getStatus()); // RUN. Migration run triggered. Status: ON_HOLD - catalogManager.getMigrationManager().runMigration("1.0.0", Collections.emptySet(), Collections.emptySet(), false, catalogManagerResource.getOpencgaHome().toString(), token); - migrationRun = catalogManager.getMigrationManager().getMigrationRuns(token) + catalogManager.getMigrationManager().runMigration("1.0.0", Collections.emptySet(), Collections.emptySet(), false, + catalogManagerResource.getOpencgaHome().toString(), token); + migrationRun = catalogManager.getMigrationManager().getMigrationRuns(organizationId, token) .stream().filter(p -> p.getKey().id().equals("test-with-jobs")).findFirst().get().getValue(); assertEquals(MigrationRun.MigrationStatus.ON_HOLD, migrationRun.getStatus()); @@ -270,7 +286,7 @@ public void testMigrationsWithJobs() throws CatalogException, IOException { catalogManager.getJobManager().update(job.getStudy().getId(), job.getId(), new ObjectMap(JobDBAdaptor.QueryParams.INTERNAL_STATUS.key(), status), new QueryOptions(), token); - migrationRun = catalogManager.getMigrationManager().getMigrationRuns(token) + migrationRun = catalogManager.getMigrationManager().getMigrationRuns(organizationId, token) .stream().filter(p -> p.getKey().id().equals("test-with-jobs")).findFirst().get().getValue(); assertEquals(MigrationRun.MigrationStatus.DONE, migrationRun.getStatus()); } @@ -282,13 +298,15 @@ public void testMigrationWithStorageReadOnly() throws Exception { try (OutputStream os = new FileOutputStream(catalogManagerResource.getOpencgaHome().resolve("conf").resolve("storage-configuration.yml").toFile())) { new StorageConfiguration().setMode(StorageConfiguration.Mode.READ_ONLY).serialize(os); } - MigrationRun migrationRun = catalogManager.getMigrationManager().runManualMigration("0.2.2", "test4-2", catalogManagerResource.getOpencgaHome(), new ObjectMap("key", "value"), token); + MigrationRun migrationRun = catalogManager.getMigrationManager().runManualMigration(organizationId, "0.2.2", "test4-2", + catalogManagerResource.getOpencgaHome(), false, false, new ObjectMap("key", "value"), token); assertEquals(MigrationRun.MigrationStatus.PENDING, migrationRun.getStatus()); try (OutputStream os = new FileOutputStream(catalogManagerResource.getOpencgaHome().resolve("conf").resolve("storage-configuration.yml").toFile())) { new StorageConfiguration().setMode(StorageConfiguration.Mode.READ_WRITE).serialize(os); } - migrationRun = catalogManager.getMigrationManager().runManualMigration("0.2.2", "test4-2", catalogManagerResource.getOpencgaHome(), new ObjectMap("key", "value"), token); + migrationRun = catalogManager.getMigrationManager().runManualMigration(organizationId, "0.2.2", "test4-2", + catalogManagerResource.getOpencgaHome(), false, false, new ObjectMap("key", "value"), token); assertEquals(MigrationRun.MigrationStatus.DONE, migrationRun.getStatus()); } diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/stats/solr/AbstractSolrManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/stats/solr/AbstractSolrManagerTest.java deleted file mode 100644 index 488925c504b..00000000000 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/stats/solr/AbstractSolrManagerTest.java +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright 2015-2020 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.catalog.stats.solr; - -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.experimental.categories.Category; -import org.junit.rules.ExpectedException; -import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.commons.test.GenericTest; -import org.opencb.opencga.TestParamConstants; -import org.opencb.opencga.catalog.db.api.DBAdaptor; -import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.managers.CatalogManager; -import org.opencb.opencga.catalog.utils.ParamUtils; -import org.opencb.opencga.core.api.ParamConstants; -import org.opencb.opencga.core.models.AclParams; -import org.opencb.opencga.core.models.cohort.Cohort; -import org.opencb.opencga.core.models.project.Project; -import org.opencb.opencga.core.models.sample.Sample; -import org.opencb.opencga.core.models.sample.SampleAclParams; -import org.opencb.opencga.core.models.study.GroupUpdateParams; -import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.models.study.StudyAclParams; -import org.opencb.opencga.core.models.user.Account; -import org.opencb.opencga.core.testclassification.duration.MediumTests; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; - -@Category(MediumTests.class) -public class AbstractSolrManagerTest extends GenericTest { - - @Rule - public ExpectedException thrown = ExpectedException.none(); - - @Rule - public SolrExternalResource solrExternalResource = new SolrExternalResource(); - - protected CatalogManager catalogManager; - protected CatalogSolrManager catalogSolrManager; - - protected String sessionIdOwner; - protected String sessionIdAdmin; - protected String sessionIdUser; - protected String sessionIdUser2; - protected String sessionIdUser3; - - protected Study study; - protected String studyFqn; - - protected static final QueryOptions INCLUDE_RESULT = new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true); - - @Before - public void setUp() throws IOException, CatalogException { - catalogManager = solrExternalResource.getCatalogManager(); - setUpCatalogManager(catalogManager); - - catalogSolrManager = new CatalogSolrManager(catalogManager); - catalogSolrManager.setSolrClient(solrExternalResource.getSolrClient()); - } - - @After - public void after() throws CatalogException { - solrExternalResource.after(); - catalogSolrManager.close(); - } - - public void setUpCatalogManager(CatalogManager catalogManager) throws IOException, CatalogException { - - catalogManager.getUserManager().create("owner", "Owner", "owner@mail.com", TestParamConstants.PASSWORD, "", null, Account.AccountType.FULL, null); - catalogManager.getUserManager().create("admin1", "Admin", "admin@mail.com", TestParamConstants.PASSWORD, "", null, Account.AccountType.GUEST, null); - catalogManager.getUserManager().create("user1", "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, "", null, Account.AccountType.GUEST, null); - catalogManager.getUserManager().create("user2", "User2 Name", "mail2@ebi.ac.uk", TestParamConstants.PASSWORD, "", null, Account.AccountType.GUEST, null); - catalogManager.getUserManager().create("user3", "User3 Name", "user.2@e.mail", TestParamConstants.PASSWORD, "ACME", null, Account.AccountType.GUEST, null); - - sessionIdOwner = catalogManager.getUserManager().login("owner", TestParamConstants.PASSWORD).getToken(); - sessionIdAdmin = catalogManager.getUserManager().login("admin1", TestParamConstants.PASSWORD).getToken(); - sessionIdUser = catalogManager.getUserManager().login("user1", TestParamConstants.PASSWORD).getToken(); - sessionIdUser2 = catalogManager.getUserManager().login("user2", TestParamConstants.PASSWORD).getToken(); - sessionIdUser3 = catalogManager.getUserManager().login("user3", TestParamConstants.PASSWORD).getToken(); - - Project project = catalogManager.getProjectManager().create("1000G", "Project about some genomes", "", "Homo sapiens", - null, "GRCh38", INCLUDE_RESULT, sessionIdOwner).first(); - studyFqn = catalogManager.getStudyManager().create(project.getFqn(), "phase1", null, "Phase 1", "Done", null, - null, null, null, INCLUDE_RESULT, sessionIdOwner).first().getFqn(); - - catalogManager.getStudyManager().updateGroup(studyFqn, "@admins", ParamUtils.BasicUpdateAction.ADD, - new GroupUpdateParams(Collections.singletonList("admin1")), sessionIdOwner); - catalogManager.getStudyManager().createGroup(studyFqn, "@study_allow", Collections.singletonList("user1"), sessionIdAdmin); - catalogManager.getStudyManager().createGroup(studyFqn, "@study_deny", Collections.singletonList("user2"), sessionIdAdmin); - - catalogManager.getStudyManager().updateAcl(Collections.singletonList(studyFqn), "@study_allow", - new StudyAclParams(null, "view_only"), ParamUtils.AclAction.ADD, sessionIdAdmin); - - study = catalogManager.getStudyManager().get("phase1", new QueryOptions(DBAdaptor.INCLUDE_ACLS, true), sessionIdOwner).first(); - - // Samples - Sample sample1 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("sample1"), INCLUDE_RESULT, - sessionIdAdmin).first(); - Sample sample2 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("sample2"), INCLUDE_RESULT, - sessionIdAdmin).first(); - Sample sample3 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("sample3"), INCLUDE_RESULT, - sessionIdAdmin).first(); - - catalogManager.getSampleManager().updateAcl(studyFqn, Collections.singletonList("sample1"), "@study_deny,user3", - new SampleAclParams(null, null, null, null, "VIEW,VIEW_ANNOTATIONS"), ParamUtils.AclAction.ADD, sessionIdAdmin); - catalogManager.getSampleManager().updateAcl(studyFqn, Collections.singletonList("sample2"), "@study_allow", - new SampleAclParams(null, null, null, null, ""), ParamUtils.AclAction.SET, sessionIdAdmin); - - // Cohorts - catalogManager.getCohortManager().create(studyFqn, new Cohort().setId("cohort1").setSamples(Arrays.asList(sample1, sample2, sample3)), - QueryOptions.empty(), sessionIdAdmin); - catalogManager.getCohortManager().create(studyFqn, new Cohort().setId("cohort2").setSamples(Arrays.asList(sample1, sample2)), - QueryOptions.empty(), sessionIdAdmin); - catalogManager.getCohortManager().create(studyFqn, new Cohort().setId("cohort3").setSamples(Arrays.asList(sample2, sample3)), - QueryOptions.empty(), sessionIdAdmin); - - catalogManager.getCohortManager().updateAcl(studyFqn, Collections.singletonList("cohort1"), "@study_deny,user3", - new AclParams("VIEW,VIEW_ANNOTATIONS"), ParamUtils.AclAction.ADD, sessionIdAdmin); - catalogManager.getCohortManager().updateAcl(studyFqn, Collections.singletonList("cohort2"), "@study_allow", - new AclParams(""), ParamUtils.AclAction.SET, sessionIdAdmin); - } - -} diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/stats/solr/CatalogSolrManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/stats/solr/CatalogSolrManagerTest.java deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/stats/solr/SolrExternalResource.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/stats/solr/SolrExternalResource.java deleted file mode 100644 index 677262d5cf8..00000000000 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/stats/solr/SolrExternalResource.java +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Copyright 2015-2020 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.catalog.stats.solr; - -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer; -import org.apache.solr.client.solrj.request.CoreAdminRequest; -import org.apache.solr.core.NodeConfig; -import org.opencb.opencga.catalog.managers.CatalogManagerExternalResource; -import org.opencb.opencga.core.common.GitRepositoryState; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; - -/** - * @see org.opencb.opencga.storage.core.variant.solr.VariantSolrExternalResource - */ -public class SolrExternalResource extends CatalogManagerExternalResource { - - private static SolrClient solrClient; - - @Override - public void before() throws Exception { - super.before(); - - Path rootDir = getOpencgaHome(); - - String version = GitRepositoryState.getInstance().getBuildVersion(); - - // Copy configuration - copyConfiguration("cohort-managed-schema", CatalogSolrManager.COHORT_CONF_SET + "-" + version); - copyConfiguration("family-managed-schema", CatalogSolrManager.FAMILY_CONF_SET + "-" + version); - copyConfiguration("file-managed-schema", CatalogSolrManager.FILE_CONF_SET + "-" + version); - copyConfiguration("individual-managed-schema", CatalogSolrManager.INDIVIDUAL_CONF_SET + "-" + version); - copyConfiguration("sample-managed-schema", CatalogSolrManager.SAMPLE_CONF_SET + "-" + version); - copyConfiguration("job-managed-schema", CatalogSolrManager.JOB_CONF_SET + "-" + version); - - String solrHome = rootDir.resolve("solr").toString(); - - solrClient = create(solrHome, rootDir.resolve("solr/configsets").toString()); - - CoreAdminRequest.Create request = new CoreAdminRequest.Create(); - request.setCoreName(getConfiguration().getDatabasePrefix() + "-" + CatalogSolrManager.COHORT_SOLR_COLLECTION); - request.setConfigSet(CatalogSolrManager.COHORT_CONF_SET + "-" + version); - request.process(solrClient); - - request.setCoreName(getConfiguration().getDatabasePrefix() + "-" + CatalogSolrManager.SAMPLE_SOLR_COLLECTION); - request.setConfigSet(CatalogSolrManager.SAMPLE_CONF_SET + "-" + version); - request.process(solrClient); - - request.setCoreName(getConfiguration().getDatabasePrefix() + "-" + CatalogSolrManager.INDIVIDUAL_SOLR_COLLECTION); - request.setConfigSet(CatalogSolrManager.INDIVIDUAL_CONF_SET + "-" + version); - request.process(solrClient); - - request.setCoreName(getConfiguration().getDatabasePrefix() + "-" + CatalogSolrManager.FILE_SOLR_COLLECTION); - request.setConfigSet(CatalogSolrManager.FILE_CONF_SET + "-" + version); - request.process(solrClient); - - request.setCoreName(getConfiguration().getDatabasePrefix() + "-" + CatalogSolrManager.FAMILY_SOLR_COLLECTION); - request.setConfigSet(CatalogSolrManager.FAMILY_CONF_SET + "-" + version); - request.process(solrClient); - - request.setCoreName(getConfiguration().getDatabasePrefix() + "-" + CatalogSolrManager.JOB_SOLR_COLLECTION); - request.setConfigSet(CatalogSolrManager.JOB_CONF_SET + "-" + version); - request.process(solrClient); - } - - private void copyConfiguration(String managedSchema, String confFolder) throws IOException { - getResourceUri("solr/configsets/solrconfig.xml", "solr/configsets/" + confFolder + "/solrconfig.xml"); - getResourceUri("solr/" + managedSchema, "solr/configsets/" + confFolder + "/schema.xml"); - getResourceUri("solr/configsets/params.json", "solr/configsets/" + confFolder + "/params.json"); - getResourceUri("solr/configsets/protwords.txt", "solr/configsets/" + confFolder + "/protwords.txt"); - getResourceUri("solr/configsets/stopwords.txt", "solr/configsets/" + confFolder + "/stopwords.txt"); - getResourceUri("solr/configsets/synonyms.txt", "solr/configsets/" + confFolder + "/synonyms.txt"); - getResourceUri("solr/configsets/lang/stopwords_en.txt", "solr/configsets/" + confFolder + "/lang/stopwords_en.txt"); - } - - @Override - public void after() { - super.after(); - close(solrClient); - } - - private void close(SolrClient solrClient) { - try { - ((MyEmbeddedSolrServer) solrClient).realClose(); - } catch (IOException e) { - e.printStackTrace(); - } finally { - solrClient = null; - } - } - - public SolrClient getSolrClient() { - return solrClient; - } -// -// /** -// * Cleans the given solrHome directory and creates a new EmbeddedSolrServer. -// * -// * @param solrHome the Solr home directory to use -// * @param configSetHome the directory containing config sets -// * @param coreName the name of the core, must have a matching directory in configHome -// * -// * @return an EmbeddedSolrServer with a core created for the given coreName -// * @throws IOException -// */ -// public static SolrClient create(final String solrHome, final String configSetHome, final String coreName) -// throws IOException, SolrServerException { -// return create(solrHome, configSetHome, coreName, true); -// } - - /** - * @param solrHome the Solr home directory to use - * @param configSetHome the directory containing config sets - * - * @return an EmbeddedSolrServer with a core created for the given coreName - * @throws IOException - */ - private SolrClient create(final String solrHome, final String configSetHome) { - - final File solrHomeDir = new File(solrHome); - if (!solrHomeDir.exists()) { - solrHomeDir.mkdirs(); - } - - // Copy the solr.xml file to the solr home folder to be used -// getResourceUri("solr/solr.xml", "solr/solr.xml"); - -// final SolrResourceLoader loader = new SolrResourceLoader(solrHomeDir.toPath()); - final Path configSetPath = Paths.get(configSetHome).toAbsolutePath(); - - final NodeConfig config = new NodeConfig.NodeConfigBuilder("embeddedSolrServerNode", solrHomeDir.toPath()) - .setConfigSetBaseDirectory(configSetPath.toString()) - .build(); - - final EmbeddedSolrServer embeddedSolrServer = new MyEmbeddedSolrServer(config, - getConfiguration().getDatabasePrefix() + "_" + CatalogSolrManager.SAMPLE_SOLR_COLLECTION); - - return embeddedSolrServer; - } - - private static class MyEmbeddedSolrServer extends EmbeddedSolrServer { - - public MyEmbeddedSolrServer(NodeConfig nodeConfig, String defaultCoreName) { - super(nodeConfig, defaultCoreName); - } - - @Override - public void close() throws IOException { - } - - private void realClose() throws IOException { - super.close(); - } - } -} diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/CatalogAnnotationsValidatorTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/CatalogAnnotationsValidatorTest.java index e085c2a08a3..ae67ac884fb 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/CatalogAnnotationsValidatorTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/CatalogAnnotationsValidatorTest.java @@ -201,7 +201,7 @@ public void mergeAnnotationsTest() { annotations.put("K", "V"); annotations.put("K2", "V2"); annotations.put("K4", false); - AnnotationSet annotationSet = new AnnotationSet("", "", annotations, "", 1, Collections.emptyMap()); + AnnotationSet annotationSet = new AnnotationSet("", "", annotations); Map newAnnotations = new ObjectMap() .append("K", "newValue") //Modify .append("K2", null) //Delete diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/CatalogFileUtilsTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/CatalogFileUtilsTest.java index 4a51358bb53..3b2fd0877dd 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/CatalogFileUtilsTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/CatalogFileUtilsTest.java @@ -18,31 +18,20 @@ import org.apache.commons.lang3.RandomStringUtils; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; -import org.junit.rules.ExpectedException; -import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.commons.datastore.mongodb.MongoDBConfiguration; -import org.opencb.opencga.TestParamConstants; import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.managers.CatalogManager; -import org.opencb.opencga.catalog.managers.CatalogManagerExternalResource; +import org.opencb.opencga.catalog.managers.AbstractManagerTest; import org.opencb.opencga.catalog.managers.FileUtils; -import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.TimeUtils; -import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.models.common.InternalStatus; import org.opencb.opencga.core.models.file.File; import org.opencb.opencga.core.models.file.FileCreateParams; import org.opencb.opencga.core.models.file.FileStatus; -import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.models.user.Account; import org.opencb.opencga.core.testclassification.duration.MediumTests; import java.io.FileOutputStream; import java.io.IOException; -import java.net.URISyntaxException; import static org.junit.Assert.*; @@ -50,45 +39,47 @@ * Created by jacobo on 28/01/15. */ @Category(MediumTests.class) -public class CatalogFileUtilsTest { - - @Rule - public ExpectedException thrown = ExpectedException.none(); +public class CatalogFileUtilsTest extends AbstractManagerTest { +// +// @Rule +// public ExpectedException thrown = ExpectedException.none(); FileUtils catalogFileUtils; - private long studyUid; - private String studyFqn; - private String userSessionId; - // private String adminSessionId; - private CatalogManager catalogManager; - - private static final QueryOptions INCLUDE_RESULT = new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true); +// private long studyUid; +// private String studyFqn; +// private String ownerToken; +// // private String adminSessionId; +// private CatalogManager catalogManager; +// private String organizationId = "test"; +// +// private static final QueryOptions INCLUDE_RESULT = new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true); @Before - public void before() throws CatalogException, IOException, URISyntaxException { - Configuration configuration = Configuration.load(getClass().getResource("/configuration-test.yml") - .openStream()); - MongoDBConfiguration mongoDBConfiguration = MongoDBConfiguration.builder() - .add("username", configuration.getCatalog().getDatabase().getUser()) - .add("password", configuration.getCatalog().getDatabase().getPassword()) - .add("authenticationDatabase", configuration.getCatalog().getDatabase().getOptions().get("authenticationDatabase")) - .build(); - - CatalogManagerExternalResource.clearCatalog(configuration); - catalogManager = new CatalogManager(configuration); - catalogManager.installCatalogDB(configuration.getAdmin().getSecretKey(), TestParamConstants.ADMIN_PASSWORD, "opencga@admin.com", "", true, true); - - String opencgaToken = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); - - //Create USER - catalogManager.getUserManager().create("user", "name", "mi@mail.com", TestParamConstants.PASSWORD, "", null, Account.AccountType.FULL, opencgaToken); - userSessionId = catalogManager.getUserManager().login("user", TestParamConstants.PASSWORD).getToken(); -// adminSessionId = catalogManager.login("admin", "admin", "--").getResults().get(0).getString("sessionId"); - String projectId = catalogManager.getProjectManager().create("proj", "proj", "", "Homo sapiens", - null, "GRCh38", INCLUDE_RESULT, userSessionId).getResults().get(0).getId(); - Study study = catalogManager.getStudyManager().create(projectId, "std", "std", "std", "", null, null, null, null, INCLUDE_RESULT, - userSessionId).getResults().get(0); - studyUid = study.getUid(); - studyFqn = study.getFqn(); + public void before() throws Exception { + setUp(); +// Configuration configuration = Configuration.load(getClass().getResource("/configuration-test.yml") +// .openStream()); +// MongoDBConfiguration mongoDBConfiguration = MongoDBConfiguration.builder() +// .add("username", configuration.getCatalog().getDatabase().getUser()) +// .add("password", configuration.getCatalog().getDatabase().getPassword()) +// .add("authenticationDatabase", configuration.getCatalog().getDatabase().getOptions().get("authenticationDatabase")) +// .build(); +// +// CatalogManagerExternalResource.clearCatalog(configuration); +// catalogManager = new CatalogManager(configuration); +// catalogManager.installCatalogDB("HS256", configuration.getAdmin().getSecretKey(), TestParamConstants.ADMIN_PASSWORD, "opencga@admin.com", true); +// +// String opencgaToken = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); +// +// //Create USER +// catalogManager.getUserManager().create(organizationId, "user", "name", "mi@mail.com", TestParamConstants.PASSWORD, "", null, Account.AccountType.FULL, opencgaToken); +// ownerToken = catalogManager.getUserManager().login(organizationId, "user", TestParamConstants.PASSWORD).getToken(); +//// adminSessionId = catalogManager.login("admin", "admin", "--").getResults().get(0).getString("sessionId"); +// String projectId = catalogManager.getProjectManager().create(organizationId, "proj", "proj", "", "Homo sapiens", +// null, "GRCh38", INCLUDE_RESULT, ownerToken).getResults().get(0).getId(); +// Study study = catalogManager.getStudyManager().create(projectId, "std", "std", "std", "", null, null, null, null, INCLUDE_RESULT, +// ownerToken).getResults().get(0); +// studyUid = study.getUid(); +// studyFqn = study.getFqn(); catalogFileUtils = new FileUtils(catalogManager); } @@ -106,31 +97,31 @@ public void checkFileTest() throws CatalogException, IOException { .setPath("item." + TimeUtils.getTimeMillis() + ".txt") .setDescription("file at root") .setContent(RandomStringUtils.randomAlphanumeric(100)), - true, userSessionId).first(); - returnedFile = catalogFileUtils.checkFile(studyFqn, file, true, userSessionId); + true, ownerToken).first(); + returnedFile = catalogFileUtils.checkFile(studyFqn, file, true, ownerToken); assertSame("Should not modify the status, so should return the same file.", file, returnedFile); assertEquals(InternalStatus.READY, file.getInternal().getStatus().getId()); // /** Check READY and existing file **/ -// catalogFileUtils.upload(sourceUri, file, null, userSessionId, false, false, false, true); +// catalogFileUtils.upload(sourceUri, file, null, ownerToken, false, false, false, true); // fileUri = catalogManager.getFileManager().getUri(file); -// file = catalogManager.getFileManager().get(studyFqn, file.getPath(), null, userSessionId).first(); -// returnedFile = catalogFileUtils.checkFile(studyFqn, file, true, userSessionId); +// file = catalogManager.getFileManager().get(studyFqn, file.getPath(), null, ownerToken).first(); +// returnedFile = catalogFileUtils.checkFile(studyFqn, file, true, ownerToken); // // assertSame("Should not modify the READY and existing file, so should return the same file.", file, returnedFile); /** Check READY and missing file **/ assertTrue(new java.io.File(file.getUri()).delete()); - returnedFile = catalogFileUtils.checkFile(studyFqn, file, true, userSessionId); + returnedFile = catalogFileUtils.checkFile(studyFqn, file, true, ownerToken); assertNotSame(file, returnedFile); assertEquals(FileStatus.MISSING, returnedFile.getInternal().getStatus().getId()); /** Check MISSING file still missing **/ - file = catalogManager.getFileManager().get(studyFqn, file.getPath(), null, userSessionId).first(); - returnedFile = catalogFileUtils.checkFile(studyFqn, file, true, userSessionId); + file = catalogManager.getFileManager().get(studyFqn, file.getPath(), null, ownerToken).first(); + returnedFile = catalogFileUtils.checkFile(studyFqn, file, true, ownerToken); assertEquals("Should not modify the still MISSING file, so should return the same file.", file.getInternal().getStatus().getId(), returnedFile.getInternal().getStatus().getId()); @@ -141,7 +132,7 @@ public void checkFileTest() throws CatalogException, IOException { os.write(RandomStringUtils.randomAlphanumeric(1000).getBytes()); os.write('\n'); os.close(); - returnedFile = catalogFileUtils.checkFile(studyFqn, file, true, userSessionId); + returnedFile = catalogFileUtils.checkFile(studyFqn, file, true, ownerToken); assertNotSame(file, returnedFile); assertEquals(FileStatus.READY, returnedFile.getInternal().getStatus().getId()); @@ -150,26 +141,26 @@ public void checkFileTest() throws CatalogException, IOException { // FileUpdateParams updateParams = new FileUpdateParams() // .setStatus(new File.FileStatus(File.FileStatus.PENDING_DELETE)); // catalogManager.getFileManager().update(studyFqn, new Query(FileDBAdaptor.QueryParams.UID.key(), file.getUid()), updateParams, -// QueryOptions.empty(), userSessionId); +// QueryOptions.empty(), ownerToken); // catalogManager.getFileManager().delete(studyFqn, new Query(FileDBAdaptor.QueryParams.UID.key(), file.getUid()), null, -// userSessionId); +// ownerToken); // // Query query = new Query() // .append(FileDBAdaptor.QueryParams.UID.key(), file.getUid()); -// DataResult fileDataResult = catalogManager.getFileManager().search(studyFqn, query, QueryOptions.empty(), userSessionId); +// DataResult fileDataResult = catalogManager.getFileManager().search(studyFqn, query, QueryOptions.empty(), ownerToken); // // file = fileDataResult.first(); -// returnedFile = catalogFileUtils.checkFile(studyFqn, file, true, userSessionId); +// returnedFile = catalogFileUtils.checkFile(studyFqn, file, true, ownerToken); // // assertSame(file, returnedFile); // assertEquals(File.FileStatus.TRASHED, returnedFile.getStatus().getName()); // // // /** Check TRASHED file with missing file **/ -//// catalogManager.getFileManager().delete(Long.toString(file.getId()), null, userSessionId); +//// catalogManager.getFileManager().delete(Long.toString(file.getId()), null, ownerToken); // assertTrue(Paths.get(file.getUri()).toFile().delete()); // -// returnedFile = catalogFileUtils.checkFile(studyFqn, file, true, userSessionId); +// returnedFile = catalogFileUtils.checkFile(studyFqn, file, true, ownerToken); // //// assertNotSame(file, returnedFile); // assertEquals(File.FileStatus.TRASHED, returnedFile.getStatus().getName()); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/CatalogSampleAnnotationsLoaderTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/CatalogSampleAnnotationsLoaderTest.java index 336d60295c5..5a41a80ee29 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/CatalogSampleAnnotationsLoaderTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/CatalogSampleAnnotationsLoaderTest.java @@ -18,7 +18,6 @@ import org.junit.Assert; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; import org.opencb.biodata.models.pedigree.Individual; @@ -26,20 +25,13 @@ import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.commons.test.GenericTest; -import org.opencb.opencga.TestParamConstants; import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.managers.CatalogManager; -import org.opencb.opencga.catalog.managers.CatalogManagerExternalResource; -import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.catalog.managers.AbstractManagerTest; import org.opencb.opencga.core.models.common.AnnotationSet; import org.opencb.opencga.core.models.file.File; -import org.opencb.opencga.core.models.project.Project; import org.opencb.opencga.core.models.sample.Sample; -import org.opencb.opencga.core.models.study.Study; import org.opencb.opencga.core.models.study.Variable; import org.opencb.opencga.core.models.study.VariableSet; -import org.opencb.opencga.core.models.user.Account; import org.opencb.opencga.core.testclassification.duration.MediumTests; import java.io.FileInputStream; @@ -49,44 +41,28 @@ import java.util.*; @Category(MediumTests.class) -public class CatalogSampleAnnotationsLoaderTest extends GenericTest { - - @Rule - public CatalogManagerExternalResource catalogManagerResource = new CatalogManagerExternalResource(); +public class CatalogSampleAnnotationsLoaderTest extends AbstractManagerTest { private static final List populations = Arrays.asList("ACB", "ASW", "BEB", "CDX", "CEU", "CHB", "CHS", "CLM", "ESN", "FIN", "GBR", "GIH", "GWD", "IBS", "ITU", "JPT", "KHV", "LWK", "MSL", "MXL", "PEL", "PJL", "PUR", "STU", "TSI", "YRI"); private static CatalogSampleAnnotationsLoader loader; private static Pedigree pedigree; - private static String sessionId; private static File pedFile; - private static CatalogManager catalogManager; - private static String userId; - private static String studyId; @Before - public void setup() throws IOException, CatalogException, URISyntaxException { - catalogManager = catalogManagerResource.getCatalogManager(); - setUpCatalogManager(catalogManager); + public void setup() throws Exception { + super.setUp(); + setUpCatalogManager(); } - public void setUpCatalogManager(CatalogManager catalogManager) throws IOException, CatalogException, URISyntaxException { + public void setUpCatalogManager() throws IOException, CatalogException, URISyntaxException { loader = new CatalogSampleAnnotationsLoader(catalogManager); String pedFileName = "20130606_g1k.ped"; URL pedFileURL = CatalogSampleAnnotationsLoader.class.getClassLoader().getResource(pedFileName); pedigree = loader.readPedigree(pedFileURL.getPath()); - - userId = "user1"; - catalogManager.getUserManager().create(userId, userId, "asdasd@asd.asd", TestParamConstants.PASSWORD, "", -1L, Account.AccountType.FULL, catalogManagerResource.getAdminToken()); - sessionId = catalogManager.getUserManager().login(userId, TestParamConstants.PASSWORD).getToken(); - Project project = catalogManager.getProjectManager().create("def", "default", "", "Homo sapiens", - null, "GRCh38", new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true), sessionId).getResults().get(0); - Study study = catalogManager.getStudyManager().create(project.getFqn(), "def", null, "default", "", null, null, null, null, - new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true), sessionId).getResults().get(0); - studyId = study.getFqn(); - pedFile = catalogManager.getFileManager().upload(studyId, new FileInputStream(new java.io.File(pedFileURL.toURI())), - new File().setPath("data/" + pedFileName), false, true, false, sessionId).first(); + pedFile = catalogManager.getFileManager().upload(studyFqn, new FileInputStream(new java.io.File(pedFileURL.toURI())), + new File().setPath("data/" + pedFileName), false, true, false, ownerToken).first(); } @Test @@ -102,11 +78,11 @@ public void testLoadPedigree_GeneratedVariableSet() throws Exception { @Test public void testLoadPedigree_GivenVariableSet() throws Exception { HashSet variables = new HashSet<>(); - variables.add(new Variable("id", "", Variable.VariableType.DOUBLE, null, true, false, Collections.emptyList(), null, 0, null, + variables.add(new Variable("id", "", Variable.VariableType.DOUBLE, null, true, false, Collections.emptyList(), null, 0, null, "", null, null)); - variables.add(new Variable("name", "", Variable.VariableType.STRING, null, true, false, Collections.emptyList(), null, 0, null, + variables.add(new Variable("name", "", Variable.VariableType.STRING, null, true, false, Collections.emptyList(), null, 0, null, "", null, null)); - variables.add(new Variable("fatherId", "", Variable.VariableType.DOUBLE, null, false, false, Collections.emptyList(), null, 0, + variables.add(new Variable("fatherId", "", Variable.VariableType.DOUBLE, null, false, false, Collections.emptyList(), null, 0, null, "", null, null)); variables.add(new Variable("Population", "", Variable.VariableType.CATEGORICAL, null, true, false, populations, null, 0, null, "", null, null)); @@ -121,7 +97,7 @@ public void testLoadPedigree_GivenVariableSet() throws Exception { @Test public void testLoadPedigreeCatalog() throws Exception { - DataResult sampleDataResult = loader.loadSampleAnnotations(pedFile, null, sessionId); + DataResult sampleDataResult = loader.loadSampleAnnotations(studyFqn, pedFile, null, ownerToken); String variableSetId = sampleDataResult.getResults().get(0).getAnnotationSets().get(0).getVariableSetId(); Query query = new Query(Constants.ANNOTATION, variableSetId + ":family=GB84"); @@ -129,19 +105,19 @@ public void testLoadPedigreeCatalog() throws Exception { .append(QueryOptions.LIMIT, 0) .append(QueryOptions.COUNT, true); - DataResult allSamples = catalogManager.getSampleManager().search(studyId, query, options, sessionId); + DataResult allSamples = catalogManager.getSampleManager().search(studyFqn, query, options, ownerToken); Assert.assertNotEquals(0, allSamples.getNumMatches()); query = new Query(Constants.ANNOTATION, variableSetId + ":sex=2;" + variableSetId + ":Population=ITU"); - DataResult femaleIta = catalogManager.getSampleManager().search(studyId, query, options, sessionId); + DataResult femaleIta = catalogManager.getSampleManager().search(studyFqn, query, options, ownerToken); Assert.assertNotEquals(0, femaleIta.getNumMatches()); query = new Query(Constants.ANNOTATION, variableSetId + ":sex=1;" + variableSetId + ":Population=ITU"); - DataResult maleIta = catalogManager.getSampleManager().search(studyId, query, options, sessionId); + DataResult maleIta = catalogManager.getSampleManager().search(studyFqn, query, options, ownerToken); Assert.assertNotEquals(0, maleIta.getNumMatches()); query = new Query(Constants.ANNOTATION, variableSetId + ":Population=ITU"); - DataResult ita = catalogManager.getSampleManager().search(studyId, query, options, sessionId); + DataResult ita = catalogManager.getSampleManager().search(studyFqn, query, options, ownerToken); Assert.assertNotEquals(0, ita.getNumMatches()); Assert.assertEquals("Fail sample query", ita.getNumMatches(), maleIta.getNumMatches() + femaleIta.getNumMatches()); @@ -151,8 +127,7 @@ public void testLoadPedigreeCatalog() throws Exception { private void validate(Pedigree pedigree, VariableSet variableSet) throws CatalogException { for (Map.Entry entry : pedigree.getIndividuals().entrySet()) { Map annotation = loader.getAnnotation(entry.getValue(), null, variableSet, pedigree.getFields()); - AnnotationUtils.checkAnnotationSet(variableSet, new AnnotationSet("", variableSet.getId(), annotation, "", 1, - null), null, true); + AnnotationUtils.checkAnnotationSet(variableSet, new AnnotationSet("", variableSet.getId(), annotation), null, true); } } } \ No newline at end of file diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/FileMetadataReaderTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/FileMetadataReaderTest.java index 8813e517553..e179b59ef5e 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/FileMetadataReaderTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/FileMetadataReaderTest.java @@ -18,28 +18,20 @@ import org.apache.commons.lang3.RandomStringUtils; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.TestParamConstants; import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.managers.CatalogManager; -import org.opencb.opencga.catalog.managers.CatalogManagerExternalResource; -import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.catalog.managers.AbstractManagerTest; import org.opencb.opencga.core.models.file.File; import org.opencb.opencga.core.models.file.FileCreateParams; import org.opencb.opencga.core.models.file.FileStatus; import org.opencb.opencga.core.models.file.FileUpdateParams; -import org.opencb.opencga.core.models.project.Project; import org.opencb.opencga.core.models.sample.Sample; -import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.models.user.Account; import org.opencb.opencga.core.testclassification.duration.MediumTests; import java.io.*; import java.net.URI; -import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -53,15 +45,16 @@ import static org.opencb.opencga.catalog.utils.FileMetadataReader.VARIANT_FILE_METADATA; @Category(MediumTests.class) -public class FileMetadataReaderTest { +public class FileMetadataReaderTest extends AbstractManagerTest { - @Rule - public CatalogManagerExternalResource catalogManagerExternalResource = new CatalogManagerExternalResource(); - - private CatalogManager catalogManager; - private String sessionIdUser; - private Project project; - private Study study; +// @Rule +// public CatalogManagerExternalResource catalogManagerResource = new CatalogManagerExternalResource(); +// +// private CatalogManager catalogManager; +// protected String organizationId = "zetta"; +// private String ownerToken; +// private Project project; +// private Study study; private File folder; private URI vcfFileUri; public static final String VCF_FILE_NAME = "variant-test-file.vcf.gz"; @@ -69,27 +62,28 @@ public class FileMetadataReaderTest { private URI bamFileUri; private final List expectedSampleNames = Arrays.asList("NA19600", "NA19660", "NA19661", "NA19685"); - protected static final QueryOptions INCLUDE_RESULT = new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true); +// protected static final QueryOptions INCLUDE_RESULT = new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true); @Before - public void setUp() throws IOException, CatalogException, URISyntaxException { - catalogManager = catalogManagerExternalResource.getCatalogManager(); - - String opencgaToken = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); - - catalogManager.getUserManager().create("user", "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, "", null, Account.AccountType.FULL, opencgaToken); - sessionIdUser = catalogManager.getUserManager().login("user", TestParamConstants.PASSWORD).getToken(); - project = catalogManager.getProjectManager().create("1000G", "Project about some genomes", "", "Homo sapiens", - null, "GRCh38", INCLUDE_RESULT, sessionIdUser).first(); - study = catalogManager.getStudyManager().create(project.getId(), "phase1", null, "Phase 1", "Done", null, null, null, null, INCLUDE_RESULT, sessionIdUser).first(); - folder = catalogManager.getFileManager().createFolder(study.getId(), Paths.get("data/vcf/").toString(), true, - null, QueryOptions.empty(), sessionIdUser).first(); - - Path vcfPath = catalogManagerExternalResource.getOpencgaHome().resolve(VCF_FILE_NAME); + public void setUp() throws Exception { + super.setUp(); +// catalogManager = catalogManagerResource.getCatalogManager(); +// +// String opencgaToken = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); +// +// catalogManager.getUserManager().create(organizationId, "user", "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, "", null, Account.AccountType.FULL, opencgaToken); +// ownerToken = catalogManager.getUserManager().login(organizationId, "user", TestParamConstants.PASSWORD).getToken(); +// project = catalogManager.getProjectManager().create(organizationId, "1000G", "Project about some genomes", "", "Homo sapiens", +// null, "GRCh38", INCLUDE_RESULT, ownerToken).first(); +// study = catalogManager.getStudyManager().create(project.getId(), "phase1", null, "Phase 1", "Done", null, null, null, null, INCLUDE_RESULT, ownerToken).first(); + folder = catalogManager.getFileManager().createFolder(studyFqn, Paths.get("data/vcf/").toString(), true, + null, QueryOptions.empty(), ownerToken).first(); + + Path vcfPath = catalogManagerResource.getOpencgaHome().resolve(VCF_FILE_NAME); Files.copy(this.getClass().getClassLoader().getResourceAsStream("biofiles/" + VCF_FILE_NAME), vcfPath, StandardCopyOption.REPLACE_EXISTING); vcfFileUri = vcfPath.toUri(); - Path bamPath = catalogManagerExternalResource.getOpencgaHome().resolve(BAM_FILE_NAME); + Path bamPath = catalogManagerResource.getOpencgaHome().resolve(BAM_FILE_NAME); Files.copy(this.getClass().getClassLoader().getResourceAsStream("biofiles/" + BAM_FILE_NAME), bamPath, StandardCopyOption.REPLACE_EXISTING); bamFileUri = bamPath.toUri(); @@ -97,16 +91,16 @@ public void setUp() throws IOException, CatalogException, URISyntaxException { @Test public void testGetBasicMetadata() throws CatalogException, IOException { - File file = catalogManager.getFileManager().create(study.getFqn(), + File file = catalogManager.getFileManager().create(studyFqn, new FileCreateParams() .setContent(RandomStringUtils.randomAlphanumeric(1000)) .setType(File.Type.FILE) .setPath(folder.getPath() + "test.txt"), - false, sessionIdUser).first(); + false, ownerToken).first(); assertEquals(1000, file.getSize()); - URI fileUri = catalogManager.getFileManager().getUri(file); + URI fileUri = catalogManager.getFileManager().getUri(organizationId, file); try { Thread.sleep(1000); //Sleep 1 second to see changes on the "modificationDate" @@ -119,7 +113,7 @@ public void testGetBasicMetadata() throws CatalogException, IOException { outputStream.close(); - FileMetadataReader.get(catalogManager).addMetadataInformation(study.getFqn(), file); + FileMetadataReader.get(catalogManager).addMetadataInformation(studyFqn, file); assertEquals(1000 + bytes2.length, file.getSize()); } @@ -128,13 +122,13 @@ public void testGetBasicMetadata() throws CatalogException, IOException { public void testGetMetadataFromVcf() throws CatalogException, IOException { File file; try (InputStream inputStream = new BufferedInputStream(new FileInputStream(new java.io.File(vcfFileUri)))) { - file = catalogManager.getFileManager().upload(study.getFqn(), inputStream, - new File().setPath(folder.getPath() + VCF_FILE_NAME), false, false, false, sessionIdUser).first(); + file = catalogManager.getFileManager().upload(studyFqn, inputStream, + new File().setPath(folder.getPath() + VCF_FILE_NAME), false, false, false, ownerToken).first(); } assertTrue(file.getSize() > 0); - FileMetadataReader.get(catalogManager).addMetadataInformation(study.getFqn(), file); + FileMetadataReader.get(catalogManager).addMetadataInformation(studyFqn, file); assertEquals(FileStatus.READY, file.getInternal().getStatus().getId()); assertEquals(File.Format.VCF, file.getFormat()); @@ -142,8 +136,8 @@ public void testGetMetadataFromVcf() throws CatalogException, IOException { assertNotNull(file.getAttributes().get(VARIANT_FILE_METADATA)); assertEquals(4, file.getSampleIds().size()); assertEquals(expectedSampleNames, ((Map) file.getAttributes().get(VARIANT_FILE_METADATA)).get("sampleIds")); -// catalogManager.getSampleManager().search(study.getFqn(), new Query(SampleDBAdaptor.QueryParams.ID.key(), -// file.getSamples().stream().map(Sample::getId).collect(Collectors.toList())), new QueryOptions(), sessionIdUser).getResults(); +// catalogManager.getSampleManager().search(studyFqn, new Query(SampleDBAdaptor.QueryParams.ID.key(), +// file.getSamples().stream().map(Sample::getId).collect(Collectors.toList())), new QueryOptions(), ownerToken).getResults(); // // assertTrue(expectedSampleNames.containsAll(file.getSamples().stream().map(Sample::getId).collect(Collectors.toSet()))); } @@ -153,7 +147,7 @@ public void testGetMetadataFromVcf() throws CatalogException, IOException { public void testGetMetadataFromVcfWithAlreadyExistingSamples() throws CatalogException, IOException { //Create the samples in the same order than in the file for (String sampleName : expectedSampleNames) { - catalogManager.getSampleManager().create(study.getFqn(), new Sample().setId(sampleName), new QueryOptions(), sessionIdUser); + catalogManager.getSampleManager().create(studyFqn, new Sample().setId(sampleName), new QueryOptions(), ownerToken); } testGetMetadataFromVcf(); } @@ -161,21 +155,21 @@ public void testGetMetadataFromVcfWithAlreadyExistingSamples() throws CatalogExc @Test public void testGetMetadataFromVcfWithAlreadyExistingSamplesUnsorted() throws CatalogException, IOException { //Create samples in a different order than the file order - catalogManager.getSampleManager().create(study.getFqn(), new Sample().setId(expectedSampleNames.get(2)), new QueryOptions(), sessionIdUser); + catalogManager.getSampleManager().create(studyFqn, new Sample().setId(expectedSampleNames.get(2)), new QueryOptions(), ownerToken); - catalogManager.getSampleManager().create(study.getFqn(), new Sample().setId(expectedSampleNames.get(0)), new QueryOptions(), sessionIdUser); - catalogManager.getSampleManager().create(study.getFqn(), new Sample().setId(expectedSampleNames.get(3)), new QueryOptions(), sessionIdUser); - catalogManager.getSampleManager().create(study.getFqn(), new Sample().setId(expectedSampleNames.get(1)), new QueryOptions(), sessionIdUser); + catalogManager.getSampleManager().create(studyFqn, new Sample().setId(expectedSampleNames.get(0)), new QueryOptions(), ownerToken); + catalogManager.getSampleManager().create(studyFqn, new Sample().setId(expectedSampleNames.get(3)), new QueryOptions(), ownerToken); + catalogManager.getSampleManager().create(studyFqn, new Sample().setId(expectedSampleNames.get(1)), new QueryOptions(), ownerToken); testGetMetadataFromVcf(); } @Test public void testGetMetadataFromVcfWithSomeExistingSamples() throws CatalogException, IOException { - catalogManager.getSampleManager().create(study.getFqn(), new Sample().setId(expectedSampleNames.get(2)), new QueryOptions(), - sessionIdUser); - catalogManager.getSampleManager().create(study.getFqn(), new Sample().setId(expectedSampleNames.get(0)), new QueryOptions(), - sessionIdUser); + catalogManager.getSampleManager().create(studyFqn, new Sample().setId(expectedSampleNames.get(2)), new QueryOptions(), + ownerToken); + catalogManager.getSampleManager().create(studyFqn, new Sample().setId(expectedSampleNames.get(0)), new QueryOptions(), + ownerToken); testGetMetadataFromVcf(); } @@ -184,8 +178,8 @@ public void testGetMetadataFromVcfWithSomeExistingSamples() throws CatalogExcept public void testDoNotOverwriteSampleIds() throws CatalogException, IOException { File file; try (InputStream inputStream = new BufferedInputStream(new FileInputStream(new java.io.File(vcfFileUri)))) { - file = catalogManager.getFileManager().upload(study.getFqn(), inputStream, - new File().setPath(folder.getPath() + VCF_FILE_NAME), false, false, false, sessionIdUser).first(); + file = catalogManager.getFileManager().upload(studyFqn, inputStream, + new File().setPath(folder.getPath() + VCF_FILE_NAME), false, false, false, ownerToken).first(); } assertEquals(FileStatus.READY, file.getInternal().getStatus().getId()); assertEquals(File.Format.VCF, file.getFormat()); @@ -194,12 +188,12 @@ public void testDoNotOverwriteSampleIds() throws CatalogException, IOException { assertEquals(4, file.getSampleIds().size()); //Add a sampleId - String sampleId = catalogManager.getSampleManager().create(study.getFqn(), new Sample().setId("Bad_Sample"), INCLUDE_RESULT, sessionIdUser) + String sampleId = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("Bad_Sample"), INCLUDE_RESULT, ownerToken) .first().getId(); - catalogManager.getFileManager().update(study.getFqn(), file.getPath(), - new FileUpdateParams().setSampleIds(Collections.singletonList(sampleId)), new QueryOptions(), sessionIdUser); + catalogManager.getFileManager().update(studyFqn, file.getPath(), + new FileUpdateParams().setSampleIds(Collections.singletonList(sampleId)), new QueryOptions(), ownerToken); - file = catalogManager.getFileManager().get(study.getFqn(), file.getPath(), null, sessionIdUser).first(); + file = catalogManager.getFileManager().get(studyFqn, file.getPath(), null, ownerToken).first(); assertEquals(5, file.getSampleIds().size()); assertEquals(sampleId, file.getSampleIds().get(4)); } @@ -208,21 +202,21 @@ public void testDoNotOverwriteSampleIds() throws CatalogException, IOException { public void testGetMetadataFromBam() throws CatalogException, IOException { File file; try (InputStream inputStream = new BufferedInputStream(new FileInputStream(new java.io.File(bamFileUri)))) { - file = catalogManager.getFileManager().upload(study.getFqn(), inputStream, - new File().setPath(folder.getPath() + BAM_FILE_NAME), false, false, false, sessionIdUser).first(); + file = catalogManager.getFileManager().upload(studyFqn, inputStream, + new File().setPath(folder.getPath() + BAM_FILE_NAME), false, false, false, ownerToken).first(); } assertTrue(file.getSize() > 0); - FileMetadataReader.get(catalogManager).addMetadataInformation(study.getFqn(), file); + FileMetadataReader.get(catalogManager).addMetadataInformation(studyFqn, file); assertEquals(FileStatus.READY, file.getInternal().getStatus().getId()); // assertEquals(File.Format.GZIP, file.getFormat()); assertEquals(File.Bioformat.ALIGNMENT, file.getBioformat()); assertNotNull(file.getAttributes().get("alignmentHeader")); assertEquals(1, file.getSampleIds().size()); -// assertEquals("HG00096", catalogManager.getSampleManager().get(study.getFqn(), file.getSamples().get(0).getId(), null, -// sessionIdUser).first().getId()); +// assertEquals("HG00096", catalogManager.getSampleManager().get(studyFqn, file.getSamples().get(0).getId(), null, +// ownerToken).first().getId()); } diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/FileScannerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/FileScannerTest.java index b09c365b86d..d026e53ccef 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/FileScannerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/FileScannerTest.java @@ -18,26 +18,20 @@ import org.apache.commons.lang3.StringUtils; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.TestParamConstants; import org.opencb.opencga.catalog.db.api.FileDBAdaptor; +import org.opencb.opencga.catalog.db.mongodb.MongoBackupUtils; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.io.IOManager; -import org.opencb.opencga.catalog.managers.CatalogManager; -import org.opencb.opencga.catalog.managers.CatalogManagerExternalResource; -import org.opencb.opencga.catalog.managers.CatalogManagerTest; -import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.catalog.managers.AbstractManagerTest; import org.opencb.opencga.core.common.IOUtils; import org.opencb.opencga.core.models.file.File; import org.opencb.opencga.core.models.file.FileStatus; -import org.opencb.opencga.core.models.project.Project; import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.models.user.Account; import org.opencb.opencga.core.testclassification.duration.MediumTests; import java.io.BufferedInputStream; @@ -58,34 +52,38 @@ import static org.junit.Assert.*; @Category(MediumTests.class) -public class FileScannerTest { - @Rule - public CatalogManagerExternalResource catalogManagerExternalResource = new CatalogManagerExternalResource(); - - private CatalogManager catalogManager; - private String sessionIdUser; +public class FileScannerTest extends AbstractManagerTest { +// @Rule +// public CatalogManagerExternalResource catalogManagerExternalResource = new CatalogManagerExternalResource(); +// +// private CatalogManager catalogManager; +// private String organizationId = "test"; +// private String ownerToken; private File folder; private Study study; - private Project project; +// private Project project; private Path directory; - protected static final QueryOptions INCLUDE_RESULT = new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true); +// protected static final QueryOptions INCLUDE_RESULT = new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true); @Before - public void setUp() throws IOException, CatalogException { - catalogManager = catalogManagerExternalResource.getCatalogManager(); - - String opencgaToken = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); - - catalogManager.getUserManager().create("user", "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, "", null, Account.AccountType.FULL, opencgaToken); - sessionIdUser = catalogManager.getUserManager().login("user", TestParamConstants.PASSWORD).getToken(); - project = catalogManager.getProjectManager().create("1000G", "Project about some genomes", "", "Homo sapiens", - null, "GRCh38", INCLUDE_RESULT, sessionIdUser).first(); - study = catalogManager.getStudyManager().create(project.getId(), "phase1", null, "Phase 1", "Done", null, null, null, null, INCLUDE_RESULT, sessionIdUser).first(); - folder = catalogManager.getFileManager().createFolder(study.getId(), Paths.get("data/test/folder/").toString(), - true, null, QueryOptions.empty(), sessionIdUser).first(); - - directory = catalogManagerExternalResource.getOpencgaHome().resolve("catalog_scan_test_folder").toAbsolutePath(); + public void setUp() throws Exception { + super.setUp(); +// catalogManager = catalogManagerExternalResource.getCatalogManager(); +// +// String opencgaToken = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); +// +// catalogManager.getUserManager().create(organizationId, "user", "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, "", null, Account.AccountType.FULL, opencgaToken); +// ownerToken = catalogManager.getUserManager().login(organizationId, "user", TestParamConstants.PASSWORD).getToken(); +// project = catalogManager.getProjectManager().create(organizationId, "1000G", "Project about some genomes", "", "Homo sapiens", +// null, "GRCh38", INCLUDE_RESULT, ownerToken).first(); +// study = catalogManager.getStudyManager().create(project.getId(), "phase1", null, "Phase 1", "Done", null, null, null, null, INCLUDE_RESULT, ownerToken).first(); +// folder = catalogManager.getFileManager().createFolder(study.getId(), Paths.get("data/test/folder/").toString(), +// true, null, QueryOptions.empty(), ownerToken).first(); + + study = catalogManager.getStudyManager().get(studyFqn, QueryOptions.empty(), ownerToken).first(); + folder = catalogManager.getFileManager().get(studyFqn, "data/test/folder/", QueryOptions.empty(), ownerToken).first(); + directory = catalogManagerResource.getOpencgaHome().resolve("catalog_scan_test_folder").toAbsolutePath(); if (directory.toFile().exists()) { IOUtils.deleteDirectory(directory); } @@ -97,18 +95,18 @@ public void testScan() throws IOException, CatalogException { Files.createDirectory(directory.resolve("subfolder")); Files.createDirectory(directory.resolve("subfolder/subsubfolder")); - CatalogManagerTest.createDebugFile(directory.resolve("file1.txt").toString()); - CatalogManagerTest.createDebugFile(directory.resolve("file2.txt").toString()); - CatalogManagerTest.createDebugFile(directory.resolve("file3.txt").toString()); - CatalogManagerTest.createDebugFile(directory.resolve("subfolder/file1.txt").toString()); - CatalogManagerTest.createDebugFile(directory.resolve("subfolder/file2.txt").toString()); - CatalogManagerTest.createDebugFile(directory.resolve("subfolder/file3.txt").toString()); - CatalogManagerTest.createDebugFile(directory.resolve("subfolder/subsubfolder/file1.txt").toString()); - CatalogManagerTest.createDebugFile(directory.resolve("subfolder/subsubfolder/file2.txt").toString()); - CatalogManagerTest.createDebugFile(directory.resolve("subfolder/subsubfolder/file3.txt").toString()); + MongoBackupUtils.createDebugFile(directory.resolve("file1.txt").toString()); + MongoBackupUtils.createDebugFile(directory.resolve("file2.txt").toString()); + MongoBackupUtils.createDebugFile(directory.resolve("file3.txt").toString()); + MongoBackupUtils.createDebugFile(directory.resolve("subfolder/file1.txt").toString()); + MongoBackupUtils.createDebugFile(directory.resolve("subfolder/file2.txt").toString()); + MongoBackupUtils.createDebugFile(directory.resolve("subfolder/file3.txt").toString()); + MongoBackupUtils.createDebugFile(directory.resolve("subfolder/subsubfolder/file1.txt").toString()); + MongoBackupUtils.createDebugFile(directory.resolve("subfolder/subsubfolder/file2.txt").toString()); + MongoBackupUtils.createDebugFile(directory.resolve("subfolder/subsubfolder/file3.txt").toString()); FileScanner fileScanner = new FileScanner(catalogManager); - List files = fileScanner.scan(folder, directory.toUri(), FileScanner.FileScannerPolicy.DELETE, true, true, sessionIdUser); + List files = fileScanner.scan(organizationId, folder, directory.toUri(), FileScanner.FileScannerPolicy.DELETE, true, true, ownerToken); assertEquals(9, files.size()); files.forEach((File file) -> assertTrue(StringUtils.isNotEmpty(file.getChecksum()))); @@ -118,15 +116,15 @@ public void testScan() throws IOException, CatalogException { @Test public void testDeleteExisting() throws IOException, CatalogException { DataResult queryResult; - try (InputStream inputStream = new BufferedInputStream(new FileInputStream(CatalogManagerTest.createDebugFile()))) { - queryResult = catalogManager.getFileManager().upload(study.getFqn(), inputStream, - new File().setPath(folder.getPath() + "file1.txt"), false, false, false, sessionIdUser); + try (InputStream inputStream = new BufferedInputStream(new FileInputStream(MongoBackupUtils.createDebugFile()))) { + queryResult = catalogManager.getFileManager().upload(studyFqn, inputStream, + new File().setPath(folder.getPath() + "file1.txt"), false, false, false, ownerToken); } File file = queryResult.first(); - CatalogManagerTest.createDebugFile(directory.resolve("file1.txt").toString()); - List files = new FileScanner(catalogManager).scan(folder, directory.toUri(), FileScanner.FileScannerPolicy.DELETE, false, - true, sessionIdUser); + MongoBackupUtils.createDebugFile(directory.resolve("file1.txt").toString()); + List files = new FileScanner(catalogManager).scan(organizationId, folder, directory.toUri(), FileScanner.FileScannerPolicy.DELETE, false, + true, ownerToken); files.forEach((File f) -> assertFalse(f.getAttributes().containsKey("checksum"))); assertEquals(FileStatus.DELETED, getDeletedFile(file.getUid()).getInternal().getStatus().getId()); @@ -136,31 +134,31 @@ public File getDeletedFile(long id) throws CatalogException { Query query = new Query() .append(FileDBAdaptor.QueryParams.UID.key(), id) .append(FileDBAdaptor.QueryParams.DELETED.key(), true); - return catalogManager.getFileManager().search(study.getFqn(), query, null, sessionIdUser).first(); + return catalogManager.getFileManager().search(studyFqn, query, null, ownerToken).first(); } @Test public void testReplaceExisting() throws IOException, CatalogException { // Create and register file1.txt and s/file2.txt File file; - try (InputStream inputStream = new BufferedInputStream(new FileInputStream(CatalogManagerTest.createDebugFile()))) { - file = catalogManager.getFileManager().upload(study.getFqn(), inputStream, - new File().setPath(folder.getPath() + "file1.txt"), false, true, true, sessionIdUser).first(); + try (InputStream inputStream = new BufferedInputStream(new FileInputStream(MongoBackupUtils.createDebugFile()))) { + file = catalogManager.getFileManager().upload(studyFqn, inputStream, + new File().setPath(folder.getPath() + "file1.txt"), false, true, true, ownerToken).first(); } - try (InputStream inputStream = new BufferedInputStream(new FileInputStream(CatalogManagerTest.createDebugFile()))) { - catalogManager.getFileManager().upload(study.getFqn(), inputStream, - new File().setPath(folder.getPath() + "s/file2.txt"), false, true, true, sessionIdUser).first(); + try (InputStream inputStream = new BufferedInputStream(new FileInputStream(MongoBackupUtils.createDebugFile()))) { + catalogManager.getFileManager().upload(studyFqn, inputStream, + new File().setPath(folder.getPath() + "s/file2.txt"), false, true, true, ownerToken).first(); } // Create same file structure, and replace - CatalogManagerTest.createDebugFile(directory.resolve("file1.txt").toString()); + MongoBackupUtils.createDebugFile(directory.resolve("file1.txt").toString()); Files.createDirectory(directory.resolve("s/")); - CatalogManagerTest.createDebugFile(directory.resolve("s/file2.txt").toString()); + MongoBackupUtils.createDebugFile(directory.resolve("s/file2.txt").toString()); FileScanner fileScanner = new FileScanner(catalogManager); - fileScanner.scan(folder, directory.toUri(), FileScanner.FileScannerPolicy.REPLACE, true, true, sessionIdUser); + fileScanner.scan(organizationId, folder, directory.toUri(), FileScanner.FileScannerPolicy.REPLACE, true, true, ownerToken); - File replacedFile = catalogManager.getFileManager().get(study.getFqn(), file.getPath(), null, sessionIdUser).first(); + File replacedFile = catalogManager.getFileManager().get(studyFqn, file.getPath(), null, ownerToken).first(); assertEquals(FileStatus.READY, replacedFile.getInternal().getStatus().getId()); assertEquals(file.getUid(), replacedFile.getUid()); assertNotEquals(replacedFile.getChecksum(), file.getChecksum()); @@ -174,18 +172,18 @@ public void testRegisterFiles() throws IOException, CatalogException { Path folder = directory.resolve("s/"); Path file3 = directory.resolve("file3.txt"); - CatalogManagerTest.createDebugFile(file1.toString()); + MongoBackupUtils.createDebugFile(file1.toString()); Files.createDirectory(folder); - CatalogManagerTest.createDebugFile(file2.toString()); - CatalogManagerTest.createDebugFile(file3.toString()); + MongoBackupUtils.createDebugFile(file2.toString()); + MongoBackupUtils.createDebugFile(file3.toString()); List filePaths = new ArrayList<>(2); filePaths.add(file1); filePaths.add(file2); FileScanner fileScanner = new FileScanner(catalogManager); -// List files = fileScanner.registerFiles(this.folder, filePaths, FileScanner.FileScannerPolicy.DELETE, true, false, sessionIdUser); +// List files = fileScanner.registerFiles(this.folder, filePaths, FileScanner.FileScannerPolicy.DELETE, true, false, ownerToken); Predicate uriPredicate = uri -> uri.getPath().endsWith("file1.txt") || uri.getPath().endsWith("file2.txt"); - List files = fileScanner.scan(this.folder, directory.toUri(), FileScanner.FileScannerPolicy.DELETE, true, false, uriPredicate, sessionIdUser); + List files = fileScanner.scan(organizationId, this.folder, directory.toUri(), FileScanner.FileScannerPolicy.DELETE, true, false, uriPredicate, ownerToken); assertEquals(2, files.size()); for (File file : files) { @@ -201,16 +199,16 @@ public void testRegisterFiles() throws IOException, CatalogException { @Test public void testScanStudyURI() throws IOException, CatalogException { - CatalogManagerTest.createDebugFile(directory.resolve("file1.txt").toString()); + MongoBackupUtils.createDebugFile(directory.resolve("file1.txt").toString()); FileScanner fileScanner = new FileScanner(catalogManager); - List files = fileScanner.scan(folder, directory.toUri(), FileScanner.FileScannerPolicy.REPLACE, true, true, sessionIdUser); + List files = fileScanner.scan(organizationId, folder, directory.toUri(), FileScanner.FileScannerPolicy.REPLACE, true, true, ownerToken); assertEquals(1, files.size()); Path studyUriPath = Paths.get(study.getUri()); - CatalogManagerTest.createDebugFile(studyUriPath.resolve("data/test/folder/").resolve("file2.txt").toString()); - File root = catalogManager.getFileManager().search(study.getFqn(), new Query("name", "."), null, sessionIdUser).first(); - files = fileScanner.scan(root, studyUriPath.toUri(), FileScanner.FileScannerPolicy.REPLACE, true, true, sessionIdUser); + MongoBackupUtils.createDebugFile(studyUriPath.resolve("data/test/folder/").resolve("file2.txt").toString()); + File root = catalogManager.getFileManager().search(studyFqn, new Query("name", "."), null, ownerToken).first(); + files = fileScanner.scan(organizationId, root, studyUriPath.toUri(), FileScanner.FileScannerPolicy.REPLACE, true, true, ownerToken); assertEquals(1, files.size()); files.forEach((f) -> assertTrue(f.getSize() > 0)); @@ -220,21 +218,21 @@ public void testScanStudyURI() throws IOException, CatalogException { @Test public void testResyncStudy() throws IOException, CatalogException { - CatalogManagerTest.createDebugFile(directory.resolve("file1.txt").toString()); + MongoBackupUtils.createDebugFile(directory.resolve("file1.txt").toString()); //ReSync study folder. Will detect any difference. FileScanner fileScanner = new FileScanner(catalogManager); List files; - files = fileScanner.reSync(study, true, sessionIdUser); + files = fileScanner.reSync(organizationId, study, true, ownerToken); assertEquals(0, files.size()); //Add one extra file. ReSync study folder. Path studyUriPath = Paths.get(study.getUri()); // Create the directories catalogManager.getIoManagerFactory().getDefault().createDirectory(studyUriPath.resolve("data/test/folder/").toUri(), true); - Path filePath = CatalogManagerTest + Path filePath = MongoBackupUtils .createDebugFile(studyUriPath.resolve("data/test/folder/").resolve("file_scanner_test_file.txt").toString()).toPath(); - files = fileScanner.reSync(study, true, sessionIdUser); + files = fileScanner.reSync(organizationId, study, true, ownerToken); assertEquals(1, files.size()); File file = files.get(0); @@ -244,15 +242,15 @@ public void testResyncStudy() throws IOException, CatalogException { //Delete file. CheckStudyFiles. Will detect one File.Status.MISSING file Files.delete(filePath); - files = fileScanner.checkStudyFiles(study, true, sessionIdUser); + files = fileScanner.checkStudyFiles(organizationId, study, true, ownerToken); assertEquals(1, files.size()); assertEquals(FileStatus.MISSING, files.get(0).getInternal().getStatus().getId()); String originalChecksum = files.get(0).getChecksum(); //Restore file. CheckStudyFiles. Will detect one re-tracked file. Checksum must be different. - CatalogManagerTest.createDebugFile(filePath.toString()); - files = fileScanner.checkStudyFiles(study, true, sessionIdUser); + MongoBackupUtils.createDebugFile(filePath.toString()); + files = fileScanner.checkStudyFiles(organizationId, study, true, ownerToken); assertEquals(1, files.size()); assertEquals(FileStatus.READY, files.get(0).getInternal().getStatus().getId()); @@ -261,15 +259,15 @@ public void testResyncStudy() throws IOException, CatalogException { //Delete file. ReSync. Will detect one File.Status.MISSING file (like checkFile) Files.delete(filePath); - files = fileScanner.reSync(study, true, sessionIdUser); + files = fileScanner.reSync(organizationId, study, true, ownerToken); assertEquals(1, files.size()); assertEquals(FileStatus.MISSING, files.get(0).getInternal().getStatus().getId()); originalChecksum = files.get(0).getChecksum(); //Restore file. CheckStudyFiles. Will detect one found file. Checksum must be different. - CatalogManagerTest.createDebugFile(filePath.toString()); - files = fileScanner.reSync(study, true, sessionIdUser); + MongoBackupUtils.createDebugFile(filePath.toString()); + files = fileScanner.reSync(organizationId, study, true, ownerToken); assertEquals(1, files.size()); assertEquals(FileStatus.READY, files.get(0).getInternal().getStatus().getId()); @@ -285,14 +283,14 @@ public void testComplexAdd() throws IOException, CatalogException, URISyntaxExce URI fileUri = getClass().getResource("/biofiles/variant-test-file.vcf.gz").toURI(); ioManager.copy(fileUri, directory.resolve("file1.vcf.gz").toUri()); - CatalogManagerTest.createDebugFile(directory.resolve("file1.vcf.variants.json").toString()); - CatalogManagerTest.createDebugFile(directory.resolve("file1.vcf.variants.json.gz").toString()); - CatalogManagerTest.createDebugFile(directory.resolve("file1.vcf.variants.json.snappy").toString()); - CatalogManagerTest.createDebugFile(directory.resolve("file2.bam").toString()); - CatalogManagerTest.createDebugFile(directory.resolve("file2.sam.gz").toString()); + MongoBackupUtils.createDebugFile(directory.resolve("file1.vcf.variants.json").toString()); + MongoBackupUtils.createDebugFile(directory.resolve("file1.vcf.variants.json.gz").toString()); + MongoBackupUtils.createDebugFile(directory.resolve("file1.vcf.variants.json.snappy").toString()); + MongoBackupUtils.createDebugFile(directory.resolve("file2.bam").toString()); + MongoBackupUtils.createDebugFile(directory.resolve("file2.sam.gz").toString()); FileScanner fileScanner = new FileScanner(catalogManager); - List files = fileScanner.scan(folder, directory.toUri(), FileScanner.FileScannerPolicy.REPLACE, true, true, sessionIdUser); + List files = fileScanner.scan(organizationId, folder, directory.toUri(), FileScanner.FileScannerPolicy.REPLACE, true, true, ownerToken); Map map = files.stream().collect(Collectors.toMap(File::getName, (f) -> f)); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/FqnUtilsTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/FqnUtilsTest.java index 911908e23ce..a9de1f72bca 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/FqnUtilsTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/FqnUtilsTest.java @@ -11,21 +11,21 @@ public class FqnUtilsTest { @Test public void testFqnFull() { - FqnUtils.FQN fqn = new FqnUtils.FQN("user@project:study"); - assertEquals("user", fqn.getUser()); + FqnUtils.FQN fqn = new FqnUtils.FQN("org@project:study"); + assertEquals("org", fqn.getOrganization()); assertEquals("project", fqn.getProject()); - assertEquals("user@project", fqn.getProjectFqn()); + assertEquals("org@project", fqn.getProjectFqn()); assertEquals("study", fqn.getStudy()); - assertEquals("user@project:study", fqn.toString()); + assertEquals("org@project:study", fqn.toString()); } @Test public void testFqnPartial() { - FqnUtils.FQN fqn = new FqnUtils.FQN("user@project"); - assertEquals("user", fqn.getUser()); + FqnUtils.FQN fqn = new FqnUtils.FQN("org@project"); + assertEquals("org", fqn.getOrganization()); assertEquals("project", fqn.getProject()); - assertEquals("user@project", fqn.getProjectFqn()); + assertEquals("org@project", fqn.getProjectFqn()); assertEquals(null, fqn.getStudy()); - assertEquals("user@project", fqn.toString()); + assertEquals("org@project", fqn.toString()); } } \ No newline at end of file diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/templates/TemplateManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/templates/TemplateManagerTest.java index 3edb612f992..ca96f875326 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/templates/TemplateManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/templates/TemplateManagerTest.java @@ -2,19 +2,15 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.TestParamConstants; +import org.opencb.opencga.catalog.managers.AbstractManagerTest; import org.opencb.opencga.catalog.managers.CatalogManager; -import org.opencb.opencga.catalog.managers.CatalogManagerExternalResource; import org.opencb.opencga.catalog.templates.TemplateManager; import org.opencb.opencga.catalog.templates.config.TemplateManifest; import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.models.user.Account; import org.opencb.opencga.core.models.user.User; import org.opencb.opencga.core.testclassification.duration.MediumTests; @@ -22,35 +18,23 @@ import java.nio.file.Paths; @Category(MediumTests.class) -public class TemplateManagerTest { - - @Rule - public CatalogManagerExternalResource catalogManagerResource = new CatalogManagerExternalResource(); - - @Before - public void before() throws Exception { - catalogManagerResource.before(); - } - - @After - public void after() { - catalogManagerResource.after(); - } +public class TemplateManagerTest extends AbstractManagerTest { @Test public void test() throws Exception { CatalogManager catalogManager = catalogManagerResource.getCatalogManager(); - String adminToken = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); - catalogManager.getUserManager().create(new User().setId("user1").setName("User 1").setAccount(new Account().setType(Account.AccountType.FULL)), - TestParamConstants.PASSWORD, adminToken); - catalogManager.getUserManager().create(new User().setId("user2").setName("User 2").setAccount(new Account().setType(Account.AccountType.GUEST)), TestParamConstants.PASSWORD, adminToken); - catalogManager.getUserManager().create(new User().setId("user3").setName("User 3").setAccount(new Account().setType(Account.AccountType.GUEST)), TestParamConstants.PASSWORD, adminToken); - catalogManager.getUserManager().create(new User().setId("user4").setName("User 4").setAccount(new Account().setType(Account.AccountType.GUEST)), TestParamConstants.PASSWORD, adminToken); + catalogManager.getUserManager().create(new User().setId("user1").setName("User 1").setOrganization(organizationId), + TestParamConstants.PASSWORD, opencgaToken); + catalogManager.getUserManager().create(new User().setId("user2").setName("User 2").setOrganization(organizationId), + TestParamConstants.PASSWORD, opencgaToken); + catalogManager.getUserManager().create(new User().setId("user3").setName("User 3").setOrganization(organizationId), + TestParamConstants.PASSWORD, opencgaToken); + catalogManager.getUserManager().create(new User().setId("user4").setName("User 4").setOrganization(organizationId), + TestParamConstants.PASSWORD, opencgaToken); - String token = catalogManager.getUserManager().login("user1", TestParamConstants.PASSWORD).getToken(); - catalogManager.getProjectManager().create("project", "Project", "", "hsapiens", "common", "GRCh38", QueryOptions.empty(), token); - catalogManager.getStudyManager().create("project", new Study().setId("study"), QueryOptions.empty(), token); + catalogManager.getProjectManager().create("project", "Project", "", "hsapiens", "common", "GRCh38", QueryOptions.empty(), ownerToken); + catalogManager.getStudyManager().create("project", new Study().setId("study"), QueryOptions.empty(), ownerToken); URI resource = catalogManagerResource.getResourceUri("templates/manifest.yml"); catalogManagerResource.getResourceUri("templates/families.members.txt"); @@ -61,27 +45,28 @@ public void test() throws Exception { ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory()); TemplateManifest manifest = objectMapper.readValue(resource.toURL(), TemplateManifest.class); - TemplateManager templateManager = new TemplateManager(catalogManager, false, false, token); + TemplateManager templateManager = new TemplateManager(catalogManager, false, false, ownerToken); templateManager.execute(manifest, Paths.get(resource).getParent()); - templateManager = new TemplateManager(catalogManager, true, true, token); + templateManager = new TemplateManager(catalogManager, true, true, ownerToken); templateManager.execute(manifest, Paths.get(resource).getParent()); } @Test public void test_yaml() throws Exception { CatalogManager catalogManager = catalogManagerResource.getCatalogManager(); - String adminToken = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); - catalogManager.getUserManager().create(new User().setId("user1").setName("User 1").setAccount(new Account().setType(Account.AccountType.FULL)), - TestParamConstants.PASSWORD, adminToken); - catalogManager.getUserManager().create(new User().setId("user2").setName("User 2").setAccount(new Account().setType(Account.AccountType.GUEST)), TestParamConstants.PASSWORD, adminToken); - catalogManager.getUserManager().create(new User().setId("user3").setName("User 3").setAccount(new Account().setType(Account.AccountType.GUEST)), TestParamConstants.PASSWORD, adminToken); - catalogManager.getUserManager().create(new User().setId("user4").setName("User 4").setAccount(new Account().setType(Account.AccountType.GUEST)), TestParamConstants.PASSWORD, adminToken); + catalogManager.getUserManager().create(new User().setId("user1").setName("User 1").setOrganization(organizationId), + TestParamConstants.PASSWORD, opencgaToken); + catalogManager.getUserManager().create(new User().setId("user2").setName("User 2").setOrganization(organizationId), + TestParamConstants.PASSWORD, opencgaToken); + catalogManager.getUserManager().create(new User().setId("user3").setName("User 3").setOrganization(organizationId), + TestParamConstants.PASSWORD, opencgaToken); + catalogManager.getUserManager().create(new User().setId("user4").setName("User 4").setOrganization(organizationId), + TestParamConstants.PASSWORD, opencgaToken); - String token = catalogManager.getUserManager().login("user1", TestParamConstants.PASSWORD).getToken(); - catalogManager.getProjectManager().create("project", "Project", "", "hsapiens", "common", "GRCh38", QueryOptions.empty(), token); - catalogManager.getStudyManager().create("project", new Study().setId("study"), QueryOptions.empty(), token); + catalogManager.getProjectManager().create("project", "Project", "", "hsapiens", "common", "GRCh38", QueryOptions.empty(), ownerToken); + catalogManager.getStudyManager().create("project", new Study().setId("study"), QueryOptions.empty(), ownerToken); URI resource = catalogManagerResource.getResourceUri("templates_yaml/manifest.yml"); catalogManagerResource.getResourceUri("templates_yaml/families.members.txt"); @@ -91,10 +76,10 @@ public void test_yaml() throws Exception { ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory()); TemplateManifest manifest = objectMapper.readValue(resource.toURL(), TemplateManifest.class); - TemplateManager templateManager = new TemplateManager(catalogManager, false, false, token); + TemplateManager templateManager = new TemplateManager(catalogManager, false, false, ownerToken); templateManager.execute(manifest, Paths.get(resource).getParent()); - templateManager = new TemplateManager(catalogManager, true, true, token); + templateManager = new TemplateManager(catalogManager, true, true, ownerToken); templateManager.execute(manifest, Paths.get(resource).getParent()); } diff --git a/opencga-catalog/src/test/resources/biofiles/clinical_analyses.json.gz b/opencga-catalog/src/test/resources/biofiles/clinical_analyses.json.gz new file mode 100644 index 00000000000..59ce26c8ad8 Binary files /dev/null and b/opencga-catalog/src/test/resources/biofiles/clinical_analyses.json.gz differ diff --git a/opencga-catalog/src/test/resources/configuration-test.yml b/opencga-catalog/src/test/resources/configuration-test.yml index 3ac6f46fe0b..d0ebcf32ba6 100644 --- a/opencga-catalog/src/test/resources/configuration-test.yml +++ b/opencga-catalog/src/test/resources/configuration-test.yml @@ -6,9 +6,7 @@ databasePrefix: "opencga_test" workspace: "/tmp/opencga/sessions" jobDir: "/tmp/opencga/JOBS" - -admin: - secretKey: "asidnadh19rh230qncfascd1.rffzasf.asd.ad.12ddeASDAsd12e1d.adsx" +maxLoginAttempts: 5 audit: manager: "" # Java manager of the audit implementation to be used to audit. If empty, catalog database will be used. @@ -16,7 +14,28 @@ audit: maxSize: 100 # Maximum size that the audit collection will have in Gigabytes (GB). analysis: + packages: # List of packages where to find analysis tools + - "org.opencb.opencga" scratchDir: "/tmp/" # Scratch folder for the analysis. + # Default URL for downloading analysis resources. + resourceUrl: "http://resources.opencb.org/opencb/opencga/analysis/" + # Docker used by OpenCGA analysis and containing external tools such as samtools, bcftools, tabix, fastqc, plink1.9, bwa and r-base + # You can indicate the version, e.g: opencb/opencga-ext-tools:2.12.0, otherwise the current OpenCGA version will be used + opencgaExtTools: "opencb/opencga-ext-tools" + tools: + - id: "exomiser" + version: "13.1" + dockerId: "exomiser/exomiser-cli:13.1.0" + resources: + HG38: "exomiser/2109_hg38.zip" + PHENOTYPE: "exomiser/2109_phenotype.zip" + - id: "exomiser" + version: "14.0" + defaultVersion: true + dockerId: "exomiser/exomiser-cli:14.0.0" + resources: + HG38: "exomiser/2402_hg38.zip" + PHENOTYPE: "exomiser/2402_phenotype.zip" execution: # Accepted values are "local", "SGE", "azure-batch", "k8s" # see org.opencb.opencga.master.monitor.executors.ExecutorFactory @@ -50,7 +69,7 @@ email: ssl: false hooks: - user@1000G:phase1: # Full Qualified Name of the study. + test@1000G:phase1: # Full Qualified Name of the study. file: # Entity where the hook will be checked - field: "name" # Field of the entity to be checked value: "~(.*)vcf.gz$" # Value to be checked @@ -70,29 +89,6 @@ catalog: password: "" options: authenticationDatabase: "" - searchEngine: ## Solr configuration, by default is the same than storage - hosts: - - "http://localhost:8983/solr/" - user: "" - password: "" - options: - mode: "core" - timeout: 30000 - insertBatchSize: 2000 - -authentication: - expiration: 1000 -#LDAP configuration example -# authenticationOrigins: -# - id: ldap # Any id -# type: LDAP # At the moment, we only support LDAP -# host: ldap://localhost:9000 -# options: -# usersSearch: dc=ge,dc=co,dc=uk # Base search to look for the users -# groupsSearch: ou=general,ou=groups,dc=ge,dc=co,dc=uk # Base search to look for the groups - -optimizations: - simplifyPermissions: false server: rest: diff --git a/opencga-catalog/src/test/resources/load-clinical-analyses.conf b/opencga-catalog/src/test/resources/load-clinical-analyses.conf new file mode 100644 index 00000000000..875947aae79 --- /dev/null +++ b/opencga-catalog/src/test/resources/load-clinical-analyses.conf @@ -0,0 +1,5 @@ +titi +Titi123456; +titi@p1:s1 +/opt/opencga/conf/configuration.yml +/home/jtarraga/reanalysis/test3 \ No newline at end of file diff --git a/opencga-catalog/src/test/resources/templates/manifest.yml b/opencga-catalog/src/test/resources/templates/manifest.yml index 07cf99f36be..b205f1f5286 100644 --- a/opencga-catalog/src/test/resources/templates/manifest.yml +++ b/opencga-catalog/src/test/resources/templates/manifest.yml @@ -1,5 +1,6 @@ configuration: - version: 2.2.0 + organizationId: test + version: 3.0.0 index: false projectId: project study: diff --git a/opencga-catalog/src/test/resources/templates_yaml/manifest.yml b/opencga-catalog/src/test/resources/templates_yaml/manifest.yml index 58f4f474ccf..b205f1f5286 100644 --- a/opencga-catalog/src/test/resources/templates_yaml/manifest.yml +++ b/opencga-catalog/src/test/resources/templates_yaml/manifest.yml @@ -1,5 +1,6 @@ configuration: - version: 2.4.0 + organizationId: test + version: 3.0.0 index: false projectId: project study: diff --git a/opencga-client/src/main/R/R/Admin-methods.R b/opencga-client/src/main/R/R/Admin-methods.R index 02700c96173..d63a1ad3478 100644 --- a/opencga-client/src/main/R/R/Admin-methods.R +++ b/opencga-client/src/main/R/R/Admin-methods.R @@ -2,7 +2,6 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-04-25 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -21,14 +20,14 @@ #' | endpointName | Endpoint WS | parameters accepted | #' | -- | :-- | --: | #' | groupByAudit | /{apiVersion}/admin/audit/groupBy | count, limit, fields[*], entity[*], action, before, after, date | -#' | indexStatsCatalog | /{apiVersion}/admin/catalog/indexStats | collection | #' | installCatalog | /{apiVersion}/admin/catalog/install | body[*] | -#' | jwtCatalog | /{apiVersion}/admin/catalog/jwt | body[*] | +#' | jwtCatalog | /{apiVersion}/admin/catalog/jwt | organization, body[*] | #' | createUsers | /{apiVersion}/admin/users/create | body[*] | -#' | importUsers | /{apiVersion}/admin/users/import | body[*] | -#' | searchUsers | /{apiVersion}/admin/users/search | include, exclude, limit, skip, count, user, account, authenticationId | -#' | syncUsers | /{apiVersion}/admin/users/sync | body[*] | -#' | usersUpdateGroups | /{apiVersion}/admin/users/{user}/groups/update | user[*], action, body[*] | +#' | importUsers | /{apiVersion}/admin/users/import | organization, body[*] | +#' | permissionsUsers | /{apiVersion}/admin/users/permissions | study, entryIds, permissions, category | +#' | searchUsers | /{apiVersion}/admin/users/search | include, exclude, limit, skip, count, organization, user, authenticationId | +#' | syncUsers | /{apiVersion}/admin/users/sync | organization, body[*] | +#' | usersUpdateGroups | /{apiVersion}/admin/users/{user}/groups/update | organization, user[*], action, body[*] | #' #' @md #' @seealso \url{http://docs.opencb.org/display/opencga/Using+OpenCGA} and the RESTful API documentation @@ -44,7 +43,7 @@ setMethod("adminClient", "OpencgaR", function(OpencgaR, user, endpointName, para #' @param count Count the number of elements matching the group. #' @param limit Maximum number of documents (groups) to be returned. #' @param fields Comma separated list of fields by which to group by. - #' @param entity Entity to be grouped by. Allowed values: ['AUDIT USER PROJECT STUDY FILE SAMPLE JOB INDIVIDUAL COHORT DISEASE_PANEL FAMILY CLINICAL_ANALYSIS INTERPRETATION VARIANT ALIGNMENT CLINICAL EXPRESSION RGA FUNCTIONAL'] + #' @param entity Entity to be grouped by. Allowed values: ['AUDIT NOTE ORGANIZATION USER PROJECT STUDY FILE SAMPLE JOB INDIVIDUAL COHORT DISEASE_PANEL FAMILY CLINICAL_ANALYSIS INTERPRETATION VARIANT ALIGNMENT CLINICAL EXPRESSION RGA FUNCTIONAL'] #' @param action Action performed. #' @param before Object before update. #' @param after Object after update. @@ -53,12 +52,6 @@ setMethod("adminClient", "OpencgaR", function(OpencgaR, user, endpointName, para subcategoryId=NULL, action="groupBy", params=params, httpMethod="GET", as.queryParam=c("fields","entity"), ...), - #' @section Endpoint /{apiVersion}/admin/catalog/indexStats: - #' Sync Catalog into the Solr. - #' @param collection Collection to be indexed (file, sample, individual, family, cohort and/or job). If not provided, all of them will be indexed. - indexStatsCatalog=fetchOpenCGA(object=OpencgaR, category="admin", categoryId=NULL, subcategory="catalog", - subcategoryId=NULL, action="indexStats", params=params, httpMethod="POST", as.queryParam=NULL, ...), - #' @section Endpoint /{apiVersion}/admin/catalog/install: #' Install OpenCGA database. #' @param data JSON containing the mandatory parameters. @@ -67,6 +60,7 @@ setMethod("adminClient", "OpencgaR", function(OpencgaR, user, endpointName, para #' @section Endpoint /{apiVersion}/admin/catalog/jwt: #' Change JWT secret key. + #' @param organization Organization id. #' @param data JSON containing the parameters. jwtCatalog=fetchOpenCGA(object=OpencgaR, category="admin", categoryId=NULL, subcategory="catalog", subcategoryId=NULL, action="jwt", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -79,10 +73,20 @@ setMethod("adminClient", "OpencgaR", function(OpencgaR, user, endpointName, para #' @section Endpoint /{apiVersion}/admin/users/import: #' Import users or a group of users from LDAP or AAD. + #' @param organization Organization id. #' @param data JSON containing the parameters. importUsers=fetchOpenCGA(object=OpencgaR, category="admin", categoryId=NULL, subcategory="users", subcategoryId=NULL, action="import", params=params, httpMethod="POST", as.queryParam=NULL, ...), + #' @section Endpoint /{apiVersion}/admin/users/permissions: + #' User permissions. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param entryIds Comma separated list of entry ids. + #' @param permissions Comma separated list of permissions to be retrieved. + #' @param category Category corresponding to the id's provided. + permissionsUsers=fetchOpenCGA(object=OpencgaR, category="admin", categoryId=NULL, subcategory="users", + subcategoryId=NULL, action="permissions", params=params, httpMethod="GET", as.queryParam=NULL, ...), + #' @section Endpoint /{apiVersion}/admin/users/search: #' User search method. #' @param include Fields included in the response, whole JSON path must be provided. @@ -90,20 +94,22 @@ setMethod("adminClient", "OpencgaR", function(OpencgaR, user, endpointName, para #' @param limit Number of results to be returned. #' @param skip Number of results to skip. #' @param count Get the total number of results matching the query. Deactivated by default. + #' @param organization Organization id. #' @param user User ID. - #' @param account Account type [GUEST, FULL, ADMINISTRATOR]. #' @param authenticationId Authentication origin ID. searchUsers=fetchOpenCGA(object=OpencgaR, category="admin", categoryId=NULL, subcategory="users", subcategoryId=NULL, action="search", params=params, httpMethod="GET", as.queryParam=NULL, ...), #' @section Endpoint /{apiVersion}/admin/users/sync: #' Synchronise a group of users from an authentication origin with a group in a study from catalog. + #' @param organization Organization id. #' @param data JSON containing the parameters. syncUsers=fetchOpenCGA(object=OpencgaR, category="admin", categoryId=NULL, subcategory="users", subcategoryId=NULL, action="sync", params=params, httpMethod="POST", as.queryParam=NULL, ...), #' @section Endpoint /{apiVersion}/admin/users/{user}/groups/update: #' Add or remove users from existing groups. + #' @param organization Organization id. #' @param user User ID. #' @param action Action to be performed: ADD or REMOVE user to/from groups. Allowed values: ['ADD REMOVE'] #' @param data JSON containing the parameters. diff --git a/opencga-client/src/main/R/R/Alignment-methods.R b/opencga-client/src/main/R/R/Alignment-methods.R index e367674574c..3c3198d89f8 100644 --- a/opencga-client/src/main/R/R/Alignment-methods.R +++ b/opencga-client/src/main/R/R/Alignment-methods.R @@ -2,7 +2,6 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-04-25 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -20,19 +19,19 @@ #' #' | endpointName | Endpoint WS | parameters accepted | #' | -- | :-- | --: | -#' | runBwa | /{apiVersion}/analysis/alignment/bwa/run | study, jobId, jobDependsOn, jobDescription, jobTags, body[*] | -#' | runCoverageIndex | /{apiVersion}/analysis/alignment/coverage/index/run | study, jobId, jobDependsOn, jobDescription, jobTags, body[*] | -#' | coverageQcGeneCoverageStatsRun | /{apiVersion}/analysis/alignment/coverage/qc/geneCoverageStats/run | study, jobId, jobDependsOn, jobDescription, jobTags, body[*] | +#' | runBwa | /{apiVersion}/analysis/alignment/bwa/run | study, jobId, jobDependsOn, jobDescription, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runCoverageIndex | /{apiVersion}/analysis/alignment/coverage/index/run | study, jobId, jobDependsOn, jobDescription, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | coverageQcGeneCoverageStatsRun | /{apiVersion}/analysis/alignment/coverage/qc/geneCoverageStats/run | study, jobId, jobDependsOn, jobDescription, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | #' | queryCoverage | /{apiVersion}/analysis/alignment/coverage/query | file[*], study, region, gene, offset, onlyExons, range, windowSize, splitResults | #' | ratioCoverage | /{apiVersion}/analysis/alignment/coverage/ratio | file1[*], file2[*], study, skipLog2, region, gene, offset, onlyExons, windowSize, splitResults | #' | statsCoverage | /{apiVersion}/analysis/alignment/coverage/stats | file[*], gene[*], study, threshold | -#' | runDeeptools | /{apiVersion}/analysis/alignment/deeptools/run | study, jobId, jobDependsOn, jobDescription, jobTags, body[*] | -#' | runFastqc | /{apiVersion}/analysis/alignment/fastqc/run | study, jobId, jobDependsOn, jobDescription, jobTags, body[*] | -#' | runIndex | /{apiVersion}/analysis/alignment/index/run | study, jobId, jobDependsOn, jobDescription, jobTags, body[*] | -#' | runPicard | /{apiVersion}/analysis/alignment/picard/run | study, jobId, jobDependsOn, jobDescription, jobTags, body[*] | -#' | runQc | /{apiVersion}/analysis/alignment/qc/run | study, jobId, jobDependsOn, jobDescription, jobTags, body[*] | +#' | runDeeptools | /{apiVersion}/analysis/alignment/deeptools/run | study, jobId, jobDependsOn, jobDescription, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runFastqc | /{apiVersion}/analysis/alignment/fastqc/run | study, jobId, jobDependsOn, jobDescription, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runIndex | /{apiVersion}/analysis/alignment/index/run | study, jobId, jobDependsOn, jobDescription, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runPicard | /{apiVersion}/analysis/alignment/picard/run | study, jobId, jobDependsOn, jobDescription, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runQc | /{apiVersion}/analysis/alignment/qc/run | study, jobId, jobDependsOn, jobDescription, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | #' | query | /{apiVersion}/analysis/alignment/query | limit, skip, count, file[*], study, region, gene, offset, onlyExons, minMappingQuality, maxNumMismatches, maxNumHits, properlyPaired, maxInsertSize, skipUnmapped, skipDuplicated, regionContained, forceMDField, binQualities, splitResults | -#' | runSamtools | /{apiVersion}/analysis/alignment/samtools/run | study, jobId, jobDependsOn, jobDescription, jobTags, body[*] | +#' | runSamtools | /{apiVersion}/analysis/alignment/samtools/run | study, jobId, jobDependsOn, jobDescription, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | #' #' @md #' @seealso \url{http://docs.opencb.org/display/opencga/Using+OpenCGA} and the RESTful API documentation @@ -50,6 +49,9 @@ setMethod("alignmentClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobDescription Job description. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data BWA parameters. runBwa=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="alignment/bwa", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -61,6 +63,9 @@ setMethod("alignmentClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobDescription Job description. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Coverage computation parameters. runCoverageIndex=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="alignment/coverage/index", subcategoryId=NULL, action="run", params=params, @@ -73,6 +78,9 @@ setMethod("alignmentClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobDescription Job description. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Gene coverage stats parameters for a given BAM file and a list of genes. coverageQcGeneCoverageStatsRun=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="alignment/coverage/qc/geneCoverageStats", subcategoryId=NULL, action="run", params=params, @@ -81,7 +89,7 @@ setMethod("alignmentClient", "OpencgaR", function(OpencgaR, endpointName, params #' @section Endpoint /{apiVersion}/analysis/alignment/coverage/query: #' Query the coverage of an alignment file for regions or genes. #' @param file File ID. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param region Comma separated list of regions 'chr:start-end, e.g.: 2,3:63500-65000. #' @param gene Comma separated list of genes, e.g.: BCRA2,TP53. #' @param offset Offset to extend the region, gene or exon at up and downstream. @@ -97,7 +105,7 @@ setMethod("alignmentClient", "OpencgaR", function(OpencgaR, endpointName, params #' Compute coverage ratio from file #1 vs file #2, (e.g. somatic vs germline). #' @param file1 Input file #1 (e.g. somatic file). #' @param file2 Input file #2 (e.g. germline file). - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param skipLog2 Do not apply Log2 to normalise the coverage ratio. #' @param region Comma separated list of regions 'chr:start-end, e.g.: 2,3:63500-65000. #' @param gene Comma separated list of genes, e.g.: BCRA2,TP53. @@ -113,7 +121,7 @@ setMethod("alignmentClient", "OpencgaR", function(OpencgaR, endpointName, params #' Compute coverage stats per transcript for a list of genes. #' @param file File ID. #' @param gene Comma separated list of genes, e.g.: BCRA2,TP53. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param threshold Only regions whose coverage depth is under this threshold will be reported. statsCoverage=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="alignment/coverage", subcategoryId=NULL, action="stats", params=params, httpMethod="GET", @@ -126,6 +134,9 @@ setMethod("alignmentClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobDescription Job description. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Deeptools parameters. Supported Deeptools commands: bamCoverage, bamCompare. runDeeptools=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="alignment/deeptools", subcategoryId=NULL, action="run", params=params, httpMethod="POST", @@ -138,6 +149,9 @@ setMethod("alignmentClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobDescription Job description. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data FastQC parameters. runFastqc=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="alignment/fastqc", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -149,6 +163,9 @@ setMethod("alignmentClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobDescription Job description. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Alignment index params. runIndex=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="alignment/index", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -160,6 +177,9 @@ setMethod("alignmentClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobDescription Job description. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Picard parameters. Supported Picard commands: CollectHsMetrics, CollectWgsMetrics, BedToIntervalList. runPicard=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="alignment/picard", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -171,6 +191,9 @@ setMethod("alignmentClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobDescription Job description. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Alignment quality control (QC) parameters. It computes: stats, flag stats and fastqc metrics. The BAM file ID is mandatory and in order to skip some metrics, use the following keywords (separated by commas): stats, flagstats, fastqc. runQc=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="alignment/qc", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -181,7 +204,7 @@ setMethod("alignmentClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param skip Number of results to skip. #' @param count Get the total number of results matching the query. Deactivated by default. #' @param file File ID. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param region Comma separated list of regions 'chr:start-end, e.g.: 2,3:63500-65000. #' @param gene Comma separated list of genes, e.g.: BCRA2,TP53. #' @param offset Offset to extend the region, gene or exon at up and downstream. @@ -207,6 +230,9 @@ setMethod("alignmentClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobDescription Job description. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Samtools parameters. Supported Samtools commands: sort, index, view, stats, flagstat, dict, faidx, depth, plot-bamstats. runSamtools=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="alignment/samtools", subcategoryId=NULL, action="run", params=params, httpMethod="POST", diff --git a/opencga-client/src/main/R/R/AllGenerics.R b/opencga-client/src/main/R/R/AllGenerics.R index 3f65ad80933..5b3e3d890ce 100644 --- a/opencga-client/src/main/R/R/AllGenerics.R +++ b/opencga-client/src/main/R/R/AllGenerics.R @@ -1,46 +1,51 @@ +# ############################################################################## +## OrganizationClient +setGeneric("organizationClient", function(OpencgaR, id, organization, user, endpointName, params=NULL, ...) + standardGeneric("organizationClient")) + # ############################################################################## ## UserClient -setGeneric("userClient", function(OpencgaR, users, user, filterId, endpointName, params=NULL, ...) +setGeneric("userClient", function(OpencgaR, filterId, user, users, endpointName, params=NULL, ...) standardGeneric("userClient")) # ############################################################################## ## ProjectClient -setGeneric("projectClient", function(OpencgaR, projects, project, endpointName, params=NULL, ...) +setGeneric("projectClient", function(OpencgaR, project, projects, endpointName, params=NULL, ...) standardGeneric("projectClient")) # ############################################################################## ## StudyClient -setGeneric("studyClient", function(OpencgaR, studies, members, variableSet, group, templateId, study, endpointName, params=NULL, ...) +setGeneric("studyClient", function(OpencgaR, group, id, members, studies, study, templateId, variableSet, endpointName, params=NULL, ...) standardGeneric("studyClient")) # ############################################################################## ## FileClient -setGeneric("fileClient", function(OpencgaR, members, file, folder, annotationSet, files, endpointName, params=NULL, ...) +setGeneric("fileClient", function(OpencgaR, annotationSet, file, files, folder, members, endpointName, params=NULL, ...) standardGeneric("fileClient")) # ############################################################################## ## JobClient -setGeneric("jobClient", function(OpencgaR, jobs, members, job, endpointName, params=NULL, ...) +setGeneric("jobClient", function(OpencgaR, job, jobs, members, endpointName, params=NULL, ...) standardGeneric("jobClient")) # ############################################################################## ## SampleClient -setGeneric("sampleClient", function(OpencgaR, annotationSet, members, samples, sample, endpointName, params=NULL, ...) +setGeneric("sampleClient", function(OpencgaR, annotationSet, members, sample, samples, endpointName, params=NULL, ...) standardGeneric("sampleClient")) # ############################################################################## ## IndividualClient -setGeneric("individualClient", function(OpencgaR, annotationSet, individual, members, individuals, endpointName, params=NULL, ...) +setGeneric("individualClient", function(OpencgaR, annotationSet, individual, individuals, members, endpointName, params=NULL, ...) standardGeneric("individualClient")) # ############################################################################## ## FamilyClient -setGeneric("familyClient", function(OpencgaR, family, members, annotationSet, families, endpointName, params=NULL, ...) +setGeneric("familyClient", function(OpencgaR, annotationSet, families, family, members, endpointName, params=NULL, ...) standardGeneric("familyClient")) # ############################################################################## ## CohortClient -setGeneric("cohortClient", function(OpencgaR, annotationSet, members, cohort, cohorts, endpointName, params=NULL, ...) +setGeneric("cohortClient", function(OpencgaR, annotationSet, cohort, cohorts, members, endpointName, params=NULL, ...) standardGeneric("cohortClient")) # ############################################################################## @@ -60,7 +65,7 @@ setGeneric("variantClient", function(OpencgaR, endpointName, params=NULL, ...) # ############################################################################## ## ClinicalClient -setGeneric("clinicalClient", function(OpencgaR, members, clinicalAnalyses, annotationSet, interpretations, interpretation, clinicalAnalysis, endpointName, params=NULL, ...) +setGeneric("clinicalClient", function(OpencgaR, annotationSet, clinicalAnalyses, clinicalAnalysis, interpretation, interpretations, members, endpointName, params=NULL, ...) standardGeneric("clinicalClient")) # ############################################################################## @@ -75,7 +80,7 @@ setGeneric("metaClient", function(OpencgaR, endpointName, params=NULL, ...) # ############################################################################## ## GA4GHClient -setGeneric("ga4ghClient", function(OpencgaR, study, file, endpointName, params=NULL, ...) +setGeneric("ga4ghClient", function(OpencgaR, file, study, endpointName, params=NULL, ...) standardGeneric("ga4ghClient")) # ############################################################################## diff --git a/opencga-client/src/main/R/R/Clinical-methods.R b/opencga-client/src/main/R/R/Clinical-methods.R index d1f276dc792..a45c200d12e 100644 --- a/opencga-client/src/main/R/R/Clinical-methods.R +++ b/opencga-client/src/main/R/R/Clinical-methods.R @@ -2,7 +2,6 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-04-25 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -24,30 +23,31 @@ #' | loadAnnotationSets | /{apiVersion}/analysis/clinical/annotationSets/load | study, variableSetId[*], path[*], parents, annotationSetId, body | #' | updateClinicalConfiguration | /{apiVersion}/analysis/clinical/clinical/configuration/update | study, body | #' | create | /{apiVersion}/analysis/clinical/create | include, exclude, study, skipCreateDefaultInterpretation, includeResult, body[*] | -#' | distinct | /{apiVersion}/analysis/clinical/distinct | study, id, uuid, type, disorder, files, sample, individual, proband, probandSamples, family, familyMembers, familyMemberSamples, panels, locked, analystId, priority, flags, creationDate, modificationDate, dueDate, qualityControlSummary, release, status, internalStatus, annotation, deleted, field[*] | -#' | distinctInterpretation | /{apiVersion}/analysis/clinical/interpretation/distinct | study, id, uuid, clinicalAnalysisId, analystId, methodName, panels, primaryFindings, secondaryFindings, creationDate, modificationDate, status, internalStatus, release, field[*] | -#' | searchInterpretation | /{apiVersion}/analysis/clinical/interpretation/search | include, exclude, limit, skip, sort, study, id, uuid, clinicalAnalysisId, analystId, methodName, panels, primaryFindings, secondaryFindings, creationDate, modificationDate, status, internalStatus, release | +#' | distinct | /{apiVersion}/analysis/clinical/distinct | study, id, uuid, type, disorder, files, sample, individual, proband, probandSamples, family, familyMembers, familyMemberSamples, panels, locked, analystId, priority, flags, creationDate, modificationDate, dueDate, qualityControlSummary, release, snapshot, status, internalStatus, annotation, deleted, field[*] | +#' | distinctInterpretation | /{apiVersion}/analysis/clinical/interpretation/distinct | study, id, uuid, name, clinicalAnalysisId, analystId, methodName, panels, primaryFindings, secondaryFindings, creationDate, modificationDate, status, internalStatus, release, field[*] | +#' | searchInterpretation | /{apiVersion}/analysis/clinical/interpretation/search | include, exclude, limit, skip, sort, study, id, uuid, name, clinicalAnalysisId, analystId, methodName, panels, primaryFindings, secondaryFindings, creationDate, modificationDate, status, internalStatus, release | #' | infoInterpretation | /{apiVersion}/analysis/clinical/interpretation/{interpretations}/info | include, exclude, interpretations[*], study, version, deleted | -#' | runInterpreterCancerTiering | /{apiVersion}/analysis/clinical/interpreter/cancerTiering/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | runInterpreterExomiser | /{apiVersion}/analysis/clinical/interpreter/exomiser/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | runInterpreterTeam | /{apiVersion}/analysis/clinical/interpreter/team/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | runInterpreterTiering | /{apiVersion}/analysis/clinical/interpreter/tiering/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | runInterpreterZetta | /{apiVersion}/analysis/clinical/interpreter/zetta/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | +#' | runInterpreterCancerTiering | /{apiVersion}/analysis/clinical/interpreter/cancerTiering/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runInterpreterExomiser | /{apiVersion}/analysis/clinical/interpreter/exomiser/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runInterpreterTeam | /{apiVersion}/analysis/clinical/interpreter/team/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runInterpreterTiering | /{apiVersion}/analysis/clinical/interpreter/tiering/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runInterpreterZetta | /{apiVersion}/analysis/clinical/interpreter/zetta/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | load | /{apiVersion}/analysis/clinical/load | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | #' | aggregationStatsRga | /{apiVersion}/analysis/clinical/rga/aggregationStats | limit, skip, sampleId, individualId, sex, phenotypes, disorders, numParents, geneId, geneName, chromosome, start, end, transcriptId, variants, dbSnps, knockoutType, filter, type, clinicalSignificance, populationFrequency, consequenceType, study, field[*] | #' | queryRgaGene | /{apiVersion}/analysis/clinical/rga/gene/query | include, exclude, limit, skip, count, includeIndividual, skipIndividual, limitIndividual, sampleId, individualId, sex, phenotypes, disorders, numParents, geneId, geneName, chromosome, start, end, transcriptId, variants, dbSnps, knockoutType, filter, type, clinicalSignificance, populationFrequency, consequenceType, study | #' | summaryRgaGene | /{apiVersion}/analysis/clinical/rga/gene/summary | limit, skip, count, sampleId, individualId, sex, phenotypes, disorders, numParents, geneId, geneName, chromosome, start, end, transcriptId, variants, dbSnps, knockoutType, filter, type, clinicalSignificance, populationFrequency, consequenceType, study | -#' | runRgaIndex | /{apiVersion}/analysis/clinical/rga/index/run | study, jobId, jobDescription, jobDependsOn, jobTags, auxiliarIndex, body[*] | +#' | runRgaIndex | /{apiVersion}/analysis/clinical/rga/index/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, auxiliarIndex, body[*] | #' | queryRgaIndividual | /{apiVersion}/analysis/clinical/rga/individual/query | include, exclude, limit, skip, count, sampleId, individualId, sex, phenotypes, disorders, numParents, geneId, geneName, chromosome, start, end, transcriptId, variants, dbSnps, knockoutType, filter, type, clinicalSignificance, populationFrequency, consequenceType, study | #' | summaryRgaIndividual | /{apiVersion}/analysis/clinical/rga/individual/summary | limit, skip, count, sampleId, individualId, sex, phenotypes, disorders, numParents, geneId, geneName, chromosome, start, end, transcriptId, variants, dbSnps, knockoutType, filter, type, clinicalSignificance, populationFrequency, consequenceType, study | #' | queryRgaVariant | /{apiVersion}/analysis/clinical/rga/variant/query | include, exclude, limit, skip, count, includeIndividual, skipIndividual, limitIndividual, sampleId, individualId, sex, phenotypes, disorders, numParents, geneId, geneName, chromosome, start, end, transcriptId, variants, dbSnps, knockoutType, filter, type, clinicalSignificance, populationFrequency, consequenceType, study | #' | summaryRgaVariant | /{apiVersion}/analysis/clinical/rga/variant/summary | limit, skip, count, sampleId, individualId, sex, phenotypes, disorders, numParents, geneId, geneName, chromosome, start, end, transcriptId, variants, dbSnps, knockoutType, filter, type, clinicalSignificance, populationFrequency, consequenceType, study | -#' | search | /{apiVersion}/analysis/clinical/search | include, exclude, limit, skip, count, flattenAnnotations, study, id, uuid, type, disorder, files, sample, individual, proband, probandSamples, family, familyMembers, familyMemberSamples, panels, locked, analystId, priority, flags, creationDate, modificationDate, dueDate, qualityControlSummary, release, status, internalStatus, annotation, deleted | +#' | search | /{apiVersion}/analysis/clinical/search | include, exclude, limit, skip, count, flattenAnnotations, study, id, uuid, type, disorder, files, sample, individual, proband, probandSamples, family, familyMembers, familyMemberSamples, panels, locked, analystId, priority, flags, creationDate, modificationDate, dueDate, qualityControlSummary, release, snapshot, status, internalStatus, annotation, deleted | #' | queryVariant | /{apiVersion}/analysis/clinical/variant/query | include, exclude, limit, skip, count, approximateCount, approximateCountSamplingSize, savedFilter, includeInterpretation, id, region, type, study, file, filter, qual, fileData, sample, sampleData, sampleAnnotation, cohort, cohortStatsRef, cohortStatsAlt, cohortStatsMaf, cohortStatsMgf, cohortStatsPass, missingAlleles, missingGenotypes, score, family, familyDisorder, familySegregation, familyMembers, familyProband, gene, ct, xref, biotype, proteinSubstitution, conservation, populationFrequencyAlt, populationFrequencyRef, populationFrequencyMaf, transcriptFlag, geneTraitId, go, expression, proteinKeyword, drug, functionalScore, clinical, clinicalSignificance, clinicalConfirmedStatus, customAnnotation, panel, panelModeOfInheritance, panelConfidence, panelRoleInCancer, panelFeatureType, panelIntersection, trait | #' | acl | /{apiVersion}/analysis/clinical/{clinicalAnalyses}/acl | clinicalAnalyses[*], study, member, silent | #' | delete | /{apiVersion}/analysis/clinical/{clinicalAnalyses}/delete | study, force, clinicalAnalyses[*] | #' | update | /{apiVersion}/analysis/clinical/{clinicalAnalyses}/update | include, exclude, clinicalAnalyses[*], study, commentsAction, flagsAction, analystsAction, filesAction, panelsAction, annotationSetsAction, includeResult, body[*] | #' | updateAnnotationSetsAnnotations | /{apiVersion}/analysis/clinical/{clinicalAnalysis}/annotationSets/{annotationSet}/annotations/update | clinicalAnalysis[*], study, annotationSet[*], action, body | -#' | info | /{apiVersion}/analysis/clinical/{clinicalAnalysis}/info | include, exclude, flattenAnnotations, clinicalAnalysis[*], study, deleted | +#' | info | /{apiVersion}/analysis/clinical/{clinicalAnalysis}/info | include, exclude, flattenAnnotations, clinicalAnalysis[*], study, version, deleted | #' | createInterpretation | /{apiVersion}/analysis/clinical/{clinicalAnalysis}/interpretation/create | include, exclude, clinicalAnalysis[*], study, setAs, includeResult, body[*] | #' | clearInterpretation | /{apiVersion}/analysis/clinical/{clinicalAnalysis}/interpretation/{interpretations}/clear | study, interpretations[*], clinicalAnalysis[*] | #' | deleteInterpretation | /{apiVersion}/analysis/clinical/{clinicalAnalysis}/interpretation/{interpretations}/delete | study, clinicalAnalysis[*], interpretations[*], setAsPrimary | @@ -61,12 +61,12 @@ #' [*]: Required parameter #' @export -setMethod("clinicalClient", "OpencgaR", function(OpencgaR, members, clinicalAnalyses, annotationSet, interpretations, interpretation, clinicalAnalysis, endpointName, params=NULL, ...) { +setMethod("clinicalClient", "OpencgaR", function(OpencgaR, annotationSet, clinicalAnalyses, clinicalAnalysis, interpretation, interpretations, members, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/analysis/clinical/acl/{members}/update: #' Update the set of permissions granted for the member. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param members Comma separated list of user or group IDs. #' @param action Action to be performed [ADD, SET, REMOVE or RESET]. Allowed values: ['SET ADD REMOVE RESET'] #' @param propagate Propagate permissions to related families, individuals, samples and files. @@ -77,7 +77,7 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, members, clinicalAnal #' @section Endpoint /{apiVersion}/analysis/clinical/annotationSets/load: #' Load annotation sets from a TSV file. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param variableSetId Variable set ID or name. #' @param path Path where the TSV file is located in OpenCGA or where it should be located. #' @param parents Flag indicating whether to create parent directories if they don't exist (only when TSV file was not previously associated). @@ -89,7 +89,7 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, members, clinicalAnal #' @section Endpoint /{apiVersion}/analysis/clinical/clinical/configuration/update: #' Update Clinical Analysis configuration. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param data Configuration params to update. updateClinicalConfiguration=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="clinical/clinical/configuration", subcategoryId=NULL, action="update", params=params, @@ -99,7 +99,7 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, members, clinicalAnal #' Create a new clinical analysis. #' @param include Fields included in the response, whole JSON path must be provided. #' @param exclude Fields excluded in the response, whole JSON path must be provided. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param skipCreateDefaultInterpretation Flag to skip creating and initialise an empty default primary interpretation (Id will be '{clinicalAnalysisId}.1'). This flag is only considered if no Interpretation object is passed. #' @param includeResult Flag indicating to include the created or updated document result in the response. #' @param data JSON containing clinical analysis information. @@ -108,7 +108,7 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, members, clinicalAnal #' @section Endpoint /{apiVersion}/analysis/clinical/distinct: #' Clinical Analysis distinct method. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param id Comma separated list of Clinical Analysis IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. #' @param uuid Comma separated list of Clinical Analysis UUIDs up to a maximum of 100. #' @param type Clinical Analysis type. @@ -131,6 +131,7 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, members, clinicalAnal #' @param dueDate Clinical Analysis due date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. #' @param qualityControlSummary Clinical Analysis quality control summary. #' @param release Release when it was created. + #' @param snapshot Snapshot value (Latest version of the entry in the specified release). #' @param status Filter by status. #' @param internalStatus Filter by internal status. #' @param annotation Annotation filters. Example: age>30;gender=FEMALE. For more information, please visit http://docs.opencb.org/display/opencga/AnnotationSets+1.4.0. @@ -141,9 +142,10 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, members, clinicalAnal #' @section Endpoint /{apiVersion}/analysis/clinical/interpretation/distinct: #' Interpretation distinct method. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param id Comma separated list of Interpretation IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. #' @param uuid Comma separated list of Interpretation UUIDs up to a maximum of 100. + #' @param name Comma separated list of Interpretation names up to a maximum of 100. #' @param clinicalAnalysisId Clinical Analysis id. #' @param analystId Analyst ID. #' @param methodName Interpretation method name. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. @@ -167,9 +169,10 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, members, clinicalAnal #' @param limit Number of results to be returned. #' @param skip Number of results to skip. #' @param sort Sort the results. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param id Comma separated list of Interpretation IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. #' @param uuid Comma separated list of Interpretation UUIDs up to a maximum of 100. + #' @param name Comma separated list of Interpretation names up to a maximum of 100. #' @param clinicalAnalysisId Clinical Analysis id. #' @param analystId Analyst ID. #' @param methodName Interpretation method name. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. @@ -190,7 +193,7 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, members, clinicalAnal #' @param include Fields included in the response, whole JSON path must be provided. #' @param exclude Fields excluded in the response, whole JSON path must be provided. #' @param interpretations Comma separated list of clinical interpretation IDs up to a maximum of 100. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param version Comma separated list of interpretation versions. 'all' to get all the interpretation versions. Not supported if multiple interpretation ids are provided. #' @param deleted Boolean to retrieve deleted entries. infoInterpretation=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, @@ -199,11 +202,14 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, members, clinicalAnal #' @section Endpoint /{apiVersion}/analysis/clinical/interpreter/cancerTiering/run: #' Run cancer tiering interpretation analysis. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Cancer tiering interpretation analysis params. runInterpreterCancerTiering=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="clinical/interpreter/cancerTiering", subcategoryId=NULL, action="run", params=params, @@ -211,23 +217,29 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, members, clinicalAnal #' @section Endpoint /{apiVersion}/analysis/clinical/interpreter/exomiser/run: #' Run exomiser interpretation analysis. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. - #' @param data Exomizer interpretation analysis params. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + #' @param data Exomiser interpretation analysis params. runInterpreterExomiser=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="clinical/interpreter/exomiser", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), #' @section Endpoint /{apiVersion}/analysis/clinical/interpreter/team/run: #' Run TEAM interpretation analysis. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data TEAM interpretation analysis params. runInterpreterTeam=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="clinical/interpreter/team", subcategoryId=NULL, action="run", params=params, @@ -235,11 +247,14 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, members, clinicalAnal #' @section Endpoint /{apiVersion}/analysis/clinical/interpreter/tiering/run: #' Run tiering interpretation analysis. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Tiering interpretation analysis params. runInterpreterTiering=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="clinical/interpreter/tiering", subcategoryId=NULL, action="run", params=params, @@ -247,16 +262,33 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, members, clinicalAnal #' @section Endpoint /{apiVersion}/analysis/clinical/interpreter/zetta/run: #' Run Zetta interpretation analysis. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Zetta interpretation analysis params. runInterpreterZetta=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="clinical/interpreter/zetta", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), + #' @section Endpoint /{apiVersion}/analysis/clinical/load: + #' Load clinical analyses from a file. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. + #' @param jobDescription Job description. + #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. + #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + #' @param data Parameters to load clinical analysis in OpenCGA catalog from a file. + load=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="clinical", + subcategoryId=NULL, action="load", params=params, httpMethod="POST", as.queryParam=NULL, ...), + #' @section Endpoint /{apiVersion}/analysis/clinical/rga/aggregationStats: #' RGA aggregation stats. #' @param limit Number of results to be returned. @@ -281,7 +313,7 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, members, clinicalAnal #' @param clinicalSignificance Filter by clinical significance. #' @param populationFrequency Filter by population frequency. #' @param consequenceType Filter by consequence type. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param field List of fields separated by semicolons, e.g.: clinicalSignificances;type. For nested fields use >>, e.g.: type>>clinicalSignificances;knockoutType. aggregationStatsRga=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="clinical/rga", subcategoryId=NULL, action="aggregationStats", params=params, @@ -317,7 +349,7 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, members, clinicalAnal #' @param clinicalSignificance Filter by clinical significance. #' @param populationFrequency Filter by population frequency. #' @param consequenceType Filter by consequence type. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. queryRgaGene=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="clinical/rga/gene", subcategoryId=NULL, action="query", params=params, httpMethod="GET", as.queryParam=NULL, ...), @@ -347,18 +379,21 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, members, clinicalAnal #' @param clinicalSignificance Filter by clinical significance. #' @param populationFrequency Filter by population frequency. #' @param consequenceType Filter by consequence type. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. summaryRgaGene=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="clinical/rga/gene", subcategoryId=NULL, action="summary", params=params, httpMethod="GET", as.queryParam=NULL, ...), #' @section Endpoint /{apiVersion}/analysis/clinical/rga/index/run: #' Generate Recessive Gene Analysis secondary index. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param auxiliarIndex Index auxiliar collection to improve performance assuming RGA is completely indexed. #' @param data Recessive Gene Analysis index params. runRgaIndex=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, @@ -392,7 +427,7 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, members, clinicalAnal #' @param clinicalSignificance Filter by clinical significance. #' @param populationFrequency Filter by population frequency. #' @param consequenceType Filter by consequence type. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. queryRgaIndividual=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="clinical/rga/individual", subcategoryId=NULL, action="query", params=params, httpMethod="GET", as.queryParam=NULL, ...), @@ -422,7 +457,7 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, members, clinicalAnal #' @param clinicalSignificance Filter by clinical significance. #' @param populationFrequency Filter by population frequency. #' @param consequenceType Filter by consequence type. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. summaryRgaIndividual=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="clinical/rga/individual", subcategoryId=NULL, action="summary", params=params, httpMethod="GET", as.queryParam=NULL, ...), @@ -457,7 +492,7 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, members, clinicalAnal #' @param clinicalSignificance Filter by clinical significance. #' @param populationFrequency Filter by population frequency. #' @param consequenceType Filter by consequence type. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. queryRgaVariant=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="clinical/rga/variant", subcategoryId=NULL, action="query", params=params, httpMethod="GET", as.queryParam=NULL, ...), @@ -487,7 +522,7 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, members, clinicalAnal #' @param clinicalSignificance Filter by clinical significance. #' @param populationFrequency Filter by population frequency. #' @param consequenceType Filter by consequence type. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. summaryRgaVariant=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="clinical/rga/variant", subcategoryId=NULL, action="summary", params=params, httpMethod="GET", as.queryParam=NULL, ...), @@ -500,7 +535,7 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, members, clinicalAnal #' @param skip Number of results to skip. #' @param count Get the total number of results matching the query. Deactivated by default. #' @param flattenAnnotations Flatten the annotations?. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param id Comma separated list of Clinical Analysis IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. #' @param uuid Comma separated list of Clinical Analysis UUIDs up to a maximum of 100. #' @param type Clinical Analysis type. @@ -523,6 +558,7 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, members, clinicalAnal #' @param dueDate Clinical Analysis due date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. #' @param qualityControlSummary Clinical Analysis quality control summary. #' @param release Release when it was created. + #' @param snapshot Snapshot value (Latest version of the entry in the specified release). #' @param status Filter by status. #' @param internalStatus Filter by internal status. #' @param annotation Annotation filters. Example: age>30;gender=FEMALE. For more information, please visit http://docs.opencb.org/display/opencga/AnnotationSets+1.4.0. @@ -544,7 +580,7 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, members, clinicalAnal #' @param id List of variant IDs in the format chrom:start:ref:alt, e.g. 19:7177679:C:T. #' @param region List of regions, these can be just a single chromosome name or regions in the format chr:start-end, e.g.: 2,3:100000-200000. #' @param type List of types, accepted values are SNV, MNV, INDEL, SV, COPY_NUMBER, COPY_NUMBER_LOSS, COPY_NUMBER_GAIN, INSERTION, DELETION, DUPLICATION, TANDEM_DUPLICATION, BREAKEND, e.g. SNV,INDEL. - #' @param study Filter variants from the given studies, these can be either the numeric ID or the alias with the format user@project:study. + #' @param study Filter variants from the given studies, these can be either the numeric ID or the alias with the format organization@project:study. #' @param file Filter variants from the files specified. This will set includeFile parameter when not provided. #' @param filter Specify the FILTER for any of the files. If 'file' filter is provided, will match the file and the filter. e.g.: PASS,LowGQX. #' @param qual Specify the QUAL for any of the files. If 'file' filter is provided, will match the file and the qual. e.g.: >123.4. @@ -600,7 +636,7 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, members, clinicalAnal #' @section Endpoint /{apiVersion}/analysis/clinical/{clinicalAnalyses}/acl: #' Returns the acl of the clinical analyses. If member is provided, it will only return the acl for the member. #' @param clinicalAnalyses Comma separated list of clinical analysis IDs or names up to a maximum of 100. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param member User or group ID. #' @param silent Boolean to retrieve all possible entries that are queried for, false to raise an exception whenever one of the entries looked for cannot be shown for whichever reason. acl=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="clinical", @@ -608,7 +644,7 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, members, clinicalAnal #' @section Endpoint /{apiVersion}/analysis/clinical/{clinicalAnalyses}/delete: #' Delete clinical analyses. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param force Force deletion if the ClinicalAnalysis contains interpretations or is locked. #' @param clinicalAnalyses Comma separated list of clinical analysis IDs or names up to a maximum of 100. delete=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="clinical", @@ -620,7 +656,7 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, members, clinicalAnal #' @param include Fields included in the response, whole JSON path must be provided. #' @param exclude Fields excluded in the response, whole JSON path must be provided. #' @param clinicalAnalyses Comma separated list of clinical analysis IDs. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param commentsAction Action to be performed if the array of comments is being updated. Allowed values: ['ADD REMOVE REPLACE'] #' @param flagsAction Action to be performed if the array of flags is being updated. Allowed values: ['ADD SET REMOVE'] #' @param analystsAction Action to be performed if the array of analysts is being updated. Allowed values: ['ADD SET REMOVE'] @@ -636,7 +672,7 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, members, clinicalAnal #' @section Endpoint /{apiVersion}/analysis/clinical/{clinicalAnalysis}/annotationSets/{annotationSet}/annotations/update: #' Update annotations from an annotationSet. #' @param clinicalAnalysis Clinical analysis ID. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param annotationSet AnnotationSet ID to be updated. #' @param action Action to be performed: ADD to add new annotations; REPLACE to replace the value of an already existing annotation; SET to set the new list of annotations removing any possible old annotations; REMOVE to remove some annotations; RESET to set some annotations to the default value configured in the corresponding variables of the VariableSet if any. Allowed values: ['ADD SET REMOVE RESET REPLACE'] #' @param data Json containing the map of annotations when the action is ADD, SET or REPLACE, a json with only the key 'remove' containing the comma separated variables to be removed as a value when the action is REMOVE or a json with only the key 'reset' containing the comma separated variables that will be set to the default value when the action is RESET. @@ -650,7 +686,8 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, members, clinicalAnal #' @param exclude Fields excluded in the response, whole JSON path must be provided. #' @param flattenAnnotations Flatten the annotations?. #' @param clinicalAnalysis Comma separated list of clinical analysis IDs or names up to a maximum of 100. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param version Comma separated list of clinical versions. 'all' to get all the clinical versions. Not supported if multiple clinical ids are provided. #' @param deleted Boolean to retrieve deleted entries. info=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="clinical", subcategoryId=clinicalAnalysis, action="info", params=params, httpMethod="GET", as.queryParam=NULL, @@ -661,7 +698,7 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, members, clinicalAnal #' @param include Fields included in the response, whole JSON path must be provided. #' @param exclude Fields excluded in the response, whole JSON path must be provided. #' @param clinicalAnalysis Clinical analysis ID. - #' @param study [[user@]project:]study id. + #' @param study [[organization@]project:]study id. #' @param setAs Set interpretation as. Allowed values: ['PRIMARY SECONDARY'] #' @param includeResult Flag indicating to include the created or updated document result in the response. #' @param data JSON containing clinical interpretation information. @@ -671,7 +708,7 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, members, clinicalAnal #' @section Endpoint /{apiVersion}/analysis/clinical/{clinicalAnalysis}/interpretation/{interpretations}/clear: #' Clear the fields of the main interpretation of the Clinical Analysis. - #' @param study [[user@]project:]study ID. + #' @param study [[organization@]project:]study ID. #' @param interpretations Interpretation IDs of the Clinical Analysis. #' @param clinicalAnalysis Clinical analysis ID. clearInterpretation=fetchOpenCGA(object=OpencgaR, category="analysis/clinical", categoryId=clinicalAnalysis, @@ -680,7 +717,7 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, members, clinicalAnal #' @section Endpoint /{apiVersion}/analysis/clinical/{clinicalAnalysis}/interpretation/{interpretations}/delete: #' Delete interpretation. - #' @param study [[user@]project:]study ID. + #' @param study [[organization@]project:]study ID. #' @param clinicalAnalysis Clinical analysis ID. #' @param interpretations Interpretation IDs of the Clinical Analysis. #' @param setAsPrimary Interpretation id to set as primary from the list of secondaries in case of deleting the actual primary one. @@ -690,7 +727,7 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, members, clinicalAnal #' @section Endpoint /{apiVersion}/analysis/clinical/{clinicalAnalysis}/interpretation/{interpretation}/revert: #' Revert to a previous interpretation version. - #' @param study [[user@]project:]study ID. + #' @param study [[organization@]project:]study ID. #' @param clinicalAnalysis Clinical analysis ID. #' @param interpretation Interpretation ID. #' @param version Version to revert to. @@ -702,7 +739,7 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, members, clinicalAnal #' Update interpretation fields. #' @param include Fields included in the response, whole JSON path must be provided. #' @param exclude Fields excluded in the response, whole JSON path must be provided. - #' @param study [[user@]project:]study ID. + #' @param study [[organization@]project:]study ID. #' @param primaryFindingsAction Action to be performed if the array of primary findings is being updated. Allowed values: ['ADD SET REMOVE REPLACE'] #' @param methodsAction Action to be performed if the array of methods is being updated. Allowed values: ['ADD SET REMOVE'] #' @param secondaryFindingsAction Action to be performed if the array of secondary findings is being updated. Allowed values: ['ADD SET REMOVE REPLACE'] @@ -722,7 +759,7 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, members, clinicalAnal #' @param include Fields included in the response, whole JSON path must be provided. #' @param exclude Fields excluded in the response, whole JSON path must be provided. #' @param clinicalAnalysis Clinical analysis ID. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param commentsAction Action to be performed if the array of comments is being updated. Allowed values: ['ADD REMOVE REPLACE'] #' @param supportingEvidencesAction Action to be performed if the array of supporting evidences is being updated. Allowed values: ['ADD SET REMOVE'] #' @param filesAction Action to be performed if the array of files is being updated. Allowed values: ['ADD SET REMOVE'] diff --git a/opencga-client/src/main/R/R/Cohort-methods.R b/opencga-client/src/main/R/R/Cohort-methods.R index 727cc6dcae3..e510a05eba8 100644 --- a/opencga-client/src/main/R/R/Cohort-methods.R +++ b/opencga-client/src/main/R/R/Cohort-methods.R @@ -2,7 +2,6 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-04-25 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -21,7 +20,6 @@ #' | endpointName | Endpoint WS | parameters accepted | #' | -- | :-- | --: | #' | updateAcl | /{apiVersion}/cohorts/acl/{members}/update | study, members[*], action[*], body[*] | -#' | aggregationStats | /{apiVersion}/cohorts/aggregationStats | study, type, creationYear, creationMonth, creationDay, creationDayOfWeek, numSamples, status, release, annotation, default, field | #' | loadAnnotationSets | /{apiVersion}/cohorts/annotationSets/load | study, variableSetId[*], path[*], parents, annotationSetId, body | #' | create | /{apiVersion}/cohorts/create | include, exclude, study, variableSet, variable, includeResult, body[*] | #' | distinct | /{apiVersion}/cohorts/distinct | study, id, name, uuid, type, creationDate, modificationDate, deleted, status, internalStatus, annotation, acl, samples, numSamples, release, field[*] | @@ -39,12 +37,12 @@ #' [*]: Required parameter #' @export -setMethod("cohortClient", "OpencgaR", function(OpencgaR, annotationSet, members, cohort, cohorts, endpointName, params=NULL, ...) { +setMethod("cohortClient", "OpencgaR", function(OpencgaR, annotationSet, cohort, cohorts, members, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/cohorts/acl/{members}/update: #' Update the set of permissions granted for the member. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param members Comma separated list of user or group ids. #' @param action Action to be performed [ADD, SET, REMOVE or RESET]. Allowed values: ['SET ADD REMOVE RESET'] #' @param data JSON containing the parameters to add ACLs. @@ -52,27 +50,9 @@ setMethod("cohortClient", "OpencgaR", function(OpencgaR, annotationSet, members, subcategoryId=members, action="update", params=params, httpMethod="POST", as.queryParam=c("action"), ...), - #' @section Endpoint /{apiVersion}/cohorts/aggregationStats: - #' Fetch catalog cohort stats. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. - #' @param type Type. - #' @param creationYear Creation year. - #' @param creationMonth Creation month (JANUARY, FEBRUARY...). - #' @param creationDay Creation day. - #' @param creationDayOfWeek Creation day of week (MONDAY, TUESDAY...). - #' @param numSamples Number of samples. - #' @param status Status. - #' @param release Release. - #' @param annotation Annotation filters. Example: age>30;gender=FEMALE. For more information, please visit http://docs.opencb.org/display/opencga/AnnotationSets+1.4.0. - #' @param default Calculate default stats. - #' @param field List of fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type;numSamples[0..10]:1. - aggregationStats=fetchOpenCGA(object=OpencgaR, category="cohorts", categoryId=NULL, subcategory=NULL, - subcategoryId=NULL, action="aggregationStats", params=params, httpMethod="GET", as.queryParam=NULL, - ...), - #' @section Endpoint /{apiVersion}/cohorts/annotationSets/load: #' Load annotation sets from a TSV file. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param variableSetId Variable set ID or name. #' @param path Path where the TSV file is located in OpenCGA or where it should be located. #' @param parents Flag indicating whether to create parent directories if they don't exist (only when TSV file was not previously associated). @@ -86,7 +66,7 @@ setMethod("cohortClient", "OpencgaR", function(OpencgaR, annotationSet, members, #' Create a cohort. #' @param include Fields included in the response, whole JSON path must be provided. #' @param exclude Fields excluded in the response, whole JSON path must be provided. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param variableSet Deprecated: Use /generate web service and filter by annotation. #' @param variable Deprecated: Use /generate web service and filter by annotation. #' @param includeResult Flag indicating to include the created or updated document result in the response. @@ -96,7 +76,7 @@ setMethod("cohortClient", "OpencgaR", function(OpencgaR, annotationSet, members, #' @section Endpoint /{apiVersion}/cohorts/distinct: #' Cohort distinct method. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param id Comma separated list of cohort IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. #' @param name Comma separated list of cohort names up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. #' @param uuid Comma separated list of cohort IDs up to a maximum of 100. @@ -119,7 +99,7 @@ setMethod("cohortClient", "OpencgaR", function(OpencgaR, annotationSet, members, #' Create a cohort based on a sample query. #' @param include Fields included in the response, whole JSON path must be provided. #' @param exclude Fields excluded in the response, whole JSON path must be provided. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param id Comma separated list sample IDs or UUIDs up to a maximum of 100. #' @param somatic Somatic sample. #' @param individualId Individual ID or UUID. @@ -146,7 +126,7 @@ setMethod("cohortClient", "OpencgaR", function(OpencgaR, annotationSet, members, #' @param skip Number of results to skip. #' @param count Get the total number of results matching the query. Deactivated by default. #' @param flattenAnnotations Flatten the annotations?. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param id Comma separated list of cohort IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. #' @param name Comma separated list of cohort names up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. #' @param uuid Comma separated list of cohort IDs up to a maximum of 100. @@ -167,7 +147,7 @@ setMethod("cohortClient", "OpencgaR", function(OpencgaR, annotationSet, members, #' @section Endpoint /{apiVersion}/cohorts/{cohorts}/acl: #' Return the acl of the cohort. If member is provided, it will only return the acl for the member. #' @param cohorts Comma separated list of cohort IDs or UUIDs up to a maximum of 100. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param member User or group id. #' @param silent Boolean to retrieve all possible entries that are queried for, false to raise an exception whenever one of the entries looked for cannot be shown for whichever reason. acl=fetchOpenCGA(object=OpencgaR, category="cohorts", categoryId=cohorts, subcategory=NULL, subcategoryId=NULL, @@ -175,7 +155,7 @@ setMethod("cohortClient", "OpencgaR", function(OpencgaR, annotationSet, members, #' @section Endpoint /{apiVersion}/cohorts/{cohorts}/delete: #' Delete cohorts. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param cohorts Comma separated list of cohort ids. delete=fetchOpenCGA(object=OpencgaR, category="cohorts", categoryId=cohorts, subcategory=NULL, subcategoryId=NULL, action="delete", params=params, httpMethod="DELETE", as.queryParam=NULL, ...), @@ -186,7 +166,7 @@ setMethod("cohortClient", "OpencgaR", function(OpencgaR, annotationSet, members, #' @param exclude Fields excluded in the response, whole JSON path must be provided. #' @param flattenAnnotations Flatten the annotations?. #' @param cohorts Comma separated list of cohort IDs or UUIDs up to a maximum of 100. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param deleted Boolean to retrieve deleted cohorts. info=fetchOpenCGA(object=OpencgaR, category="cohorts", categoryId=cohorts, subcategory=NULL, subcategoryId=NULL, action="info", params=params, httpMethod="GET", as.queryParam=NULL, ...), @@ -196,7 +176,7 @@ setMethod("cohortClient", "OpencgaR", function(OpencgaR, annotationSet, members, #' @param include Fields included in the response, whole JSON path must be provided. #' @param exclude Fields excluded in the response, whole JSON path must be provided. #' @param cohorts Comma separated list of cohort ids. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param samplesAction Action to be performed if the array of samples is being updated. Allowed values: ['ADD SET REMOVE'] #' @param annotationSetsAction Action to be performed if the array of annotationSets is being updated. Allowed values: ['ADD SET REMOVE'] #' @param includeResult Flag indicating to include the created or updated document result in the response. @@ -207,7 +187,7 @@ setMethod("cohortClient", "OpencgaR", function(OpencgaR, annotationSet, members, #' @section Endpoint /{apiVersion}/cohorts/{cohort}/annotationSets/{annotationSet}/annotations/update: #' Update annotations from an annotationSet. #' @param cohort Cohort ID. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param annotationSet AnnotationSet ID to be updated. #' @param action Action to be performed: ADD to add new annotations; REPLACE to replace the value of an already existing annotation; SET to set the new list of annotations removing any possible old annotations; REMOVE to remove some annotations; RESET to set some annotations to the default value configured in the corresponding variables of the VariableSet if any. Allowed values: ['ADD SET REMOVE RESET REPLACE'] #' @param data Json containing the map of annotations when the action is ADD, SET or REPLACE, a json with only the key 'remove' containing the comma separated variables to be removed as a value when the action is REMOVE or a json with only the key 'reset' containing the comma separated variables that will be set to the default value when the action is RESET. diff --git a/opencga-client/src/main/R/R/Family-methods.R b/opencga-client/src/main/R/R/Family-methods.R index 3111f2a51ed..de04b212196 100644 --- a/opencga-client/src/main/R/R/Family-methods.R +++ b/opencga-client/src/main/R/R/Family-methods.R @@ -2,7 +2,6 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-04-25 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -21,7 +20,6 @@ #' | endpointName | Endpoint WS | parameters accepted | #' | -- | :-- | --: | #' | updateAcl | /{apiVersion}/families/acl/{members}/update | study, members[*], action[*], propagate, body[*] | -#' | aggregationStats | /{apiVersion}/families/aggregationStats | study, creationYear, creationMonth, creationDay, creationDayOfWeek, status, phenotypes, release, version, numMembers, expectedSize, annotation, default, field | #' | loadAnnotationSets | /{apiVersion}/families/annotationSets/load | study, variableSetId[*], path[*], parents, annotationSetId, body | #' | create | /{apiVersion}/families/create | include, exclude, study, members, includeResult, body[*] | #' | distinct | /{apiVersion}/families/distinct | study, id, name, uuid, members, expectedSize, samples, phenotypes, disorders, creationDate, modificationDate, deleted, internalStatus, status, annotation, acl, release, snapshot, field[*] | @@ -38,12 +36,12 @@ #' [*]: Required parameter #' @export -setMethod("familyClient", "OpencgaR", function(OpencgaR, family, members, annotationSet, families, endpointName, params=NULL, ...) { +setMethod("familyClient", "OpencgaR", function(OpencgaR, annotationSet, families, family, members, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/families/acl/{members}/update: #' Update the set of permissions granted for the member. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param members Comma separated list of user or group ids. #' @param action Action to be performed [ADD, SET, REMOVE or RESET]. Allowed values: ['SET ADD REMOVE RESET'] #' @param propagate Propagate family permissions to related individuals and samples. Allowed values: ['NO YES YES_AND_VARIANT_VIEW'] @@ -52,29 +50,9 @@ setMethod("familyClient", "OpencgaR", function(OpencgaR, family, members, annota subcategoryId=members, action="update", params=params, httpMethod="POST", as.queryParam=c("action"), ...), - #' @section Endpoint /{apiVersion}/families/aggregationStats: - #' Fetch catalog family stats. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. - #' @param creationYear Creation year. - #' @param creationMonth Creation month (JANUARY, FEBRUARY...). - #' @param creationDay Creation day. - #' @param creationDayOfWeek Creation day of week (MONDAY, TUESDAY...). - #' @param status Status. - #' @param phenotypes Phenotypes. - #' @param release Release. - #' @param version Version. - #' @param numMembers Number of members. - #' @param expectedSize Expected size. - #' @param annotation Annotation filters. Example: age>30;gender=FEMALE. For more information, please visit http://docs.opencb.org/display/opencga/AnnotationSets+1.4.0. - #' @param default Calculate default stats. - #' @param field List of fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type;numSamples[0..10]:1. - aggregationStats=fetchOpenCGA(object=OpencgaR, category="families", categoryId=NULL, subcategory=NULL, - subcategoryId=NULL, action="aggregationStats", params=params, httpMethod="GET", as.queryParam=NULL, - ...), - #' @section Endpoint /{apiVersion}/families/annotationSets/load: #' Load annotation sets from a TSV file. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param variableSetId Variable set ID or name. #' @param path Path where the TSV file is located in OpenCGA or where it should be located. #' @param parents Flag indicating whether to create parent directories if they don't exist (only when TSV file was not previously associated). @@ -88,7 +66,7 @@ setMethod("familyClient", "OpencgaR", function(OpencgaR, family, members, annota #' Create family and the individual objects if they do not exist. #' @param include Fields included in the response, whole JSON path must be provided. #' @param exclude Fields excluded in the response, whole JSON path must be provided. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param members Comma separated list of member ids to be associated to the created family. #' @param includeResult Flag indicating to include the created or updated document result in the response. #' @param data JSON containing family information. @@ -97,7 +75,7 @@ setMethod("familyClient", "OpencgaR", function(OpencgaR, family, members, annota #' @section Endpoint /{apiVersion}/families/distinct: #' Family distinct method. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param id Comma separated list family IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. #' @param name Comma separated list family names up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. #' @param uuid Comma separated list family UUIDs up to a maximum of 100. @@ -127,7 +105,7 @@ setMethod("familyClient", "OpencgaR", function(OpencgaR, family, members, annota #' @param skip Number of results to skip. #' @param count Get the total number of results matching the query. Deactivated by default. #' @param flattenAnnotations Flatten the annotations?. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param id Comma separated list family IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. #' @param name Comma separated list family names up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. #' @param uuid Comma separated list family UUIDs up to a maximum of 100. @@ -151,7 +129,7 @@ setMethod("familyClient", "OpencgaR", function(OpencgaR, family, members, annota #' @section Endpoint /{apiVersion}/families/{families}/acl: #' Returns the acl of the families. If member is provided, it will only return the acl for the member. #' @param families Comma separated list of family IDs or names up to a maximum of 100. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param member User or group id. #' @param silent Boolean to retrieve all possible entries that are queried for, false to raise an exception whenever one of the entries looked for cannot be shown for whichever reason. acl=fetchOpenCGA(object=OpencgaR, category="families", categoryId=families, subcategory=NULL, @@ -159,7 +137,7 @@ setMethod("familyClient", "OpencgaR", function(OpencgaR, family, members, annota #' @section Endpoint /{apiVersion}/families/{families}/delete: #' Delete existing families. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param families Comma separated list of family ids. delete=fetchOpenCGA(object=OpencgaR, category="families", categoryId=families, subcategory=NULL, subcategoryId=NULL, action="delete", params=params, httpMethod="DELETE", as.queryParam=NULL, ...), @@ -170,7 +148,7 @@ setMethod("familyClient", "OpencgaR", function(OpencgaR, family, members, annota #' @param exclude Fields excluded in the response, whole JSON path must be provided. #' @param flattenAnnotations Flatten the annotations?. #' @param families Comma separated list of family IDs or names up to a maximum of 100. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param version Comma separated list of family versions. 'all' to get all the family versions. Not supported if multiple family ids are provided. #' @param deleted Boolean to retrieve deleted families. info=fetchOpenCGA(object=OpencgaR, category="families", categoryId=families, subcategory=NULL, @@ -181,7 +159,7 @@ setMethod("familyClient", "OpencgaR", function(OpencgaR, family, members, annota #' @param include Fields included in the response, whole JSON path must be provided. #' @param exclude Fields excluded in the response, whole JSON path must be provided. #' @param families Comma separated list of family ids. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param updateRoles Update the member roles within the family. #' @param updatePedigreeGraph Update the family pedigree graph. #' @param annotationSetsAction Action to be performed if the array of annotationSets is being updated. Allowed values: ['ADD SET REMOVE'] @@ -193,7 +171,7 @@ setMethod("familyClient", "OpencgaR", function(OpencgaR, family, members, annota #' @section Endpoint /{apiVersion}/families/{family}/annotationSets/{annotationSet}/annotations/update: #' Update annotations from an annotationSet. #' @param family Family id. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param annotationSet AnnotationSet ID to be updated. #' @param action Action to be performed: ADD to add new annotations; REPLACE to replace the value of an already existing annotation; SET to set the new list of annotations removing any possible old annotations; REMOVE to remove some annotations; RESET to set some annotations to the default value configured in the corresponding variables of the VariableSet if any. Allowed values: ['ADD SET REMOVE RESET REPLACE'] #' @param data Json containing the map of annotations when the action is ADD, SET or REPLACE, a json with only the key 'remove' containing the comma separated variables to be removed as a value when the action is REMOVE or a json with only the key 'reset' containing the comma separated variables that will be set to the default value when the action is RESET. diff --git a/opencga-client/src/main/R/R/File-methods.R b/opencga-client/src/main/R/R/File-methods.R index a0c896dcffa..da3baba3e62 100644 --- a/opencga-client/src/main/R/R/File-methods.R +++ b/opencga-client/src/main/R/R/File-methods.R @@ -2,7 +2,6 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-04-25 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -21,16 +20,15 @@ #' | endpointName | Endpoint WS | parameters accepted | #' | -- | :-- | --: | #' | updateAcl | /{apiVersion}/files/acl/{members}/update | study, members[*], action[*], body[*] | -#' | aggregationStats | /{apiVersion}/files/aggregationStats | study, name, type, format, bioformat, creationYear, creationMonth, creationDay, creationDayOfWeek, status, release, external, size, software, experiment, numSamples, numRelatedFiles, annotation, default, field | #' | loadAnnotationSets | /{apiVersion}/files/annotationSets/load | study, variableSetId[*], path[*], parents, annotationSetId, body | #' | bioformats | /{apiVersion}/files/bioformats | | #' | create | /{apiVersion}/files/create | study, parents, body[*] | #' | distinct | /{apiVersion}/files/distinct | study, id, uuid, name, path, uri, type, bioformat, format, external, status, internalStatus, internalVariantIndexStatus, softwareName, directory, creationDate, modificationDate, description, tags, size, sampleIds, jobId, annotation, acl, deleted, release, field[*] | -#' | fetch | /{apiVersion}/files/fetch | jobId, jobDescription, jobDependsOn, jobTags, study, body[*] | +#' | fetch | /{apiVersion}/files/fetch | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, study, body[*] | #' | formats | /{apiVersion}/files/formats | | #' | link | /{apiVersion}/files/link | study, parents, body[*] | -#' | runLink | /{apiVersion}/files/link/run | study, jobId, jobDependsOn, jobDescription, jobTags, body[*] | -#' | runPostlink | /{apiVersion}/files/postlink/run | study, jobId, jobDependsOn, jobDescription, jobTags, body[*] | +#' | runLink | /{apiVersion}/files/link/run | study, jobId, jobDependsOn, jobDescription, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runPostlink | /{apiVersion}/files/postlink/run | study, jobId, jobDependsOn, jobDescription, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | #' | search | /{apiVersion}/files/search | include, exclude, limit, skip, count, flattenAnnotations, study, id, uuid, name, path, uri, type, bioformat, format, external, status, internalStatus, internalVariantIndexStatus, softwareName, directory, creationDate, modificationDate, description, tags, size, sampleIds, jobId, annotation, acl, deleted, release | #' | upload | /{apiVersion}/files/upload | file, fileName, fileFormat, bioformat, checksum, study, relativeFilePath, description, parents | #' | acl | /{apiVersion}/files/{files}/acl | files[*], study, member, silent | @@ -55,12 +53,12 @@ #' [*]: Required parameter #' @export -setMethod("fileClient", "OpencgaR", function(OpencgaR, members, file, folder, annotationSet, files, endpointName, params=NULL, ...) { +setMethod("fileClient", "OpencgaR", function(OpencgaR, annotationSet, file, files, folder, members, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/files/acl/{members}/update: #' Update the set of permissions granted for the member. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param members Comma separated list of user or group ids. #' @param action Action to be performed [ADD, SET, REMOVE or RESET]. Allowed values: ['SET ADD REMOVE RESET'] #' @param data JSON containing the parameters to add ACLs. @@ -68,35 +66,9 @@ setMethod("fileClient", "OpencgaR", function(OpencgaR, members, file, folder, an subcategoryId=members, action="update", params=params, httpMethod="POST", as.queryParam=c("action"), ...), - #' @section Endpoint /{apiVersion}/files/aggregationStats: - #' Fetch catalog file stats. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. - #' @param name Name. - #' @param type Type. - #' @param format Format. - #' @param bioformat Bioformat. - #' @param creationYear Creation year. - #' @param creationMonth Creation month (JANUARY, FEBRUARY...). - #' @param creationDay Creation day. - #' @param creationDayOfWeek Creation day of week (MONDAY, TUESDAY...). - #' @param status Status. - #' @param release Release. - #' @param external External. - #' @param size Size. - #' @param software Software. - #' @param experiment Experiment. - #' @param numSamples Number of samples. - #' @param numRelatedFiles Number of related files. - #' @param annotation Annotation filters. Example: age>30;gender=FEMALE. For more information, please visit http://docs.opencb.org/display/opencga/AnnotationSets+1.4.0. - #' @param default Calculate default stats. - #' @param field List of fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type;numSamples[0..10]:1. - aggregationStats=fetchOpenCGA(object=OpencgaR, category="files", categoryId=NULL, subcategory=NULL, - subcategoryId=NULL, action="aggregationStats", params=params, httpMethod="GET", as.queryParam=NULL, - ...), - #' @section Endpoint /{apiVersion}/files/annotationSets/load: #' Load annotation sets from a TSV file. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param variableSetId Variable set ID or name. #' @param path Path where the TSV file is located in OpenCGA or where it should be located. #' @param parents Flag indicating whether to create parent directories if they don't exist (only when TSV file was not previously associated). @@ -114,7 +86,7 @@ setMethod("fileClient", "OpencgaR", function(OpencgaR, members, file, folder, an #' @section Endpoint /{apiVersion}/files/create: #' Create file or folder. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param parents Create the parent directories if they do not exist. #' @param data File parameters. create=fetchOpenCGA(object=OpencgaR, category="files", categoryId=NULL, subcategory=NULL, subcategoryId=NULL, @@ -122,7 +94,7 @@ setMethod("fileClient", "OpencgaR", function(OpencgaR, members, file, folder, an #' @section Endpoint /{apiVersion}/files/distinct: #' File distinct method. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param id Comma separated list of file IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. #' @param uuid Comma separated list file UUIDs up to a maximum of 100. #' @param name Comma separated list of file names. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. @@ -158,7 +130,10 @@ setMethod("fileClient", "OpencgaR", function(OpencgaR, members, file, folder, an #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param data Fetch parameters. fetch=fetchOpenCGA(object=OpencgaR, category="files", categoryId=NULL, subcategory=NULL, subcategoryId=NULL, action="fetch", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -171,7 +146,7 @@ setMethod("fileClient", "OpencgaR", function(OpencgaR, members, file, folder, an #' @section Endpoint /{apiVersion}/files/link: #' Link an external file into catalog. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param parents Create the parent directories if they do not exist. #' @param data File parameters. link=fetchOpenCGA(object=OpencgaR, category="files", categoryId=NULL, subcategory=NULL, subcategoryId=NULL, @@ -179,22 +154,28 @@ setMethod("fileClient", "OpencgaR", function(OpencgaR, members, file, folder, an #' @section Endpoint /{apiVersion}/files/link/run: #' Link an external file into catalog asynchronously. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobDescription Job description. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data File parameters. runLink=fetchOpenCGA(object=OpencgaR, category="files", categoryId=NULL, subcategory="link", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), #' @section Endpoint /{apiVersion}/files/postlink/run: #' Associate non-registered samples for files with high volumes of samples. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobDescription Job description. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data File parameters. runPostlink=fetchOpenCGA(object=OpencgaR, category="files", categoryId=NULL, subcategory="postlink", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -207,7 +188,7 @@ setMethod("fileClient", "OpencgaR", function(OpencgaR, members, file, folder, an #' @param skip Number of results to skip. #' @param count Get the total number of results matching the query. Deactivated by default. #' @param flattenAnnotations Boolean indicating to flatten the annotations. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param id Comma separated list of file IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. #' @param uuid Comma separated list file UUIDs up to a maximum of 100. #' @param name Comma separated list of file names. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. @@ -243,7 +224,7 @@ setMethod("fileClient", "OpencgaR", function(OpencgaR, members, file, folder, an #' @param fileFormat File format. Allowed values: ['VCF BCF GVCF TBI BIGWIG SAM BAM BAI CRAM CRAI FASTQ FASTA PED TAB_SEPARATED_VALUES COMMA_SEPARATED_VALUES XML PROTOCOL_BUFFER JSON AVRO PARQUET PDF IMAGE PLAIN BINARY NONE UNKNOWN'] #' @param bioformat File bioformat. Allowed values: ['MICROARRAY_EXPRESSION_ONECHANNEL_AGILENT MICROARRAY_EXPRESSION_ONECHANNEL_AFFYMETRIX MICROARRAY_EXPRESSION_ONECHANNEL_GENEPIX MICROARRAY_EXPRESSION_TWOCHANNELS_AGILENT MICROARRAY_EXPRESSION_TWOCHANNELS_GENEPIX DATAMATRIX_EXPRESSION IDLIST IDLIST_RANKED ANNOTATION_GENEVSANNOTATION OTHER_NEWICK OTHER_BLAST OTHER_INTERACTION OTHER_GENOTYPE OTHER_PLINK OTHER_VCF OTHER_PED VCF4 VARIANT ALIGNMENT COVERAGE SEQUENCE PEDIGREE REFERENCE_GENOME NONE UNKNOWN'] #' @param checksum Expected MD5 file checksum. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param relativeFilePath Path within catalog where the file will be located (default: root folder). #' @param description description. #' @param parents Create the parent directories if they do not exist. @@ -253,7 +234,7 @@ setMethod("fileClient", "OpencgaR", function(OpencgaR, members, file, folder, an #' @section Endpoint /{apiVersion}/files/{files}/acl: #' Return the acl defined for the file or folder. If member is provided, it will only return the acl for the member. #' @param files Comma separated list of file IDs or names up to a maximum of 100. - #' @param study Comma separated list of Studies [[user@]project:]study where study and project can be either the ID or UUID up to a maximum of 100. + #' @param study Comma separated list of Studies [[organization@]project:]study where study and project can be either the ID or UUID up to a maximum of 100. #' @param member User or group id. #' @param silent Boolean to retrieve all possible entries that are queried for, false to raise an exception whenever one of the entries looked for cannot be shown for whichever reason. acl=fetchOpenCGA(object=OpencgaR, category="files", categoryId=files, subcategory=NULL, subcategoryId=NULL, @@ -261,7 +242,7 @@ setMethod("fileClient", "OpencgaR", function(OpencgaR, members, file, folder, an #' @section Endpoint /{apiVersion}/files/{files}/delete: #' Delete existing files and folders. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param files Comma separated list of file ids, names or paths. #' @param skipTrash Skip trash and delete the files/folders from disk directly (CANNOT BE RECOVERED). delete=fetchOpenCGA(object=OpencgaR, category="files", categoryId=files, subcategory=NULL, subcategoryId=NULL, @@ -273,14 +254,14 @@ setMethod("fileClient", "OpencgaR", function(OpencgaR, members, file, folder, an #' @param exclude Fields excluded in the response, whole JSON path must be provided. #' @param flattenAnnotations Flatten the annotations?. #' @param files Comma separated list of file IDs or names up to a maximum of 100. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param deleted Boolean to retrieve deleted files. info=fetchOpenCGA(object=OpencgaR, category="files", categoryId=files, subcategory=NULL, subcategoryId=NULL, action="info", params=params, httpMethod="GET", as.queryParam=NULL, ...), #' @section Endpoint /{apiVersion}/files/{files}/unlink: #' Unlink linked files and folders. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param files Comma separated list of file ids, names or paths. unlink=fetchOpenCGA(object=OpencgaR, category="files", categoryId=files, subcategory=NULL, subcategoryId=NULL, action="unlink", params=params, httpMethod="DELETE", as.queryParam=NULL, ...), @@ -290,7 +271,7 @@ setMethod("fileClient", "OpencgaR", function(OpencgaR, members, file, folder, an #' @param include Fields included in the response, whole JSON path must be provided. #' @param exclude Fields excluded in the response, whole JSON path must be provided. #' @param files Comma separated list of file ids, names or paths. Paths must be separated by : instead of /. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param sampleIdsAction Action to be performed if the array of samples is being updated. Allowed values: ['ADD SET REMOVE'] #' @param annotationSetsAction Action to be performed if the array of annotationSets is being updated. Allowed values: ['ADD SET REMOVE'] #' @param relatedFilesAction Action to be performed if the array of relatedFiles is being updated. Allowed values: ['ADD SET REMOVE'] @@ -302,7 +283,7 @@ setMethod("fileClient", "OpencgaR", function(OpencgaR, members, file, folder, an #' @section Endpoint /{apiVersion}/files/{file}/annotationSets/{annotationSet}/annotations/update: #' Update annotations from an annotationSet. #' @param file File id, name or path. Paths must be separated by : instead of /. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param annotationSet AnnotationSet ID to be updated. #' @param action Action to be performed: ADD to add new annotations; REPLACE to replace the value of an already existing annotation; SET to set the new list of annotations removing any possible old annotations; REMOVE to remove some annotations; RESET to set some annotations to the default value configured in the corresponding variables of the VariableSet if any. Allowed values: ['ADD SET REMOVE RESET REPLACE'] #' @param data Json containing the map of annotations when the action is ADD, SET or REPLACE, a json with only the key 'remove' containing the comma separated variables to be removed as a value when the action is REMOVE or a json with only the key 'reset' containing the comma separated variables that will be set to the default value when the action is RESET. @@ -313,14 +294,14 @@ setMethod("fileClient", "OpencgaR", function(OpencgaR, members, file, folder, an #' @section Endpoint /{apiVersion}/files/{file}/download: #' Download file. #' @param file File id, name or path. Paths must be separated by : instead of /. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. download=fetchOpenCGA(object=OpencgaR, category="files", categoryId=file, subcategory=NULL, subcategoryId=NULL, action="download", params=params, httpMethod="GET", as.queryParam=NULL, ...), #' @section Endpoint /{apiVersion}/files/{file}/grep: #' Filter lines of the file containing the pattern. #' @param file File uuid, id, or name. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param pattern String pattern. #' @param ignoreCase Flag to perform a case insensitive search. #' @param maxCount Stop reading a file after 'n' matching lines. 0 means no limit. @@ -330,7 +311,7 @@ setMethod("fileClient", "OpencgaR", function(OpencgaR, members, file, folder, an #' @section Endpoint /{apiVersion}/files/{file}/head: #' Show the first lines of a file (up to a limit). #' @param file File uuid, id, or name. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param offset Starting byte from which the file will be read. #' @param lines Maximum number of lines to be returned up to a maximum of 1000. head=fetchOpenCGA(object=OpencgaR, category="files", categoryId=file, subcategory=NULL, subcategoryId=NULL, @@ -339,7 +320,7 @@ setMethod("fileClient", "OpencgaR", function(OpencgaR, members, file, folder, an #' @section Endpoint /{apiVersion}/files/{file}/image: #' Obtain the base64 content of an image. #' @param file File ID. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. image=fetchOpenCGA(object=OpencgaR, category="files", categoryId=file, subcategory=NULL, subcategoryId=NULL, action="image", params=params, httpMethod="GET", as.queryParam=NULL, ...), @@ -348,7 +329,7 @@ setMethod("fileClient", "OpencgaR", function(OpencgaR, members, file, folder, an #' @param include Fields included in the response, whole JSON path must be provided. #' @param exclude Fields excluded in the response, whole JSON path must be provided. #' @param file File id, UUID or name. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param data Parameters to modify. move=fetchOpenCGA(object=OpencgaR, category="files", categoryId=file, subcategory=NULL, subcategoryId=NULL, action="move", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -356,14 +337,14 @@ setMethod("fileClient", "OpencgaR", function(OpencgaR, members, file, folder, an #' @section Endpoint /{apiVersion}/files/{file}/refresh: #' Refresh metadata from the selected file or folder. Return updated files. #' @param file File id, name or path. Paths must be separated by : instead of /. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. refresh=fetchOpenCGA(object=OpencgaR, category="files", categoryId=file, subcategory=NULL, subcategoryId=NULL, action="refresh", params=params, httpMethod="GET", as.queryParam=NULL, ...), #' @section Endpoint /{apiVersion}/files/{file}/tail: #' Show the last lines of a file (up to a limit). #' @param file File uuid, id, or name. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param lines Maximum number of lines to be returned up to a maximum of 1000. tail=fetchOpenCGA(object=OpencgaR, category="files", categoryId=file, subcategory=NULL, subcategoryId=NULL, action="tail", params=params, httpMethod="GET", as.queryParam=NULL, ...), @@ -376,7 +357,7 @@ setMethod("fileClient", "OpencgaR", function(OpencgaR, members, file, folder, an #' @param skip Number of results to skip. #' @param count Get the total number of results matching the query. Deactivated by default. #' @param folder Folder ID, name or path. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. list=fetchOpenCGA(object=OpencgaR, category="files", categoryId=folder, subcategory=NULL, subcategoryId=NULL, action="list", params=params, httpMethod="GET", as.queryParam=NULL, ...), @@ -385,7 +366,7 @@ setMethod("fileClient", "OpencgaR", function(OpencgaR, members, file, folder, an #' @param include Fields included in the response, whole JSON path must be provided. #' @param exclude Fields excluded in the response, whole JSON path must be provided. #' @param folder Folder id or name. Paths must be separated by : instead of /. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param maxDepth Maximum depth to get files from. tree=fetchOpenCGA(object=OpencgaR, category="files", categoryId=folder, subcategory=NULL, subcategoryId=NULL, action="tree", params=params, httpMethod="GET", as.queryParam=NULL, ...), diff --git a/opencga-client/src/main/R/R/GA4GH-methods.R b/opencga-client/src/main/R/R/GA4GH-methods.R index 2406d77ff2c..a6053502605 100644 --- a/opencga-client/src/main/R/R/GA4GH-methods.R +++ b/opencga-client/src/main/R/R/GA4GH-methods.R @@ -2,7 +2,6 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-04-25 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -31,7 +30,7 @@ #' [*]: Required parameter #' @export -setMethod("ga4ghClient", "OpencgaR", function(OpencgaR, study, file, endpointName, params=NULL, ...) { +setMethod("ga4ghClient", "OpencgaR", function(OpencgaR, file, study, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/ga4gh/reads/search: @@ -43,7 +42,7 @@ setMethod("ga4ghClient", "OpencgaR", function(OpencgaR, study, file, endpointNam #' @section Endpoint /{apiVersion}/ga4gh/reads/{study}/{file}: #' Fetch alignment files using HTSget protocol. #' @param file File id, name or path. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param referenceName Reference sequence name (Example: 'chr1', '1' or 'chrX'. #' @param start The start position of the range on the reference, 0-based, inclusive. #' @param end The end position of the range on the reference, 0-based, exclusive. diff --git a/opencga-client/src/main/R/R/Individual-methods.R b/opencga-client/src/main/R/R/Individual-methods.R index b48136ae582..9a3bc092adc 100644 --- a/opencga-client/src/main/R/R/Individual-methods.R +++ b/opencga-client/src/main/R/R/Individual-methods.R @@ -2,7 +2,6 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-04-25 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -21,7 +20,6 @@ #' | endpointName | Endpoint WS | parameters accepted | #' | -- | :-- | --: | #' | updateAcl | /{apiVersion}/individuals/acl/{members}/update | study, members[*], action[*], propagate, body[*] | -#' | aggregationStats | /{apiVersion}/individuals/aggregationStats | study, hasFather, hasMother, sex, karyotypicSex, ethnicity, population, creationYear, creationMonth, creationDay, creationDayOfWeek, status, lifeStatus, phenotypes, numSamples, parentalConsanguinity, release, version, annotation, default, field | #' | loadAnnotationSets | /{apiVersion}/individuals/annotationSets/load | study, variableSetId[*], path[*], parents, annotationSetId, body | #' | create | /{apiVersion}/individuals/create | include, exclude, study, samples, includeResult, body[*] | #' | distinct | /{apiVersion}/individuals/distinct | study, id, uuid, name, familyIds, father, mother, samples, sex, ethnicity, dateOfBirth, disorders, phenotypes, populationName, populationSubpopulation, karyotypicSex, lifeStatus, internalStatus, status, deleted, creationDate, modificationDate, annotation, acl, release, snapshot, field[*] | @@ -39,12 +37,12 @@ #' [*]: Required parameter #' @export -setMethod("individualClient", "OpencgaR", function(OpencgaR, annotationSet, individual, members, individuals, endpointName, params=NULL, ...) { +setMethod("individualClient", "OpencgaR", function(OpencgaR, annotationSet, individual, individuals, members, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/individuals/acl/{members}/update: #' Update the set of permissions granted for the member. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param members Comma separated list of user or group ids. #' @param action Action to be performed [ADD, SET, REMOVE or RESET]. Allowed values: ['SET ADD REMOVE RESET'] #' @param propagate Propagate individual permissions to related samples. @@ -53,36 +51,9 @@ setMethod("individualClient", "OpencgaR", function(OpencgaR, annotationSet, indi subcategoryId=members, action="update", params=params, httpMethod="POST", as.queryParam=c("action"), ...), - #' @section Endpoint /{apiVersion}/individuals/aggregationStats: - #' Fetch catalog individual stats. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. - #' @param hasFather Has father. - #' @param hasMother Has mother. - #' @param sex Sex. - #' @param karyotypicSex Karyotypic sex. - #' @param ethnicity Ethnicity. - #' @param population Population. - #' @param creationYear Creation year. - #' @param creationMonth Creation month (JANUARY, FEBRUARY...). - #' @param creationDay Creation day. - #' @param creationDayOfWeek Creation day of week (MONDAY, TUESDAY...). - #' @param status Status. - #' @param lifeStatus Life status. - #' @param phenotypes Phenotypes. - #' @param numSamples Number of samples. - #' @param parentalConsanguinity Parental consanguinity. - #' @param release Release. - #' @param version Version. - #' @param annotation Annotation filters. Example: age>30;gender=FEMALE. For more information, please visit http://docs.opencb.org/display/opencga/AnnotationSets+1.4.0. - #' @param default Calculate default stats. - #' @param field List of fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type;numSamples[0..10]:1. - aggregationStats=fetchOpenCGA(object=OpencgaR, category="individuals", categoryId=NULL, subcategory=NULL, - subcategoryId=NULL, action="aggregationStats", params=params, httpMethod="GET", as.queryParam=NULL, - ...), - #' @section Endpoint /{apiVersion}/individuals/annotationSets/load: #' Load annotation sets from a TSV file. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param variableSetId Variable set ID or name. #' @param path Path where the TSV file is located in OpenCGA or where it should be located. #' @param parents Flag indicating whether to create parent directories if they don't exist (only when TSV file was not previously associated). @@ -96,7 +67,7 @@ setMethod("individualClient", "OpencgaR", function(OpencgaR, annotationSet, indi #' Create individual. #' @param include Fields included in the response, whole JSON path must be provided. #' @param exclude Fields excluded in the response, whole JSON path must be provided. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param samples Comma separated list of sample ids to be associated to the created individual. #' @param includeResult Flag indicating to include the created or updated document result in the response. #' @param data JSON containing individual information. @@ -105,7 +76,7 @@ setMethod("individualClient", "OpencgaR", function(OpencgaR, annotationSet, indi #' @section Endpoint /{apiVersion}/individuals/distinct: #' Individual distinct method. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param id Comma separated list individual IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. #' @param uuid Comma separated list individual UUIDs up to a maximum of 100. #' @param name Comma separated list individual names up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. @@ -143,7 +114,7 @@ setMethod("individualClient", "OpencgaR", function(OpencgaR, annotationSet, indi #' @param skip Number of results to skip. #' @param count Get the total number of results matching the query. Deactivated by default. #' @param flattenAnnotations Flatten the annotations?. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param id Comma separated list individual IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. #' @param uuid Comma separated list individual UUIDs up to a maximum of 100. #' @param name Comma separated list individual names up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. @@ -175,7 +146,7 @@ setMethod("individualClient", "OpencgaR", function(OpencgaR, annotationSet, indi #' @section Endpoint /{apiVersion}/individuals/{individuals}/acl: #' Return the acl of the individual. If member is provided, it will only return the acl for the member. #' @param individuals Comma separated list of individual IDs, names or UUIDs up to a maximum of 100. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param member User or group id. #' @param silent Boolean to retrieve all possible entries that are queried for, false to raise an exception whenever one of the entries looked for cannot be shown for whichever reason. acl=fetchOpenCGA(object=OpencgaR, category="individuals", categoryId=individuals, subcategory=NULL, @@ -184,7 +155,7 @@ setMethod("individualClient", "OpencgaR", function(OpencgaR, annotationSet, indi #' @section Endpoint /{apiVersion}/individuals/{individuals}/delete: #' Delete existing individuals. #' @param force Force the deletion of individuals that already belong to families. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param individuals Comma separated list of individual ids. delete=fetchOpenCGA(object=OpencgaR, category="individuals", categoryId=individuals, subcategory=NULL, subcategoryId=NULL, action="delete", params=params, httpMethod="DELETE", as.queryParam=NULL, ...), @@ -195,7 +166,7 @@ setMethod("individualClient", "OpencgaR", function(OpencgaR, annotationSet, indi #' @param exclude Fields excluded in the response, whole JSON path must be provided. #' @param flattenAnnotations Flatten the annotations?. #' @param individuals Comma separated list of individual IDs, names or UUIDs up to a maximum of 100. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param version Comma separated list of individual versions. 'all' to get all the individual versions. Not supported if multiple individual ids are provided. #' @param deleted Boolean to retrieve deleted individuals. info=fetchOpenCGA(object=OpencgaR, category="individuals", categoryId=individuals, subcategory=NULL, @@ -206,7 +177,7 @@ setMethod("individualClient", "OpencgaR", function(OpencgaR, annotationSet, indi #' @param include Fields included in the response, whole JSON path must be provided. #' @param exclude Fields excluded in the response, whole JSON path must be provided. #' @param individuals Comma separated list of individual ids. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param samplesAction Action to be performed if the array of samples is being updated. Allowed values: ['ADD SET REMOVE'] #' @param phenotypesAction Action to be performed if the array of phenotypes is being updated [SET, ADD, REMOVE]. Allowed values: ['ADD SET REMOVE'] #' @param disordersAction Action to be performed if the array of disorders is being updated [SET, ADD, REMOVE]. Allowed values: ['ADD SET REMOVE'] @@ -219,7 +190,7 @@ setMethod("individualClient", "OpencgaR", function(OpencgaR, annotationSet, indi #' @section Endpoint /{apiVersion}/individuals/{individual}/annotationSets/{annotationSet}/annotations/update: #' Update annotations from an annotationSet. #' @param individual Individual ID, name or UUID. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param annotationSet AnnotationSet ID to be updated. #' @param action Action to be performed: ADD to add new annotations; REPLACE to replace the value of an already existing annotation; SET to set the new list of annotations removing any possible old annotations; REMOVE to remove some annotations; RESET to set some annotations to the default value configured in the corresponding variables of the VariableSet if any. Allowed values: ['ADD SET REMOVE RESET REPLACE'] #' @param data Json containing the map of annotations when the action is ADD, SET or REPLACE, a json with only the key 'remove' containing the comma separated variables to be removed as a value when the action is REMOVE or a json with only the key 'reset' containing the comma separated variables that will be set to the default value when the action is RESET. @@ -233,7 +204,7 @@ setMethod("individualClient", "OpencgaR", function(OpencgaR, annotationSet, indi #' @param exclude Fields excluded in the response, whole JSON path must be provided. #' @param flattenAnnotations Flatten the annotations?. #' @param individual Individual ID, name or UUID. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param degree Pedigree degree. relatives=fetchOpenCGA(object=OpencgaR, category="individuals", categoryId=individual, subcategory=NULL, subcategoryId=NULL, action="relatives", params=params, httpMethod="GET", as.queryParam=NULL, ...), diff --git a/opencga-client/src/main/R/R/Job-methods.R b/opencga-client/src/main/R/R/Job-methods.R index b6c47021122..32df3f06506 100644 --- a/opencga-client/src/main/R/R/Job-methods.R +++ b/opencga-client/src/main/R/R/Job-methods.R @@ -2,7 +2,6 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-04-25 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -21,16 +20,16 @@ #' | endpointName | Endpoint WS | parameters accepted | #' | -- | :-- | --: | #' | updateAcl | /{apiVersion}/jobs/acl/{members}/update | members[*], action[*], body[*] | -#' | aggregationStats | /{apiVersion}/jobs/aggregationStats | study, toolId, toolScope, toolType, toolResource, userId, priority, tags, executorId, executorFramework, creationYear, creationMonth, creationDay, creationDayOfWeek, status, release, default, field | #' | create | /{apiVersion}/jobs/create | study, body[*] | #' | distinct | /{apiVersion}/jobs/distinct | study, otherStudies, id, uuid, toolId, toolType, userId, priority, status, internalStatus, creationDate, modificationDate, visited, tags, input, output, acl, release, deleted, field[*] | -#' | retry | /{apiVersion}/jobs/retry | jobId, jobDescription, jobDependsOn, jobTags, study, body[*] | +#' | retry | /{apiVersion}/jobs/retry | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, study, body[*] | #' | search | /{apiVersion}/jobs/search | include, exclude, limit, skip, count, study, otherStudies, id, uuid, toolId, toolType, userId, priority, status, internalStatus, creationDate, modificationDate, visited, tags, input, output, acl, release, deleted | #' | top | /{apiVersion}/jobs/top | limit, study, internalStatus, priority, userId, toolId | #' | acl | /{apiVersion}/jobs/{jobs}/acl | jobs[*], member, silent | #' | delete | /{apiVersion}/jobs/{jobs}/delete | study, jobs[*] | #' | info | /{apiVersion}/jobs/{jobs}/info | include, exclude, jobs[*], study, deleted | #' | update | /{apiVersion}/jobs/{jobs}/update | include, exclude, jobs[*], study, includeResult, body | +#' | kill | /{apiVersion}/jobs/{job}/kill | job[*], study | #' | headLog | /{apiVersion}/jobs/{job}/log/head | job[*], study, offset, lines, type | #' | tailLog | /{apiVersion}/jobs/{job}/log/tail | job[*], study, lines, type | #' @@ -40,7 +39,7 @@ #' [*]: Required parameter #' @export -setMethod("jobClient", "OpencgaR", function(OpencgaR, jobs, members, job, endpointName, params=NULL, ...) { +setMethod("jobClient", "OpencgaR", function(OpencgaR, job, jobs, members, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/jobs/acl/{members}/update: @@ -52,40 +51,16 @@ setMethod("jobClient", "OpencgaR", function(OpencgaR, jobs, members, job, endpoi subcategoryId=members, action="update", params=params, httpMethod="POST", as.queryParam=c("action"), ...), - #' @section Endpoint /{apiVersion}/jobs/aggregationStats: - #' Fetch catalog job stats. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. - #' @param toolId Tool id. - #' @param toolScope Tool scope. - #' @param toolType Tool type. - #' @param toolResource Tool resource. - #' @param userId User id. - #' @param priority Priority. - #' @param tags Tags. - #' @param executorId Executor id. - #' @param executorFramework Executor framework. - #' @param creationYear Creation year. - #' @param creationMonth Creation month (JANUARY, FEBRUARY...). - #' @param creationDay Creation day. - #' @param creationDayOfWeek Creation day of week (MONDAY, TUESDAY...). - #' @param status Status. - #' @param release Release. - #' @param default Calculate default stats. - #' @param field List of fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type;numSamples[0..10]:1. - aggregationStats=fetchOpenCGA(object=OpencgaR, category="jobs", categoryId=NULL, subcategory=NULL, - subcategoryId=NULL, action="aggregationStats", params=params, httpMethod="GET", as.queryParam=NULL, - ...), - #' @section Endpoint /{apiVersion}/jobs/create: #' Register an executed job with POST method. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param data job. create=fetchOpenCGA(object=OpencgaR, category="jobs", categoryId=NULL, subcategory=NULL, subcategoryId=NULL, action="create", params=params, httpMethod="POST", as.queryParam=NULL, ...), #' @section Endpoint /{apiVersion}/jobs/distinct: #' Job distinct method. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param otherStudies Flag indicating the entries being queried can belong to any related study, not just the primary one. #' @param id Comma separated list of job IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. #' @param uuid Comma separated list of job UUIDs up to a maximum of 100. @@ -114,7 +89,8 @@ setMethod("jobClient", "OpencgaR", function(OpencgaR, jobs, members, job, endpoi #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param data job. retry=fetchOpenCGA(object=OpencgaR, category="jobs", categoryId=NULL, subcategory=NULL, subcategoryId=NULL, action="retry", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -126,7 +102,7 @@ setMethod("jobClient", "OpencgaR", function(OpencgaR, jobs, members, job, endpoi #' @param limit Number of results to be returned. #' @param skip Number of results to skip. #' @param count Get the total number of results matching the query. Deactivated by default. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param otherStudies Flag indicating the entries being queried can belong to any related study, not just the primary one. #' @param id Comma separated list of job IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. #' @param uuid Comma separated list of job UUIDs up to a maximum of 100. @@ -151,7 +127,7 @@ setMethod("jobClient", "OpencgaR", function(OpencgaR, jobs, members, job, endpoi #' @section Endpoint /{apiVersion}/jobs/top: #' Provide a summary of the running jobs. #' @param limit Maximum number of jobs to be returned. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param internalStatus Filter by internal status. #' @param priority Priority of the job. #' @param userId User that created the job. @@ -169,7 +145,7 @@ setMethod("jobClient", "OpencgaR", function(OpencgaR, jobs, members, job, endpoi #' @section Endpoint /{apiVersion}/jobs/{jobs}/delete: #' Delete existing jobs. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param jobs Comma separated list of job ids. delete=fetchOpenCGA(object=OpencgaR, category="jobs", categoryId=jobs, subcategory=NULL, subcategoryId=NULL, action="delete", params=params, httpMethod="DELETE", as.queryParam=NULL, ...), @@ -179,7 +155,7 @@ setMethod("jobClient", "OpencgaR", function(OpencgaR, jobs, members, job, endpoi #' @param include Fields included in the response, whole JSON path must be provided. #' @param exclude Fields excluded in the response, whole JSON path must be provided. #' @param jobs Comma separated list of job IDs or UUIDs up to a maximum of 100. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param deleted Boolean to retrieve deleted jobs. info=fetchOpenCGA(object=OpencgaR, category="jobs", categoryId=jobs, subcategory=NULL, subcategoryId=NULL, action="info", params=params, httpMethod="GET", as.queryParam=NULL, ...), @@ -189,16 +165,23 @@ setMethod("jobClient", "OpencgaR", function(OpencgaR, jobs, members, job, endpoi #' @param include Fields included in the response, whole JSON path must be provided. #' @param exclude Fields excluded in the response, whole JSON path must be provided. #' @param jobs Comma separated list of job IDs or UUIDs up to a maximum of 100. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param includeResult Flag indicating to include the created or updated document result in the response. #' @param data body. update=fetchOpenCGA(object=OpencgaR, category="jobs", categoryId=jobs, subcategory=NULL, subcategoryId=NULL, action="update", params=params, httpMethod="POST", as.queryParam=NULL, ...), + #' @section Endpoint /{apiVersion}/jobs/{job}/kill: + #' Send a signal to kill a pending or running job. + #' @param job Job ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + kill=fetchOpenCGA(object=OpencgaR, category="jobs", categoryId=job, subcategory=NULL, subcategoryId=NULL, + action="kill", params=params, httpMethod="POST", as.queryParam=NULL, ...), + #' @section Endpoint /{apiVersion}/jobs/{job}/log/head: #' Show the first lines of a log file (up to a limit). #' @param job Job ID or UUID. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param offset Starting byte from which the file will be read. #' @param lines Maximum number of lines to be returned up to a maximum of 1000. #' @param type Log file to be shown (stdout or stderr). @@ -208,7 +191,7 @@ setMethod("jobClient", "OpencgaR", function(OpencgaR, jobs, members, job, endpoi #' @section Endpoint /{apiVersion}/jobs/{job}/log/tail: #' Show the last lines of a log file (up to a limit). #' @param job Job ID or UUID. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param lines Maximum number of lines to be returned up to a maximum of 1000. #' @param type Log file to be shown (stdout or stderr). tailLog=fetchOpenCGA(object=OpencgaR, category="jobs", categoryId=job, subcategory="log", subcategoryId=NULL, diff --git a/opencga-client/src/main/R/R/Meta-methods.R b/opencga-client/src/main/R/R/Meta-methods.R index 6e427607ade..8da8f31ca44 100644 --- a/opencga-client/src/main/R/R/Meta-methods.R +++ b/opencga-client/src/main/R/R/Meta-methods.R @@ -2,7 +2,6 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-04-25 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/R/R/Operation-methods.R b/opencga-client/src/main/R/R/Operation-methods.R index 0431fa7a235..1c3fc8d7072 100644 --- a/opencga-client/src/main/R/R/Operation-methods.R +++ b/opencga-client/src/main/R/R/Operation-methods.R @@ -2,7 +2,6 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-04-25 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -21,33 +20,34 @@ #' | endpointName | Endpoint WS | parameters accepted | #' | -- | :-- | --: | #' | configureCellbase | /{apiVersion}/operation/cellbase/configure | project, annotationUpdate, annotationSaveId, body | -#' | aggregateVariant | /{apiVersion}/operation/variant/aggregate | jobId, jobDescription, jobDependsOn, jobTags, study, body | -#' | deleteVariantAnnotation | /{apiVersion}/operation/variant/annotation/delete | jobId, jobDescription, jobDependsOn, jobTags, project, annotationId | -#' | indexVariantAnnotation | /{apiVersion}/operation/variant/annotation/index | jobId, jobDescription, jobDependsOn, jobTags, project, study, body | -#' | saveVariantAnnotation | /{apiVersion}/operation/variant/annotation/save | jobId, jobDescription, jobDependsOn, jobTags, project, body | +#' | aggregateVariant | /{apiVersion}/operation/variant/aggregate | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, study, body | +#' | deleteVariantAnnotation | /{apiVersion}/operation/variant/annotation/delete | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, project, annotationId | +#' | indexVariantAnnotation | /{apiVersion}/operation/variant/annotation/index | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, project, study, body | +#' | saveVariantAnnotation | /{apiVersion}/operation/variant/annotation/save | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, project, body | #' | configureVariant | /{apiVersion}/operation/variant/configure | project, study, body | -#' | deleteVariant | /{apiVersion}/operation/variant/delete | jobId, jobDescription, jobDependsOn, jobTags, study, body | -#' | aggregateVariantFamily | /{apiVersion}/operation/variant/family/aggregate | jobId, jobDescription, jobDependsOn, jobTags, study, body | -#' | indexVariantFamily | /{apiVersion}/operation/variant/family/index | jobId, jobDescription, jobDependsOn, jobTags, study, body | -#' | indexVariant | /{apiVersion}/operation/variant/index | jobId, jobDescription, jobDependsOn, jobTags, study, body | -#' | launcherVariantIndex | /{apiVersion}/operation/variant/index/launcher | jobId, jobDescription, jobDependsOn, jobTags, study, body | -#' | runVariantJulie | /{apiVersion}/operation/variant/julie/run | jobId, jobDescription, jobDependsOn, jobTags, project, body[*] | -#' | repairVariantMetadata | /{apiVersion}/operation/variant/metadata/repair | jobId, jobDescription, jobDependsOn, jobTags, body | -#' | synchronizeVariantMetadata | /{apiVersion}/operation/variant/metadata/synchronize | jobId, jobDescription, jobDependsOn, jobTags, study, body | -#' | pruneVariant | /{apiVersion}/operation/variant/prune | jobId, jobDescription, jobDependsOn, jobTags, body | -#' | deleteVariantSample | /{apiVersion}/operation/variant/sample/delete | jobId, jobDescription, jobDependsOn, jobTags, study, body | -#' | indexVariantSample | /{apiVersion}/operation/variant/sample/index | jobId, jobDescription, jobDependsOn, jobTags, study, body | +#' | deleteVariant | /{apiVersion}/operation/variant/delete | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, study, body | +#' | aggregateVariantFamily | /{apiVersion}/operation/variant/family/aggregate | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, study, body | +#' | indexVariantFamily | /{apiVersion}/operation/variant/family/index | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, study, body | +#' | indexVariant | /{apiVersion}/operation/variant/index | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, study, body | +#' | launcherVariantIndex | /{apiVersion}/operation/variant/index/launcher | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, study, body | +#' | runVariantJulie | /{apiVersion}/operation/variant/julie/run | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, project, body[*] | +#' | repairVariantMetadata | /{apiVersion}/operation/variant/metadata/repair | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body | +#' | synchronizeVariantMetadata | /{apiVersion}/operation/variant/metadata/synchronize | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, study, body | +#' | pruneVariant | /{apiVersion}/operation/variant/prune | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body | +#' | deleteVariantSample | /{apiVersion}/operation/variant/sample/delete | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, study, body | +#' | indexVariantSample | /{apiVersion}/operation/variant/sample/index | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, study, body | #' | variantSampleIndexConfigure | /{apiVersion}/operation/variant/sample/index/configure | study, skipRebuild, body | -#' | deleteVariantScore | /{apiVersion}/operation/variant/score/delete | jobId, jobDescription, jobDependsOn, jobTags, study, name, resume, force | -#' | indexVariantScore | /{apiVersion}/operation/variant/score/index | jobId, jobDescription, jobDependsOn, jobTags, study, body | -#' | variantSecondaryAnnotationIndex | /{apiVersion}/operation/variant/secondary/annotation/index | jobId, jobDescription, jobDependsOn, jobTags, project, study, body | -#' | variantSecondarySampleIndex | /{apiVersion}/operation/variant/secondary/sample/index | jobId, jobDescription, jobDependsOn, jobTags, study, body | +#' | deleteVariantScore | /{apiVersion}/operation/variant/score/delete | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, study, name, resume, force | +#' | indexVariantScore | /{apiVersion}/operation/variant/score/index | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, study, body | +#' | variantSecondaryAnnotationIndex | /{apiVersion}/operation/variant/secondary/annotation/index | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, project, study, body | +#' | variantSecondarySampleIndex | /{apiVersion}/operation/variant/secondary/sample/index | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, study, body | #' | configureVariantSecondarySampleIndex | /{apiVersion}/operation/variant/secondary/sample/index/configure | study, skipRebuild, body | -#' | secondaryIndexVariant | /{apiVersion}/operation/variant/secondaryIndex | jobId, jobDescription, jobDependsOn, jobTags, project, study, body | -#' | deleteVariantSecondaryIndex | /{apiVersion}/operation/variant/secondaryIndex/delete | jobId, jobDescription, jobDependsOn, jobTags, study, samples | -#' | deleteVariantStats | /{apiVersion}/operation/variant/stats/delete | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | indexVariantStats | /{apiVersion}/operation/variant/stats/index | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | deleteVariantStudy | /{apiVersion}/operation/variant/study/delete | jobId, jobDescription, jobDependsOn, jobTags, study, body | +#' | secondaryIndexVariant | /{apiVersion}/operation/variant/secondaryIndex | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, project, study, body | +#' | deleteVariantSecondaryIndex | /{apiVersion}/operation/variant/secondaryIndex/delete | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, study, samples | +#' | setupVariant | /{apiVersion}/operation/variant/setup | study, body | +#' | deleteVariantStats | /{apiVersion}/operation/variant/stats/delete | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | indexVariantStats | /{apiVersion}/operation/variant/stats/index | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | deleteVariantStudy | /{apiVersion}/operation/variant/study/delete | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, study, body | #' #' @md #' @seealso \url{http://docs.opencb.org/display/opencga/Using+OpenCGA} and the RESTful API documentation @@ -60,7 +60,7 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @section Endpoint /{apiVersion}/operation/cellbase/configure: #' Update Cellbase configuration. - #' @param project Project [user@]project where project can be either the ID or the alias. + #' @param project Project [organization@]project where project can be either the ID or the alias. #' @param annotationUpdate Create and load variant annotations into the database. #' @param annotationSaveId Save a copy of the current variant annotation at the database. #' @param data New cellbase configuration. @@ -73,7 +73,10 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param data Variant aggregate params. aggregateVariant=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, subcategory="variant", subcategoryId=NULL, action="aggregate", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -84,7 +87,10 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. - #' @param project Project [user@]project where project can be either the ID or the alias. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + #' @param project Project [organization@]project where project can be either the ID or the alias. #' @param annotationId Annotation identifier. deleteVariantAnnotation=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, subcategory="variant/annotation", subcategoryId=NULL, action="delete", params=params, @@ -96,8 +102,11 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. - #' @param project Project [user@]project where project can be either the ID or the alias. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + #' @param project Project [organization@]project where project can be either the ID or the alias. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param data Variant annotation index params. indexVariantAnnotation=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, subcategory="variant/annotation", subcategoryId=NULL, action="index", params=params, httpMethod="POST", @@ -109,7 +118,10 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. - #' @param project Project [user@]project where project can be either the ID or the alias. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + #' @param project Project [organization@]project where project can be either the ID or the alias. #' @param data Variant annotation save params. saveVariantAnnotation=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, subcategory="variant/annotation", subcategoryId=NULL, action="save", params=params, httpMethod="POST", @@ -117,8 +129,8 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @section Endpoint /{apiVersion}/operation/variant/configure: #' Update Variant Storage Engine configuration. Can be updated at Project or Study level. - #' @param project Project [user@]project where project can be either the ID or the alias. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param project Project [organization@]project where project can be either the ID or the alias. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param data Configuration params to update. configureVariant=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, subcategory="variant", subcategoryId=NULL, action="configure", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -129,7 +141,10 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param data Variant delete file params. deleteVariant=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, subcategory="variant", subcategoryId=NULL, action="delete", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -140,7 +155,10 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param data Variant aggregate family params. aggregateVariantFamily=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, subcategory="variant/family", subcategoryId=NULL, action="aggregate", params=params, httpMethod="POST", @@ -152,7 +170,10 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param data Variant family index params. indexVariantFamily=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, subcategory="variant/family", subcategoryId=NULL, action="index", params=params, httpMethod="POST", @@ -164,7 +185,10 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param data Variant index params. indexVariant=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, subcategory="variant", subcategoryId=NULL, action="index", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -175,7 +199,10 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param data . launcherVariantIndex=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, subcategory="variant/index", subcategoryId=NULL, action="launcher", params=params, httpMethod="POST", @@ -187,6 +214,9 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param project project. #' @param data Julie tool params. Specify list of cohorts from multiple studies with {study}:{cohort}. runVariantJulie=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, @@ -199,6 +229,9 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Variant storage metadata repair params. repairVariantMetadata=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, subcategory="variant/metadata", subcategoryId=NULL, action="repair", params=params, httpMethod="POST", @@ -210,7 +243,10 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param data Variant storage metadata synchronize params. synchronizeVariantMetadata=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, subcategory="variant/metadata", subcategoryId=NULL, action="synchronize", params=params, @@ -222,6 +258,9 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Variant prune params. Use dry-run to just generate a report with the orphan variants. pruneVariant=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, subcategory="variant", subcategoryId=NULL, action="prune", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -232,7 +271,10 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param data Variant delete sample params. deleteVariantSample=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, subcategory="variant/sample", subcategoryId=NULL, action="delete", params=params, httpMethod="POST", @@ -244,7 +286,10 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param data Variant sample index params. indexVariantSample=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, subcategory="variant/sample", subcategoryId=NULL, action="index", params=params, httpMethod="POST", @@ -252,7 +297,7 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @section Endpoint /{apiVersion}/operation/variant/sample/index/configure: #' DEPRECATED You should use the new sample index configure method. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param skipRebuild Skip sample index re-build. #' @param data New SampleIndexConfiguration. variantSampleIndexConfigure=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, @@ -265,7 +310,10 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param name Unique name of the score within the study. #' @param resume Resume a previously failed remove. #' @param force Force remove of partially indexed scores. @@ -279,7 +327,10 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param data Variant score index params. scoreName: Unique name of the score within the study. cohort1: Cohort used to compute the score. Use the cohort 'ALL' if all samples from the study where used to compute the score. cohort2: Second cohort used to compute the score, typically to compare against the first cohort. If only one cohort was used to compute the score, leave empty. inputColumns: Indicate which columns to load from the input file. Provide the column position (starting in 0) for the column with the score with 'SCORE=n'. Optionally, the PValue column with 'PVALUE=n'. The, to indicate the variant associated with the score, provide either the columns ['CHROM', 'POS', 'REF', 'ALT'], or the column 'VAR' containing a variant representation with format 'chr:start:ref:alt'. e.g. 'CHROM=0,POS=1,REF=3,ALT=4,SCORE=5,PVALUE=6' or 'VAR=0,SCORE=1,PVALUE=2'. resume: Resume a previously failed indexation. indexVariantScore=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, subcategory="variant/score", subcategoryId=NULL, action="index", params=params, httpMethod="POST", @@ -291,8 +342,11 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. - #' @param project Project [user@]project where project can be either the ID or the alias. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + #' @param project Project [organization@]project where project can be either the ID or the alias. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param data Variant secondary annotation index params. variantSecondaryAnnotationIndex=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, subcategory="variant/secondary/annotation", subcategoryId=NULL, action="index", params=params, @@ -304,7 +358,10 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param data Variant sample index params. variantSecondarySampleIndex=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, subcategory="variant/secondary/sample", subcategoryId=NULL, action="index", params=params, @@ -312,7 +369,7 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @section Endpoint /{apiVersion}/operation/variant/secondary/sample/index/configure: #' Update SampleIndex configuration (New!). - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param skipRebuild Skip sample index re-build. #' @param data New SampleIndexConfiguration. configureVariantSecondarySampleIndex=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, @@ -325,8 +382,11 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. - #' @param project Project [user@]project where project can be either the ID or the alias. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + #' @param project Project [organization@]project where project can be either the ID or the alias. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param data Variant secondary annotation index params. secondaryIndexVariant=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, subcategory="variant", subcategoryId=NULL, action="secondaryIndex", params=params, httpMethod="POST", @@ -338,19 +398,32 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param samples Samples to remove. Needs to provide all the samples in the secondary index. deleteVariantSecondaryIndex=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, subcategory="variant/secondaryIndex", subcategoryId=NULL, action="delete", params=params, httpMethod="DELETE", as.queryParam=NULL, ...), + #' @section Endpoint /{apiVersion}/operation/variant/setup: + #' Execute Variant Setup to allow using the variant engine. This setup is necessary before starting any variant operation. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param data Variant setup params. + setupVariant=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, subcategory="variant", + subcategoryId=NULL, action="setup", params=params, httpMethod="POST", as.queryParam=NULL, ...), + #' @section Endpoint /{apiVersion}/operation/variant/stats/delete: #' Deletes the VariantStats of a cohort/s from the database. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Variant stats delete params. deleteVariantStats=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, subcategory="variant/stats", subcategoryId=NULL, action="delete", params=params, httpMethod="POST", @@ -358,11 +431,14 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @section Endpoint /{apiVersion}/operation/variant/stats/index: #' Compute variant stats for any cohort and any set of variants and index the result in the variant storage database. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Variant stats params. indexVariantStats=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, subcategory="variant/stats", subcategoryId=NULL, action="index", params=params, httpMethod="POST", @@ -374,7 +450,10 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param data Variant delete study params. deleteVariantStudy=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, subcategory="variant/study", subcategoryId=NULL, action="delete", params=params, httpMethod="POST", diff --git a/opencga-client/src/main/R/R/Organization-methods.R b/opencga-client/src/main/R/R/Organization-methods.R new file mode 100644 index 00000000000..14e44864ed9 --- /dev/null +++ b/opencga-client/src/main/R/R/Organization-methods.R @@ -0,0 +1,148 @@ + +# WARNING: AUTOGENERATED CODE +# +# This code was generated by a tool. +# +# Manual changes to this file may cause unexpected behavior in your application. +# Manual changes to this file will be overwritten if the code is regenerated. + + +# ############################################################################## +#' OrganizationClient methods +#' @include AllClasses.R +#' @include AllGenerics.R +#' @include commons.R + +#' @description This function implements the OpenCGA calls for managing Organizations. + +#' The following table summarises the available *actions* for this client: +#' +#' | endpointName | Endpoint WS | parameters accepted | +#' | -- | :-- | --: | +#' | create | /{apiVersion}/organizations/create | include, exclude, includeResult, body[*] | +#' | createNotes | /{apiVersion}/organizations/notes/create | include, exclude, includeResult, body[*] | +#' | searchNotes | /{apiVersion}/organizations/notes/search | include, exclude, creationDate, modificationDate, id, scope, visibility, uuid, userId, tags, version | +#' | deleteNotes | /{apiVersion}/organizations/notes/{id}/delete | id[*], includeResult | +#' | updateNotes | /{apiVersion}/organizations/notes/{id}/update | include, exclude, id[*], tagsAction, includeResult, body[*] | +#' | userUpdateStatus | /{apiVersion}/organizations/user/{user}/status/update | include, exclude, user[*], organization, includeResult, body[*] | +#' | updateUser | /{apiVersion}/organizations/user/{user}/update | include, exclude, user[*], organization, includeResult, body[*] | +#' | updateConfiguration | /{apiVersion}/organizations/{organization}/configuration/update | include, exclude, organization[*], includeResult, authenticationOriginsAction, body[*] | +#' | info | /{apiVersion}/organizations/{organization}/info | include, exclude, organization[*] | +#' | update | /{apiVersion}/organizations/{organization}/update | include, exclude, organization[*], includeResult, adminsAction, body[*] | +#' +#' @md +#' @seealso \url{http://docs.opencb.org/display/opencga/Using+OpenCGA} and the RESTful API documentation +#' \url{http://bioinfo.hpc.cam.ac.uk/opencga-prod/webservices/} +#' [*]: Required parameter +#' @export + +setMethod("organizationClient", "OpencgaR", function(OpencgaR, id, organization, user, endpointName, params=NULL, ...) { + switch(endpointName, + + #' @section Endpoint /{apiVersion}/organizations/create: + #' Create a new organization. + #' @param include Fields included in the response, whole JSON path must be provided. + #' @param exclude Fields excluded in the response, whole JSON path must be provided. + #' @param includeResult Flag indicating to include the created or updated document result in the response. + #' @param data JSON containing the organization to be created. + create=fetchOpenCGA(object=OpencgaR, category="organizations", categoryId=NULL, subcategory=NULL, + subcategoryId=NULL, action="create", params=params, httpMethod="POST", as.queryParam=NULL, ...), + + #' @section Endpoint /{apiVersion}/organizations/notes/create: + #' Create a new note. + #' @param include Fields included in the response, whole JSON path must be provided. + #' @param exclude Fields excluded in the response, whole JSON path must be provided. + #' @param includeResult Flag indicating to include the created or updated document result in the response. + #' @param data JSON containing the Note to be added. + createNotes=fetchOpenCGA(object=OpencgaR, category="organizations", categoryId=NULL, subcategory="notes", + subcategoryId=NULL, action="create", params=params, httpMethod="POST", as.queryParam=NULL, ...), + + #' @section Endpoint /{apiVersion}/organizations/notes/search: + #' Search for notes of scope ORGANIZATION. + #' @param include Fields included in the response, whole JSON path must be provided. + #' @param exclude Fields excluded in the response, whole JSON path must be provided. + #' @param creationDate Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + #' @param modificationDate Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + #' @param id Note unique identifier. + #' @param scope Scope of the Note. + #' @param visibility Visibility of the Note. + #' @param uuid Unique 32-character identifier assigned automatically by OpenCGA. + #' @param userId User that wrote that Note. + #' @param tags Note tags. + #' @param version Autoincremental version assigned to the registered entry. By default, updates does not create new versions. To enable versioning, users must set the `incVersion` flag from the /update web service when updating the document. + searchNotes=fetchOpenCGA(object=OpencgaR, category="organizations", categoryId=NULL, subcategory="notes", + subcategoryId=NULL, action="search", params=params, httpMethod="GET", as.queryParam=NULL, ...), + + #' @section Endpoint /{apiVersion}/organizations/notes/{id}/delete: + #' Delete note. + #' @param id Note unique identifier. + #' @param includeResult Flag indicating to include the created or updated document result in the response. + deleteNotes=fetchOpenCGA(object=OpencgaR, category="organizations", categoryId=NULL, subcategory="notes", + subcategoryId=id, action="delete", params=params, httpMethod="DELETE", as.queryParam=NULL, ...), + + #' @section Endpoint /{apiVersion}/organizations/notes/{id}/update: + #' Update a note. + #' @param include Fields included in the response, whole JSON path must be provided. + #' @param exclude Fields excluded in the response, whole JSON path must be provided. + #' @param id Note unique identifier. + #' @param tagsAction Action to be performed if the array of tags is being updated. Allowed values: ['ADD SET REMOVE'] + #' @param includeResult Flag indicating to include the created or updated document result in the response. + #' @param data JSON containing the Note fields to be updated. + updateNotes=fetchOpenCGA(object=OpencgaR, category="organizations", categoryId=NULL, subcategory="notes", + subcategoryId=id, action="update", params=params, httpMethod="POST", as.queryParam=NULL, ...), + + #' @section Endpoint /{apiVersion}/organizations/user/{user}/status/update: + #' Update the user status. + #' @param include Fields included in the response, whole JSON path must be provided. + #' @param exclude Fields excluded in the response, whole JSON path must be provided. + #' @param user User ID. + #' @param organization Organization id. + #' @param includeResult Flag indicating to include the created or updated document result in the response. + #' @param data JSON containing the User fields to be updated. + userUpdateStatus=fetchOpenCGA(object=OpencgaR, category="organizations/user", categoryId=user, + subcategory="status", subcategoryId=NULL, action="update", params=params, httpMethod="POST", + as.queryParam=NULL, ...), + + #' @section Endpoint /{apiVersion}/organizations/user/{user}/update: + #' Update the user information. + #' @param include Fields included in the response, whole JSON path must be provided. + #' @param exclude Fields excluded in the response, whole JSON path must be provided. + #' @param user User ID. + #' @param organization Organization id. + #' @param includeResult Flag indicating to include the created or updated document result in the response. + #' @param data JSON containing the User fields to be updated. + updateUser=fetchOpenCGA(object=OpencgaR, category="organizations", categoryId=NULL, subcategory="user", + subcategoryId=user, action="update", params=params, httpMethod="POST", as.queryParam=NULL, ...), + + #' @section Endpoint /{apiVersion}/organizations/{organization}/configuration/update: + #' Update the Organization configuration attributes. + #' @param include Fields included in the response, whole JSON path must be provided. + #' @param exclude Fields excluded in the response, whole JSON path must be provided. + #' @param organization Organization id. + #' @param includeResult Flag indicating to include the created or updated document result in the response. + #' @param authenticationOriginsAction Action to be performed if the array of authenticationOrigins is being updated. Allowed values: ['ADD SET REMOVE REPLACE'] + #' @param data JSON containing the params to be updated. + updateConfiguration=fetchOpenCGA(object=OpencgaR, category="organizations", categoryId=organization, + subcategory="configuration", subcategoryId=NULL, action="update", params=params, httpMethod="POST", + as.queryParam=NULL, ...), + + #' @section Endpoint /{apiVersion}/organizations/{organization}/info: + #' Return the organization information. + #' @param include Fields included in the response, whole JSON path must be provided. + #' @param exclude Fields excluded in the response, whole JSON path must be provided. + #' @param organization Organization id. + info=fetchOpenCGA(object=OpencgaR, category="organizations", categoryId=organization, subcategory=NULL, + subcategoryId=NULL, action="info", params=params, httpMethod="GET", as.queryParam=NULL, ...), + + #' @section Endpoint /{apiVersion}/organizations/{organization}/update: + #' Update some organization attributes. + #' @param include Fields included in the response, whole JSON path must be provided. + #' @param exclude Fields excluded in the response, whole JSON path must be provided. + #' @param organization Organization id. + #' @param includeResult Flag indicating to include the created or updated document result in the response. + #' @param adminsAction Action to be performed if the array of admins is being updated. Allowed values: ['ADD REMOVE'] + #' @param data JSON containing the params to be updated. + update=fetchOpenCGA(object=OpencgaR, category="organizations", categoryId=organization, subcategory=NULL, + subcategoryId=NULL, action="update", params=params, httpMethod="POST", as.queryParam=NULL, ...), + ) +}) \ No newline at end of file diff --git a/opencga-client/src/main/R/R/Panel-methods.R b/opencga-client/src/main/R/R/Panel-methods.R index ab0e75d32a3..36c512a86f2 100644 --- a/opencga-client/src/main/R/R/Panel-methods.R +++ b/opencga-client/src/main/R/R/Panel-methods.R @@ -2,7 +2,6 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-04-25 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -23,7 +22,7 @@ #' | updateAcl | /{apiVersion}/panels/acl/{members}/update | study, members[*], action[*], body[*] | #' | create | /{apiVersion}/panels/create | include, exclude, study, includeResult, body | #' | distinct | /{apiVersion}/panels/distinct | study, id, uuid, name, internalStatus, disorders, variants, genes, source, regions, categories, tags, deleted, status, creationDate, modificationDate, acl, release, snapshot, field[*] | -#' | importPanels | /{apiVersion}/panels/import | study, jobId, jobDependsOn, jobDescription, jobTags, body | +#' | importPanels | /{apiVersion}/panels/import | study, jobId, jobDependsOn, jobDescription, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body | #' | search | /{apiVersion}/panels/search | include, exclude, limit, skip, count, study, id, uuid, name, internalStatus, disorders, variants, genes, source, regions, categories, tags, deleted, status, creationDate, modificationDate, acl, release, snapshot | #' | acl | /{apiVersion}/panels/{panels}/acl | panels[*], study, member, silent | #' | delete | /{apiVersion}/panels/{panels}/delete | study, panels[*] | @@ -41,7 +40,7 @@ setMethod("panelClient", "OpencgaR", function(OpencgaR, members, panels, endpoin #' @section Endpoint /{apiVersion}/panels/acl/{members}/update: #' Update the set of permissions granted for the member. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param members Comma separated list of user or group ids. #' @param action Action to be performed [ADD, SET, REMOVE or RESET]. Allowed values: ['SET ADD REMOVE RESET'] #' @param data JSON containing the parameters to update the permissions. @@ -53,7 +52,7 @@ setMethod("panelClient", "OpencgaR", function(OpencgaR, members, panels, endpoin #' Create a panel. #' @param include Fields included in the response, whole JSON path must be provided. #' @param exclude Fields excluded in the response, whole JSON path must be provided. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param includeResult Flag indicating to include the created or updated document result in the response. #' @param data Panel parameters. create=fetchOpenCGA(object=OpencgaR, category="panels", categoryId=NULL, subcategory=NULL, subcategoryId=NULL, @@ -61,7 +60,7 @@ setMethod("panelClient", "OpencgaR", function(OpencgaR, members, panels, endpoin #' @section Endpoint /{apiVersion}/panels/distinct: #' Panel distinct method. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param id Comma separated list of panel IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. #' @param uuid Comma separated list of panel UUIDs up to a maximum of 100. #' @param name Comma separated list of panel names up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. @@ -86,11 +85,14 @@ setMethod("panelClient", "OpencgaR", function(OpencgaR, members, panels, endpoin #' @section Endpoint /{apiVersion}/panels/import: #' Import panels. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobDescription Job description. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Panel parameters. importPanels=fetchOpenCGA(object=OpencgaR, category="panels", categoryId=NULL, subcategory=NULL, subcategoryId=NULL, action="import", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -102,7 +104,7 @@ setMethod("panelClient", "OpencgaR", function(OpencgaR, members, panels, endpoin #' @param limit Number of results to be returned. #' @param skip Number of results to skip. #' @param count Get the total number of results matching the query. Deactivated by default. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param id Comma separated list of panel IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. #' @param uuid Comma separated list of panel UUIDs up to a maximum of 100. #' @param name Comma separated list of panel names up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. @@ -127,7 +129,7 @@ setMethod("panelClient", "OpencgaR", function(OpencgaR, members, panels, endpoin #' @section Endpoint /{apiVersion}/panels/{panels}/acl: #' Returns the acl of the panels. If member is provided, it will only return the acl for the member. #' @param panels Comma separated list of panel IDs up to a maximum of 100. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param member User or group id. #' @param silent Boolean to retrieve all possible entries that are queried for, false to raise an exception whenever one of the entries looked for cannot be shown for whichever reason. acl=fetchOpenCGA(object=OpencgaR, category="panels", categoryId=panels, subcategory=NULL, subcategoryId=NULL, @@ -135,7 +137,7 @@ setMethod("panelClient", "OpencgaR", function(OpencgaR, members, panels, endpoin #' @section Endpoint /{apiVersion}/panels/{panels}/delete: #' Delete existing panels. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param panels Comma separated list of panel ids. delete=fetchOpenCGA(object=OpencgaR, category="panels", categoryId=panels, subcategory=NULL, subcategoryId=NULL, action="delete", params=params, httpMethod="DELETE", as.queryParam=NULL, ...), @@ -145,7 +147,7 @@ setMethod("panelClient", "OpencgaR", function(OpencgaR, members, panels, endpoin #' @param include Fields included in the response, whole JSON path must be provided. #' @param exclude Fields excluded in the response, whole JSON path must be provided. #' @param panels Comma separated list of panel IDs up to a maximum of 100. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param version Comma separated list of panel versions. 'all' to get all the panel versions. Not supported if multiple panel ids are provided. #' @param deleted Boolean to retrieve deleted panels. info=fetchOpenCGA(object=OpencgaR, category="panels", categoryId=panels, subcategory=NULL, subcategoryId=NULL, @@ -155,7 +157,7 @@ setMethod("panelClient", "OpencgaR", function(OpencgaR, members, panels, endpoin #' Update panel attributes. #' @param include Fields included in the response, whole JSON path must be provided. #' @param exclude Fields excluded in the response, whole JSON path must be provided. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param panels Comma separated list of panel ids. #' @param includeResult Flag indicating to include the created or updated document result in the response. #' @param data Panel parameters. diff --git a/opencga-client/src/main/R/R/Project-methods.R b/opencga-client/src/main/R/R/Project-methods.R index c75b843e860..380d33ea815 100644 --- a/opencga-client/src/main/R/R/Project-methods.R +++ b/opencga-client/src/main/R/R/Project-methods.R @@ -2,7 +2,6 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-04-25 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -21,8 +20,7 @@ #' | endpointName | Endpoint WS | parameters accepted | #' | -- | :-- | --: | #' | create | /{apiVersion}/projects/create | include, exclude, includeResult, body[*] | -#' | search | /{apiVersion}/projects/search | include, exclude, limit, skip, owner, id, name, fqn, organization, description, study, creationDate, modificationDate, internalStatus, attributes | -#' | aggregationStats | /{apiVersion}/projects/{projects}/aggregationStats | projects[*], default, fileFields, individualFields, familyFields, sampleFields, cohortFields, jobFields | +#' | search | /{apiVersion}/projects/search | include, exclude, limit, skip, organization, id, name, fqn, organization, description, study, creationDate, modificationDate, internalStatus, attributes | #' | info | /{apiVersion}/projects/{projects}/info | include, exclude, projects[*] | #' | incRelease | /{apiVersion}/projects/{project}/incRelease | project[*] | #' | studies | /{apiVersion}/projects/{project}/studies | include, exclude, limit, skip, project[*] | @@ -34,7 +32,7 @@ #' [*]: Required parameter #' @export -setMethod("projectClient", "OpencgaR", function(OpencgaR, projects, project, endpointName, params=NULL, ...) { +setMethod("projectClient", "OpencgaR", function(OpencgaR, project, projects, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/projects/create: @@ -52,11 +50,10 @@ setMethod("projectClient", "OpencgaR", function(OpencgaR, projects, project, end #' @param exclude Fields excluded in the response, whole JSON path must be provided. #' @param limit Number of results to be returned. #' @param skip Number of results to skip. - #' @param owner Owner of the project. - #' @param id Project [user@]project where project can be either the ID or the alias. + #' @param organization Project organization. + #' @param id Project [organization@]project where project can be either the ID or the alias. #' @param name Project name. #' @param fqn Project fqn. - #' @param organization Project organization. #' @param description Project description. #' @param study Study id. #' @param creationDate Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. @@ -66,31 +63,17 @@ setMethod("projectClient", "OpencgaR", function(OpencgaR, projects, project, end search=fetchOpenCGA(object=OpencgaR, category="projects", categoryId=NULL, subcategory=NULL, subcategoryId=NULL, action="search", params=params, httpMethod="GET", as.queryParam=NULL, ...), - #' @section Endpoint /{apiVersion}/projects/{projects}/aggregationStats: - #' Fetch catalog project stats. - #' @param projects Comma separated list of projects [user@]project up to a maximum of 100. - #' @param default Calculate default stats. - #' @param fileFields List of file fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type. - #' @param individualFields List of individual fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type. - #' @param familyFields List of family fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type. - #' @param sampleFields List of sample fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type. - #' @param cohortFields List of cohort fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type. - #' @param jobFields List of job fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type. - aggregationStats=fetchOpenCGA(object=OpencgaR, category="projects", categoryId=projects, subcategory=NULL, - subcategoryId=NULL, action="aggregationStats", params=params, httpMethod="GET", as.queryParam=NULL, - ...), - #' @section Endpoint /{apiVersion}/projects/{projects}/info: #' Fetch project information. #' @param include Fields included in the response, whole JSON path must be provided. #' @param exclude Fields excluded in the response, whole JSON path must be provided. - #' @param projects Comma separated list of projects [user@]project up to a maximum of 100. + #' @param projects Comma separated list of projects [organization@]project up to a maximum of 100. info=fetchOpenCGA(object=OpencgaR, category="projects", categoryId=projects, subcategory=NULL, subcategoryId=NULL, action="info", params=params, httpMethod="GET", as.queryParam=NULL, ...), #' @section Endpoint /{apiVersion}/projects/{project}/incRelease: #' Increment current release number in the project. - #' @param project Project [user@]project where project can be either the ID or the alias. + #' @param project Project [organization@]project where project can be either the ID or the alias. incRelease=fetchOpenCGA(object=OpencgaR, category="projects", categoryId=project, subcategory=NULL, subcategoryId=NULL, action="incRelease", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -100,7 +83,7 @@ setMethod("projectClient", "OpencgaR", function(OpencgaR, projects, project, end #' @param exclude Fields excluded in the response, whole JSON path must be provided. #' @param limit Number of results to be returned. #' @param skip Number of results to skip. - #' @param project Project [user@]project where project can be either the ID or the alias. + #' @param project Project [organization@]project where project can be either the ID or the alias. studies=fetchOpenCGA(object=OpencgaR, category="projects", categoryId=project, subcategory=NULL, subcategoryId=NULL, action="studies", params=params, httpMethod="GET", as.queryParam=NULL, ...), @@ -108,7 +91,7 @@ setMethod("projectClient", "OpencgaR", function(OpencgaR, projects, project, end #' Update some project attributes. #' @param include Fields included in the response, whole JSON path must be provided. #' @param exclude Fields excluded in the response, whole JSON path must be provided. - #' @param project Project [user@]project where project can be either the ID or the alias. + #' @param project Project [organization@]project where project can be either the ID or the alias. #' @param includeResult Flag indicating to include the created or updated document result in the response. #' @param data JSON containing the params to be updated. It will be only possible to update organism fields not previously defined. update=fetchOpenCGA(object=OpencgaR, category="projects", categoryId=project, subcategory=NULL, diff --git a/opencga-client/src/main/R/R/Sample-methods.R b/opencga-client/src/main/R/R/Sample-methods.R index 2ab6077f9b4..a5434ddb451 100644 --- a/opencga-client/src/main/R/R/Sample-methods.R +++ b/opencga-client/src/main/R/R/Sample-methods.R @@ -2,7 +2,6 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-04-25 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -21,7 +20,6 @@ #' | endpointName | Endpoint WS | parameters accepted | #' | -- | :-- | --: | #' | updateAcl | /{apiVersion}/samples/acl/{members}/update | study, members[*], action[*], body[*] | -#' | aggregationStats | /{apiVersion}/samples/aggregationStats | study, source, creationYear, creationMonth, creationDay, creationDayOfWeek, status, type, phenotypes, release, version, somatic, annotation, default, field | #' | loadAnnotationSets | /{apiVersion}/samples/annotationSets/load | study, variableSetId[*], path[*], parents, annotationSetId, body | #' | create | /{apiVersion}/samples/create | include, exclude, study, includeResult, body[*] | #' | distinct | /{apiVersion}/samples/distinct | study, id, uuid, somatic, individualId, fileIds, cohortIds, creationDate, modificationDate, internalStatus, status, processingProduct, processingPreparationMethod, processingExtractionMethod, processingLabSampleId, collectionFrom, collectionType, collectionMethod, phenotypes, annotation, acl, internalRgaStatus, release, snapshot, deleted, statsId, statsVariantCount, statsChromosomeCount, statsTypeCount, statsGenotypeCount, statsTiTvRatio, statsQualityAvg, statsQualityStdDev, statsHeterozygosityRate, statsDepthCount, statsBiotypeCount, statsClinicalSignificanceCount, statsConsequenceTypeCount, field[*] | @@ -39,12 +37,12 @@ #' [*]: Required parameter #' @export -setMethod("sampleClient", "OpencgaR", function(OpencgaR, annotationSet, members, samples, sample, endpointName, params=NULL, ...) { +setMethod("sampleClient", "OpencgaR", function(OpencgaR, annotationSet, members, sample, samples, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/samples/acl/{members}/update: #' Update the set of permissions granted for the member. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param members Comma separated list of user or group ids. #' @param action Action to be performed [ADD, SET, REMOVE or RESET]. Allowed values: ['SET ADD REMOVE RESET'] #' @param data JSON containing the parameters to update the permissions. If propagate flag is set to true, it will propagate the permissions defined to the individuals that are associated to the matching samples. @@ -52,30 +50,9 @@ setMethod("sampleClient", "OpencgaR", function(OpencgaR, annotationSet, members, subcategoryId=members, action="update", params=params, httpMethod="POST", as.queryParam=c("action"), ...), - #' @section Endpoint /{apiVersion}/samples/aggregationStats: - #' Fetch catalog sample stats. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. - #' @param source Source. - #' @param creationYear Creation year. - #' @param creationMonth Creation month (JANUARY, FEBRUARY...). - #' @param creationDay Creation day. - #' @param creationDayOfWeek Creation day of week (MONDAY, TUESDAY...). - #' @param status Status. - #' @param type Type. - #' @param phenotypes Phenotypes. - #' @param release Release. - #' @param version Version. - #' @param somatic Somatic. - #' @param annotation Annotation filters. Example: age>30;gender=FEMALE. For more information, please visit http://docs.opencb.org/display/opencga/AnnotationSets+1.4.0. - #' @param default Calculate default stats. - #' @param field List of fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type;numSamples[0..10]:1. - aggregationStats=fetchOpenCGA(object=OpencgaR, category="samples", categoryId=NULL, subcategory=NULL, - subcategoryId=NULL, action="aggregationStats", params=params, httpMethod="GET", as.queryParam=NULL, - ...), - #' @section Endpoint /{apiVersion}/samples/annotationSets/load: #' Load annotation sets from a TSV file. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param variableSetId Variable set ID or name. #' @param path Path where the TSV file is located in OpenCGA or where it should be located. #' @param parents Flag indicating whether to create parent directories if they don't exist (only when TSV file was not previously associated). @@ -89,7 +66,7 @@ setMethod("sampleClient", "OpencgaR", function(OpencgaR, annotationSet, members, #' Create sample. #' @param include Fields included in the response, whole JSON path must be provided. #' @param exclude Fields excluded in the response, whole JSON path must be provided. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param includeResult Flag indicating to include the created or updated document result in the response. #' @param data JSON containing sample information. create=fetchOpenCGA(object=OpencgaR, category="samples", categoryId=NULL, subcategory=NULL, subcategoryId=NULL, @@ -97,7 +74,7 @@ setMethod("sampleClient", "OpencgaR", function(OpencgaR, annotationSet, members, #' @section Endpoint /{apiVersion}/samples/distinct: #' Sample distinct method. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param id Comma separated list sample IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. #' @param uuid Comma separated list sample UUIDs up to a maximum of 100. #' @param somatic Somatic sample. @@ -141,7 +118,7 @@ setMethod("sampleClient", "OpencgaR", function(OpencgaR, annotationSet, members, #' @section Endpoint /{apiVersion}/samples/load: #' Load samples from a ped file [EXPERIMENTAL]. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param file file. #' @param variableSet variableSet. load=fetchOpenCGA(object=OpencgaR, category="samples", categoryId=NULL, subcategory=NULL, subcategoryId=NULL, @@ -156,7 +133,7 @@ setMethod("sampleClient", "OpencgaR", function(OpencgaR, annotationSet, members, #' @param count Get the total number of results matching the query. Deactivated by default. #' @param includeIndividual Include Individual object as an attribute. #' @param flattenAnnotations Flatten the annotations?. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param id Comma separated list sample IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. #' @param uuid Comma separated list sample UUIDs up to a maximum of 100. #' @param somatic Somatic sample. @@ -200,7 +177,7 @@ setMethod("sampleClient", "OpencgaR", function(OpencgaR, annotationSet, members, #' @section Endpoint /{apiVersion}/samples/{samples}/acl: #' Returns the acl of the samples. If member is provided, it will only return the acl for the member. #' @param samples Comma separated list sample IDs or UUIDs up to a maximum of 100. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param member User or group id. #' @param silent Boolean to retrieve all possible entries that are queried for, false to raise an exception whenever one of the entries looked for cannot be shown for whichever reason. acl=fetchOpenCGA(object=OpencgaR, category="samples", categoryId=samples, subcategory=NULL, subcategoryId=NULL, @@ -211,7 +188,7 @@ setMethod("sampleClient", "OpencgaR", function(OpencgaR, annotationSet, members, #' @param force Force the deletion of samples even if they are associated to files, individuals or cohorts. #' @param emptyFilesAction Action to be performed over files that were associated only to the sample to be deleted. Possible actions are NONE, TRASH, DELETE. #' @param deleteEmptyCohorts Boolean indicating if the cohorts associated only to the sample to be deleted should be also deleted. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param samples Comma separated list sample IDs or UUIDs up to a maximum of 100. delete=fetchOpenCGA(object=OpencgaR, category="samples", categoryId=samples, subcategory=NULL, subcategoryId=NULL, action="delete", params=params, httpMethod="DELETE", as.queryParam=NULL, ...), @@ -223,7 +200,7 @@ setMethod("sampleClient", "OpencgaR", function(OpencgaR, annotationSet, members, #' @param includeIndividual Include Individual object as an attribute. #' @param flattenAnnotations Flatten the annotations?. #' @param samples Comma separated list sample IDs or UUIDs up to a maximum of 100. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param version Comma separated list of sample versions. 'all' to get all the sample versions. Not supported if multiple sample ids are provided. #' @param deleted Boolean to retrieve deleted entries. info=fetchOpenCGA(object=OpencgaR, category="samples", categoryId=samples, subcategory=NULL, @@ -234,7 +211,7 @@ setMethod("sampleClient", "OpencgaR", function(OpencgaR, annotationSet, members, #' @param include Fields included in the response, whole JSON path must be provided. #' @param exclude Fields excluded in the response, whole JSON path must be provided. #' @param samples Comma separated list sample IDs or UUIDs up to a maximum of 100. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param annotationSetsAction Action to be performed if the array of annotationSets is being updated. Allowed values: ['ADD SET REMOVE'] #' @param phenotypesAction Action to be performed if the array of phenotypes is being updated [SET, ADD, REMOVE]. Allowed values: ['ADD SET REMOVE'] #' @param includeResult Flag indicating to include the created or updated document result in the response. @@ -245,7 +222,7 @@ setMethod("sampleClient", "OpencgaR", function(OpencgaR, annotationSet, members, #' @section Endpoint /{apiVersion}/samples/{sample}/annotationSets/{annotationSet}/annotations/update: #' Update annotations from an annotationSet. #' @param sample Sample ID. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param annotationSet AnnotationSet ID to be updated. #' @param action Action to be performed: ADD to add new annotations; REPLACE to replace the value of an already existing annotation; SET to set the new list of annotations removing any possible old annotations; REMOVE to remove some annotations; RESET to set some annotations to the default value configured in the corresponding variables of the VariableSet if any. Allowed values: ['ADD SET REMOVE RESET REPLACE'] #' @param data Json containing the map of annotations when the action is ADD, SET or REPLACE, a json with only the key 'remove' containing the comma separated variables to be removed as a value when the action is REMOVE or a json with only the key 'reset' containing the comma separated variables that will be set to the default value when the action is RESET. diff --git a/opencga-client/src/main/R/R/Study-methods.R b/opencga-client/src/main/R/R/Study-methods.R index a3d83d55ecd..ee3f4509354 100644 --- a/opencga-client/src/main/R/R/Study-methods.R +++ b/opencga-client/src/main/R/R/Study-methods.R @@ -2,7 +2,6 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-04-25 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -24,17 +23,20 @@ #' | create | /{apiVersion}/studies/create | include, exclude, project, includeResult, body[*] | #' | search | /{apiVersion}/studies/search | include, exclude, limit, skip, count, project[*], name, id, alias, fqn, creationDate, modificationDate, internalStatus, status, attributes, release | #' | acl | /{apiVersion}/studies/{studies}/acl | studies[*], member, silent | -#' | aggregationStats | /{apiVersion}/studies/{studies}/aggregationStats | studies[*], default, fileFields, individualFields, familyFields, sampleFields, cohortFields, jobFields | #' | info | /{apiVersion}/studies/{studies}/info | include, exclude, studies[*] | #' | searchAudit | /{apiVersion}/studies/{study}/audit/search | include, exclude, limit, skip, count, study[*], operationId, userId, action, resource, resourceId, resourceUuid, status, date | #' | groups | /{apiVersion}/studies/{study}/groups | study[*], id, silent | #' | updateGroups | /{apiVersion}/studies/{study}/groups/update | study[*], action, body[*] | #' | updateGroupsUsers | /{apiVersion}/studies/{study}/groups/{group}/users/update | study[*], group[*], action, body[*] | +#' | createNotes | /{apiVersion}/studies/{study}/notes/create | include, exclude, study[*], includeResult, body[*] | +#' | searchNotes | /{apiVersion}/studies/{study}/notes/search | include, exclude, study[*], creationDate, modificationDate, id, uuid, userId, tags, visibility, version | +#' | deleteNotes | /{apiVersion}/studies/{study}/notes/{id}/delete | study[*], id[*], includeResult | +#' | updateNotes | /{apiVersion}/studies/{study}/notes/{id}/update | include, exclude, study[*], id[*], tagsAction, includeResult, body[*] | #' | permissionRules | /{apiVersion}/studies/{study}/permissionRules | study[*], entity[*] | #' | updatePermissionRules | /{apiVersion}/studies/{study}/permissionRules/update | study[*], entity[*], action, body[*] | -#' | runTemplates | /{apiVersion}/studies/{study}/templates/run | study[*], jobId, jobDependsOn, jobDescription, jobTags, body[*] | +#' | runTemplates | /{apiVersion}/studies/{study}/templates/run | study[*], jobId, jobDependsOn, jobDescription, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | #' | uploadTemplates | /{apiVersion}/studies/{study}/templates/upload | file, study[*] | -#' | deleteTemplates | /{apiVersion}/studies/{study}/templates/{templateId}/delete | study, templateId[*] | +#' | deleteTemplates | /{apiVersion}/studies/{study}/templates/{templateId}/delete | study[*], templateId[*] | #' | update | /{apiVersion}/studies/{study}/update | include, exclude, study[*], includeResult, body[*] | #' | variableSets | /{apiVersion}/studies/{study}/variableSets | study[*], id | #' | updateVariableSets | /{apiVersion}/studies/{study}/variableSets/update | study[*], action, body[*] | @@ -46,7 +48,7 @@ #' [*]: Required parameter #' @export -setMethod("studyClient", "OpencgaR", function(OpencgaR, studies, members, variableSet, group, templateId, study, endpointName, params=NULL, ...) { +setMethod("studyClient", "OpencgaR", function(OpencgaR, group, id, members, studies, study, templateId, variableSet, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/studies/acl/{members}/update: @@ -62,7 +64,7 @@ setMethod("studyClient", "OpencgaR", function(OpencgaR, studies, members, variab #' Create a new study. #' @param include Fields included in the response, whole JSON path must be provided. #' @param exclude Fields excluded in the response, whole JSON path must be provided. - #' @param project Project [user@]project where project can be either the ID or the alias. + #' @param project Project [organization@]project where project can be either the ID or the alias. #' @param includeResult Flag indicating to include the created or updated document result in the response. #' @param data study. create=fetchOpenCGA(object=OpencgaR, category="studies", categoryId=NULL, subcategory=NULL, subcategoryId=NULL, @@ -75,7 +77,7 @@ setMethod("studyClient", "OpencgaR", function(OpencgaR, studies, members, variab #' @param limit Number of results to be returned. #' @param skip Number of results to skip. #' @param count Get the total number of results matching the query. Deactivated by default. - #' @param project Project [user@]project where project can be either the ID or the alias. + #' @param project Project [organization@]project where project can be either the ID or the alias. #' @param name Study name. #' @param id Study ID. #' @param alias Study alias. @@ -91,31 +93,17 @@ setMethod("studyClient", "OpencgaR", function(OpencgaR, studies, members, variab #' @section Endpoint /{apiVersion}/studies/{studies}/acl: #' Return the acl of the study. If member is provided, it will only return the acl for the member. - #' @param studies Comma separated list of Studies [[user@]project:]study where study and project can be either the ID or UUID up to a maximum of 100. + #' @param studies Comma separated list of Studies [[organization@]project:]study where study and project can be either the ID or UUID up to a maximum of 100. #' @param member User or group id. #' @param silent Boolean to retrieve all possible entries that are queried for, false to raise an exception whenever one of the entries looked for cannot be shown for whichever reason. acl=fetchOpenCGA(object=OpencgaR, category="studies", categoryId=studies, subcategory=NULL, subcategoryId=NULL, action="acl", params=params, httpMethod="GET", as.queryParam=NULL, ...), - #' @section Endpoint /{apiVersion}/studies/{studies}/aggregationStats: - #' Fetch catalog study stats. - #' @param studies Comma separated list of studies [[user@]project:]study up to a maximum of 100. - #' @param default Calculate default stats. - #' @param fileFields List of file fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type. - #' @param individualFields List of individual fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type. - #' @param familyFields List of family fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type. - #' @param sampleFields List of sample fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type. - #' @param cohortFields List of cohort fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type. - #' @param jobFields List of job fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type. - aggregationStats=fetchOpenCGA(object=OpencgaR, category="studies", categoryId=studies, subcategory=NULL, - subcategoryId=NULL, action="aggregationStats", params=params, httpMethod="GET", as.queryParam=NULL, - ...), - #' @section Endpoint /{apiVersion}/studies/{studies}/info: #' Fetch study information. #' @param include Fields included in the response, whole JSON path must be provided. #' @param exclude Fields excluded in the response, whole JSON path must be provided. - #' @param studies Comma separated list of Studies [[user@]project:]study where study and project can be either the ID or UUID up to a maximum of 100. + #' @param studies Comma separated list of Studies [[organization@]project:]study where study and project can be either the ID or UUID up to a maximum of 100. info=fetchOpenCGA(object=OpencgaR, category="studies", categoryId=studies, subcategory=NULL, subcategoryId=NULL, action="info", params=params, httpMethod="GET", as.queryParam=NULL, ...), @@ -130,7 +118,7 @@ setMethod("studyClient", "OpencgaR", function(OpencgaR, studies, members, variab #' @param operationId Audit operation UUID. #' @param userId User ID. #' @param action Action performed by the user. - #' @param resource Resource involved. Allowed values: ['AUDIT USER PROJECT STUDY FILE SAMPLE JOB INDIVIDUAL COHORT DISEASE_PANEL FAMILY CLINICAL_ANALYSIS INTERPRETATION VARIANT ALIGNMENT CLINICAL EXPRESSION RGA FUNCTIONAL'] + #' @param resource Resource involved. Allowed values: ['AUDIT NOTE ORGANIZATION USER PROJECT STUDY FILE SAMPLE JOB INDIVIDUAL COHORT DISEASE_PANEL FAMILY CLINICAL_ANALYSIS INTERPRETATION VARIANT ALIGNMENT CLINICAL EXPRESSION RGA FUNCTIONAL'] #' @param resourceId Resource ID. #' @param resourceUuid resource UUID. #' @param status Filter by status. Allowed values: ['SUCCESS ERROR'] @@ -140,7 +128,7 @@ setMethod("studyClient", "OpencgaR", function(OpencgaR, studies, members, variab #' @section Endpoint /{apiVersion}/studies/{study}/groups: #' Return the groups present in the study. For owners and administrators only. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param id Group id. If provided, it will only fetch information for the provided group. #' @param silent Boolean to retrieve all possible entries that are queried for, false to raise an exception whenever one of the entries looked for cannot be shown for whichever reason. groups=fetchOpenCGA(object=OpencgaR, category="studies", categoryId=study, subcategory=NULL, @@ -148,7 +136,7 @@ setMethod("studyClient", "OpencgaR", function(OpencgaR, studies, members, variab #' @section Endpoint /{apiVersion}/studies/{study}/groups/update: #' Add or remove a group. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param action Action to be performed: ADD or REMOVE a group. Allowed values: ['ADD REMOVE'] #' @param data JSON containing the parameters. updateGroups=fetchOpenCGA(object=OpencgaR, category="studies", categoryId=study, subcategory="groups", @@ -156,16 +144,62 @@ setMethod("studyClient", "OpencgaR", function(OpencgaR, studies, members, variab #' @section Endpoint /{apiVersion}/studies/{study}/groups/{group}/users/update: #' Add, set or remove users from an existing group. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param group Group name. #' @param action Action to be performed: ADD, SET or REMOVE users to/from a group. Allowed values: ['ADD SET REMOVE'] #' @param data JSON containing the parameters. updateGroupsUsers=fetchOpenCGA(object=OpencgaR, category="studies", categoryId=study, subcategory="groups", subcategoryId=group, action="users/update", params=params, httpMethod="POST", as.queryParam=NULL, ...), + #' @section Endpoint /{apiVersion}/studies/{study}/notes/create: + #' Create a new note. + #' @param include Fields included in the response, whole JSON path must be provided. + #' @param exclude Fields excluded in the response, whole JSON path must be provided. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param includeResult Flag indicating to include the created or updated document result in the response. + #' @param data JSON containing the Note to be added. + createNotes=fetchOpenCGA(object=OpencgaR, category="studies", categoryId=study, subcategory="notes", + subcategoryId=NULL, action="create", params=params, httpMethod="POST", as.queryParam=NULL, ...), + + #' @section Endpoint /{apiVersion}/studies/{study}/notes/search: + #' Search for notes of scope STUDY. + #' @param include Fields included in the response, whole JSON path must be provided. + #' @param exclude Fields excluded in the response, whole JSON path must be provided. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param creationDate Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + #' @param modificationDate Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + #' @param id Note unique identifier. + #' @param uuid Unique 32-character identifier assigned automatically by OpenCGA. + #' @param userId User that wrote that Note. + #' @param tags Note tags. + #' @param visibility Visibility of the Note. + #' @param version Autoincremental version assigned to the registered entry. By default, updates does not create new versions. To enable versioning, users must set the `incVersion` flag from the /update web service when updating the document. + searchNotes=fetchOpenCGA(object=OpencgaR, category="studies", categoryId=study, subcategory="notes", + subcategoryId=NULL, action="search", params=params, httpMethod="GET", as.queryParam=NULL, ...), + + #' @section Endpoint /{apiVersion}/studies/{study}/notes/{id}/delete: + #' Delete note. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param id Note unique identifier. + #' @param includeResult Flag indicating to include the created or updated document result in the response. + deleteNotes=fetchOpenCGA(object=OpencgaR, category="studies", categoryId=study, subcategory="notes", + subcategoryId=id, action="delete", params=params, httpMethod="DELETE", as.queryParam=NULL, ...), + + #' @section Endpoint /{apiVersion}/studies/{study}/notes/{id}/update: + #' Update a note. + #' @param include Fields included in the response, whole JSON path must be provided. + #' @param exclude Fields excluded in the response, whole JSON path must be provided. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param id Note unique identifier. + #' @param tagsAction Action to be performed if the array of tags is being updated. Allowed values: ['ADD SET REMOVE'] + #' @param includeResult Flag indicating to include the created or updated document result in the response. + #' @param data JSON containing the Note fields to be updated. + updateNotes=fetchOpenCGA(object=OpencgaR, category="studies", categoryId=study, subcategory="notes", + subcategoryId=id, action="update", params=params, httpMethod="POST", as.queryParam=NULL, ...), + #' @section Endpoint /{apiVersion}/studies/{study}/permissionRules: #' Fetch permission rules. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param entity Entity where the permission rules should be applied to. Allowed values: ['SAMPLES FILES COHORTS INDIVIDUALS FAMILIES JOBS CLINICAL_ANALYSES DISEASE_PANELS'] permissionRules=fetchOpenCGA(object=OpencgaR, category="studies", categoryId=study, subcategory=NULL, subcategoryId=NULL, action="permissionRules", params=params, httpMethod="GET", @@ -173,7 +207,7 @@ setMethod("studyClient", "OpencgaR", function(OpencgaR, studies, members, variab #' @section Endpoint /{apiVersion}/studies/{study}/permissionRules/update: #' Add or remove a permission rule. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param entity Entity where the permission rules should be applied to. Allowed values: ['SAMPLES FILES COHORTS INDIVIDUALS FAMILIES JOBS CLINICAL_ANALYSES DISEASE_PANELS'] #' @param action Action to be performed: ADD to add a new permission rule; REMOVE to remove all permissions assigned by an existing permission rule (even if it overlaps any manual permission); REVERT to remove all permissions assigned by an existing permission rule (keep manual overlaps); NONE to remove an existing permission rule without removing any permissions that could have been assigned already by the permission rule. Allowed values: ['ADD REMOVE REVERT NONE'] #' @param data JSON containing the permission rule to be created or removed. @@ -183,11 +217,14 @@ setMethod("studyClient", "OpencgaR", function(OpencgaR, studies, members, variab #' @section Endpoint /{apiVersion}/studies/{study}/templates/run: #' Execute template. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobDescription Job description. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Template loader parameters. runTemplates=fetchOpenCGA(object=OpencgaR, category="studies", categoryId=study, subcategory="templates", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -195,13 +232,13 @@ setMethod("studyClient", "OpencgaR", function(OpencgaR, studies, members, variab #' @section Endpoint /{apiVersion}/studies/{study}/templates/upload: #' Resource to upload a zipped template. #' @param file File to upload. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. uploadTemplates=fetchOpenCGA(object=OpencgaR, category="studies", categoryId=study, subcategory="templates", subcategoryId=NULL, action="upload", params=params, httpMethod="POST", as.queryParam=NULL, ...), #' @section Endpoint /{apiVersion}/studies/{study}/templates/{templateId}/delete: #' Delete template. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param templateId Template id. deleteTemplates=fetchOpenCGA(object=OpencgaR, category="studies", categoryId=study, subcategory="templates", subcategoryId=templateId, action="delete", params=params, httpMethod="DELETE", as.queryParam=NULL, ...), @@ -210,7 +247,7 @@ setMethod("studyClient", "OpencgaR", function(OpencgaR, studies, members, variab #' Update some study attributes. #' @param include Fields included in the response, whole JSON path must be provided. #' @param exclude Fields excluded in the response, whole JSON path must be provided. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param includeResult Flag indicating to include the created or updated document result in the response. #' @param data JSON containing the params to be updated. update=fetchOpenCGA(object=OpencgaR, category="studies", categoryId=study, subcategory=NULL, @@ -218,14 +255,14 @@ setMethod("studyClient", "OpencgaR", function(OpencgaR, studies, members, variab #' @section Endpoint /{apiVersion}/studies/{study}/variableSets: #' Fetch variableSets from a study. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param id Id of the variableSet to be retrieved. If no id is passed, it will show all the variableSets of the study. variableSets=fetchOpenCGA(object=OpencgaR, category="studies", categoryId=study, subcategory=NULL, subcategoryId=NULL, action="variableSets", params=params, httpMethod="GET", as.queryParam=NULL, ...), #' @section Endpoint /{apiVersion}/studies/{study}/variableSets/update: #' Add or remove a variableSet. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param action Action to be performed: ADD, REMOVE or FORCE_REMOVE a variableSet. Allowed values: ['ADD REMOVE FORCE_REMOVE'] #' @param data JSON containing the VariableSet to be created or removed. updateVariableSets=fetchOpenCGA(object=OpencgaR, category="studies", categoryId=study, @@ -234,7 +271,7 @@ setMethod("studyClient", "OpencgaR", function(OpencgaR, studies, members, variab #' @section Endpoint /{apiVersion}/studies/{study}/variableSets/{variableSet}/variables/update: #' Add or remove variables to a VariableSet. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param variableSet VariableSet id of the VariableSet to be updated. #' @param action Action to be performed: ADD or REMOVE a variable. Allowed values: ['ADD REMOVE'] #' @param data JSON containing the variable to be added or removed. For removing, only the variable id will be needed. diff --git a/opencga-client/src/main/R/R/User-methods.R b/opencga-client/src/main/R/R/User-methods.R index 266a3ee51d5..ffc1c599ca9 100644 --- a/opencga-client/src/main/R/R/User-methods.R +++ b/opencga-client/src/main/R/R/User-methods.R @@ -2,7 +2,6 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-04-25 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -20,16 +19,18 @@ #' #' | endpointName | Endpoint WS | parameters accepted | #' | -- | :-- | --: | +#' | anonymous | /{apiVersion}/users/anonymous | organization[*] | +#' | create | /{apiVersion}/users/create | body[*] | #' | login | /{apiVersion}/users/login | body | #' | password | /{apiVersion}/users/password | body[*] | -#' | info | /{apiVersion}/users/{users}/info | include, exclude, users[*] | +#' | search | /{apiVersion}/users/search | include, exclude, limit, skip, count, organization, id, authenticationId | +#' | info | /{apiVersion}/users/{users}/info | include, exclude, organization, users[*] | #' | configs | /{apiVersion}/users/{user}/configs | user[*], name | #' | updateConfigs | /{apiVersion}/users/{user}/configs/update | user[*], action, body[*] | #' | filters | /{apiVersion}/users/{user}/filters | user[*], id | #' | updateFilters | /{apiVersion}/users/{user}/filters/update | user[*], action, body[*] | #' | updateFilter | /{apiVersion}/users/{user}/filters/{filterId}/update | user[*], filterId[*], body[*] | #' | resetPassword | /{apiVersion}/users/{user}/password/reset | user[*] | -#' | projects | /{apiVersion}/users/{user}/projects | include, exclude, limit, skip, user[*] | #' | update | /{apiVersion}/users/{user}/update | include, exclude, user[*], includeResult, body[*] | #' #' @md @@ -38,9 +39,22 @@ #' [*]: Required parameter #' @export -setMethod("userClient", "OpencgaR", function(OpencgaR, users, user, filterId, endpointName, params=NULL, ...) { +setMethod("userClient", "OpencgaR", function(OpencgaR, filterId, user, users, endpointName, params=NULL, ...) { switch(endpointName, + #' @section Endpoint /{apiVersion}/users/anonymous: + #' Get an anonymous token to gain access to the system. + #' @param organization Organization id. + anonymous=fetchOpenCGA(object=OpencgaR, category="users", categoryId=NULL, subcategory=NULL, + subcategoryId=NULL, action="anonymous", params=params, httpMethod="POST", + as.queryParam=c("organization"), ...), + + #' @section Endpoint /{apiVersion}/users/create: + #' Create a new user. + #' @param data JSON containing the parameters. + create=fetchOpenCGA(object=OpencgaR, category="users", categoryId=NULL, subcategory=NULL, subcategoryId=NULL, + action="create", params=params, httpMethod="POST", as.queryParam=NULL, ...), + #' @section Endpoint /{apiVersion}/users/login: #' Get identified and gain access to the system. #' @param data JSON containing the authentication parameters. @@ -53,10 +67,24 @@ setMethod("userClient", "OpencgaR", function(OpencgaR, users, user, filterId, en password=fetchOpenCGA(object=OpencgaR, category="users", categoryId=NULL, subcategory=NULL, subcategoryId=NULL, action="password", params=params, httpMethod="POST", as.queryParam=NULL, ...), + #' @section Endpoint /{apiVersion}/users/search: + #' User search method. + #' @param include Fields included in the response, whole JSON path must be provided. + #' @param exclude Fields excluded in the response, whole JSON path must be provided. + #' @param limit Number of results to be returned. + #' @param skip Number of results to skip. + #' @param count Get the total number of results matching the query. Deactivated by default. + #' @param organization Organization id. + #' @param id Comma separated list user IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + #' @param authenticationId Authentication origin ID. + search=fetchOpenCGA(object=OpencgaR, category="users", categoryId=NULL, subcategory=NULL, subcategoryId=NULL, + action="search", params=params, httpMethod="GET", as.queryParam=NULL, ...), + #' @section Endpoint /{apiVersion}/users/{users}/info: #' Return the user information including its projects and studies. #' @param include Fields included in the response, whole JSON path must be provided. #' @param exclude Fields excluded in the response, whole JSON path must be provided. + #' @param organization Organization id. #' @param users Comma separated list of user IDs. info=fetchOpenCGA(object=OpencgaR, category="users", categoryId=users, subcategory=NULL, subcategoryId=NULL, action="info", params=params, httpMethod="GET", as.queryParam=NULL, ...), @@ -105,16 +133,6 @@ setMethod("userClient", "OpencgaR", function(OpencgaR, users, user, filterId, en resetPassword=fetchOpenCGA(object=OpencgaR, category="users", categoryId=user, subcategory="password", subcategoryId=NULL, action="reset", params=params, httpMethod="GET", as.queryParam=NULL, ...), - #' @section Endpoint /{apiVersion}/users/{user}/projects: - #' Retrieve the projects of the user. - #' @param include Fields included in the response, whole JSON path must be provided. - #' @param exclude Fields excluded in the response, whole JSON path must be provided. - #' @param limit Number of results to be returned. - #' @param skip Number of results to skip. - #' @param user User ID. - projects=fetchOpenCGA(object=OpencgaR, category="users", categoryId=user, subcategory=NULL, subcategoryId=NULL, - action="projects", params=params, httpMethod="GET", as.queryParam=NULL, ...), - #' @section Endpoint /{apiVersion}/users/{user}/update: #' Update some user attributes. #' @param include Fields included in the response, whole JSON path must be provided. diff --git a/opencga-client/src/main/R/R/Variant-methods.R b/opencga-client/src/main/R/R/Variant-methods.R index c17b30b4f0c..5413a7604b5 100644 --- a/opencga-client/src/main/R/R/Variant-methods.R +++ b/opencga-client/src/main/R/R/Variant-methods.R @@ -2,7 +2,6 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2024-04-25 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -26,39 +25,39 @@ #' | runCircos | /{apiVersion}/analysis/variant/circos/run | study, body[*] | #' | deleteCohortStats | /{apiVersion}/analysis/variant/cohort/stats/delete | study, cohort | #' | infoCohortStats | /{apiVersion}/analysis/variant/cohort/stats/info | study, cohort[*] | -#' | runCohortStats | /{apiVersion}/analysis/variant/cohort/stats/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | runExomiser | /{apiVersion}/analysis/variant/exomiser/run | study, jobId, jobDependsOn, jobDescription, jobTags, body[*] | -#' | runExport | /{apiVersion}/analysis/variant/export/run | include, exclude, project, study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | +#' | runCohortStats | /{apiVersion}/analysis/variant/cohort/stats/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runExomiser | /{apiVersion}/analysis/variant/exomiser/run | study, jobId, jobDependsOn, jobDescription, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runExport | /{apiVersion}/analysis/variant/export/run | include, exclude, project, study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | #' | genotypesFamily | /{apiVersion}/analysis/variant/family/genotypes | study, family, clinicalAnalysis, modeOfInheritance[*], penetrance, disorder | -#' | runFamilyQc | /{apiVersion}/analysis/variant/family/qc/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | deleteFile | /{apiVersion}/analysis/variant/file/delete | jobId, jobDescription, jobDependsOn, jobTags, study, file, resume | -#' | runGatk | /{apiVersion}/analysis/variant/gatk/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | runGenomePlot | /{apiVersion}/analysis/variant/genomePlot/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | runGwas | /{apiVersion}/analysis/variant/gwas/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | runHrDetect | /{apiVersion}/analysis/variant/hrDetect/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | runIndex | /{apiVersion}/analysis/variant/index/run | study, jobId, jobDependsOn, jobDescription, jobTags, body[*] | -#' | runIndividualQc | /{apiVersion}/analysis/variant/individual/qc/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | runInferredSex | /{apiVersion}/analysis/variant/inferredSex/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | +#' | runFamilyQc | /{apiVersion}/analysis/variant/family/qc/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | deleteFile | /{apiVersion}/analysis/variant/file/delete | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, study, file, resume | +#' | runGatk | /{apiVersion}/analysis/variant/gatk/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runGenomePlot | /{apiVersion}/analysis/variant/genomePlot/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runGwas | /{apiVersion}/analysis/variant/gwas/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runHrDetect | /{apiVersion}/analysis/variant/hrDetect/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runIndex | /{apiVersion}/analysis/variant/index/run | study, jobId, jobDependsOn, jobDescription, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runIndividualQc | /{apiVersion}/analysis/variant/individual/qc/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runInferredSex | /{apiVersion}/analysis/variant/inferredSex/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | #' | queryKnockoutGene | /{apiVersion}/analysis/variant/knockout/gene/query | limit, skip, study, job | #' | queryKnockoutIndividual | /{apiVersion}/analysis/variant/knockout/individual/query | limit, skip, study, job | -#' | runKnockout | /{apiVersion}/analysis/variant/knockout/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | runMendelianError | /{apiVersion}/analysis/variant/mendelianError/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | +#' | runKnockout | /{apiVersion}/analysis/variant/knockout/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runMendelianError | /{apiVersion}/analysis/variant/mendelianError/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | #' | metadata | /{apiVersion}/analysis/variant/metadata | project, study, file, sample, includeStudy, includeFile, includeSample, include, exclude | #' | queryMutationalSignature | /{apiVersion}/analysis/variant/mutationalSignature/query | study, sample, type, ct, biotype, fileData, filter, qual, region, gene, panel, panelModeOfInheritance, panelConfidence, panelFeatureType, panelRoleInCancer, panelIntersection, msId, msDescription | -#' | runMutationalSignature | /{apiVersion}/analysis/variant/mutationalSignature/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | runPlink | /{apiVersion}/analysis/variant/plink/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | +#' | runMutationalSignature | /{apiVersion}/analysis/variant/mutationalSignature/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runPlink | /{apiVersion}/analysis/variant/plink/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | #' | query | /{apiVersion}/analysis/variant/query | include, exclude, limit, skip, count, sort, summary, approximateCount, approximateCountSamplingSize, savedFilter, id, region, type, reference, alternate, project, study, file, filter, qual, fileData, sample, genotype, sampleData, sampleAnnotation, sampleMetadata, unknownGenotype, sampleLimit, sampleSkip, cohort, cohortStatsRef, cohortStatsAlt, cohortStatsMaf, cohortStatsMgf, cohortStatsPass, missingAlleles, missingGenotypes, score, family, familyDisorder, familySegregation, familyMembers, familyProband, includeStudy, includeFile, includeSample, includeSampleData, includeGenotype, includeSampleId, annotationExists, gene, ct, xref, biotype, proteinSubstitution, conservation, populationFrequencyAlt, populationFrequencyRef, populationFrequencyMaf, transcriptFlag, geneTraitId, go, expression, proteinKeyword, drug, functionalScore, clinical, clinicalSignificance, clinicalConfirmedStatus, customAnnotation, panel, panelModeOfInheritance, panelConfidence, panelRoleInCancer, panelFeatureType, panelIntersection, trait | -#' | runRelatedness | /{apiVersion}/analysis/variant/relatedness/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | runRvtests | /{apiVersion}/analysis/variant/rvtests/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | +#' | runRelatedness | /{apiVersion}/analysis/variant/relatedness/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runRvtests | /{apiVersion}/analysis/variant/rvtests/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | #' | aggregationStatsSample | /{apiVersion}/analysis/variant/sample/aggregationStats | savedFilter, region, type, project, study, file, filter, sample, genotype, sampleAnnotation, family, familyDisorder, familySegregation, familyMembers, familyProband, ct, biotype, populationFrequencyAlt, clinical, clinicalSignificance, clinicalConfirmedStatus, field | -#' | runSampleEligibility | /{apiVersion}/analysis/variant/sample/eligibility/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | runSampleQc | /{apiVersion}/analysis/variant/sample/qc/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | +#' | runSampleEligibility | /{apiVersion}/analysis/variant/sample/eligibility/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runSampleQc | /{apiVersion}/analysis/variant/sample/qc/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | #' | querySample | /{apiVersion}/analysis/variant/sample/query | limit, skip, variant, study, genotype | -#' | runSample | /{apiVersion}/analysis/variant/sample/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | +#' | runSample | /{apiVersion}/analysis/variant/sample/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | #' | querySampleStats | /{apiVersion}/analysis/variant/sample/stats/query | region, type, study, file, filter, sampleData, ct, biotype, transcriptFlag, populationFrequencyAlt, clinical, clinicalSignificance, clinicalConfirmedStatus, study, filterTranscript, sample[*] | -#' | runSampleStats | /{apiVersion}/analysis/variant/sample/stats/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | runStatsExport | /{apiVersion}/analysis/variant/stats/export/run | project, study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | runStats | /{apiVersion}/analysis/variant/stats/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | +#' | runSampleStats | /{apiVersion}/analysis/variant/sample/stats/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runStatsExport | /{apiVersion}/analysis/variant/stats/export/run | project, study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runStats | /{apiVersion}/analysis/variant/stats/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | #' #' @md #' @seealso \url{http://docs.opencb.org/display/opencga/Using+OpenCGA} and the RESTful API documentation @@ -74,8 +73,8 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @param savedFilter Use a saved filter at User level. #' @param region List of regions, these can be just a single chromosome name or regions in the format chr:start-end, e.g.: 2,3:100000-200000. #' @param type List of types, accepted values are SNV, MNV, INDEL, SV, COPY_NUMBER, COPY_NUMBER_LOSS, COPY_NUMBER_GAIN, INSERTION, DELETION, DUPLICATION, TANDEM_DUPLICATION, BREAKEND, e.g. SNV,INDEL. - #' @param project Project [user@]project where project can be either the ID or the alias. - #' @param study Filter variants from the given studies, these can be either the numeric ID or the alias with the format user@project:study. + #' @param project Project [organization@]project where project can be either the ID or the alias. + #' @param study Filter variants from the given studies, these can be either the numeric ID or the alias with the format organization@project:study. #' @param cohort Select variants with calculated stats for the selected cohorts. #' @param cohortStatsRef Reference Allele Frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL<=0.4. #' @param cohortStatsAlt Alternate Allele Frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL<=0.4. @@ -115,7 +114,7 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @section Endpoint /{apiVersion}/analysis/variant/annotation/metadata: #' Read variant annotations metadata from any saved versions. #' @param annotationId Annotation identifier. - #' @param project Project [user@]project where project can be either the ID or the alias. + #' @param project Project [organization@]project where project can be either the ID or the alias. metadataAnnotation=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/annotation", subcategoryId=NULL, action="metadata", params=params, httpMethod="GET", as.queryParam=NULL, ...), @@ -158,11 +157,14 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @section Endpoint /{apiVersion}/analysis/variant/cohort/stats/run: #' Compute cohort variant stats for the selected list of samples. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Cohort variant stats params. runCohortStats=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/cohort/stats", subcategoryId=NULL, action="run", params=params, httpMethod="POST", @@ -175,6 +177,9 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobDescription Job description. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Exomiser parameters. runExomiser=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/exomiser", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -183,19 +188,22 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' Filter and export variants from the variant storage to a file. #' @param include Fields included in the response, whole JSON path must be provided. #' @param exclude Fields excluded in the response, whole JSON path must be provided. - #' @param project Project [user@]project where project can be either the ID or the alias. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param project Project [organization@]project where project can be either the ID or the alias. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Variant export params. runExport=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/export", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), #' @section Endpoint /{apiVersion}/analysis/variant/family/genotypes: #' Calculate the possible genotypes for the members of a family. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param family Family id. #' @param clinicalAnalysis Clinical analysis id. #' @param modeOfInheritance Mode of inheritance. Allowed values: ['AUTOSOMAL_DOMINANT AUTOSOMAL_RECESSIVE X_LINKED_DOMINANT X_LINKED_RECESSIVE Y_LINKED MITOCHONDRIAL DE_NOVO MENDELIAN_ERROR COMPOUND_HETEROZYGOUS UNKNOWN'] @@ -207,11 +215,14 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @section Endpoint /{apiVersion}/analysis/variant/family/qc/run: #' Run quality control (QC) for a given family. It computes the relatedness scores among the family members. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Family QC analysis params. Family ID. Relatedness method, by default 'PLINK/IBD'. Minor allele frequence (MAF) is used to filter variants before computing relatedness, e.g.: 1000G:CEU>0.35 or cohort:ALL>0.05. runFamilyQc=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/family/qc", subcategoryId=NULL, action="run", params=params, httpMethod="POST", @@ -223,7 +234,10 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param file Files to remove. #' @param resume Resume a previously failed indexation. deleteFile=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/file", @@ -236,17 +250,23 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Gatk parameters. Supported Gatk commands: HaplotypeCaller. runGatk=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/gatk", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), #' @section Endpoint /{apiVersion}/analysis/variant/genomePlot/run: #' Generate a genome plot for a given sample. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Genome plot analysis params to customize the plot. The configuration file includes the title, the plot density (i.e., the number of points to display), the general query and the list of tracks. Currently, the supported track types are: COPY-NUMBER, INDEL, REARRANGEMENT and SNV. In addition, each track can contain a specific query. runGenomePlot=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/genomePlot", subcategoryId=NULL, action="run", params=params, httpMethod="POST", @@ -254,44 +274,56 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @section Endpoint /{apiVersion}/analysis/variant/gwas/run: #' Run a Genome Wide Association Study between two cohorts. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Gwas analysis params. runGwas=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/gwas", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), #' @section Endpoint /{apiVersion}/analysis/variant/hrDetect/run: #' Run HRDetect analysis for a given somatic sample. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data HRDetect analysis parameters. runHrDetect=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/hrDetect", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), #' @section Endpoint /{apiVersion}/analysis/variant/index/run: #' [DEPRECATED] Use operation/variant/index. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobDescription Job description. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Variant index params. runIndex=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/index", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), #' @section Endpoint /{apiVersion}/analysis/variant/individual/qc/run: #' Run quality control (QC) for a given individual. It includes inferred sex and mendelian errors (UDP). - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Individual QC analysis params. runIndividualQc=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/individual/qc", subcategoryId=NULL, action="run", params=params, @@ -299,11 +331,14 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @section Endpoint /{apiVersion}/analysis/variant/inferredSex/run: #' Infer sex from chromosome mean coverages. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Inferred sex analysis params. runInferredSex=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/inferredSex", subcategoryId=NULL, action="run", params=params, httpMethod="POST", @@ -336,17 +371,23 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Gene knockout analysis params. runKnockout=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/knockout", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), #' @section Endpoint /{apiVersion}/analysis/variant/mendelianError/run: #' Run mendelian error analysis to infer uniparental disomy regions. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Mendelian error analysis params. runMendelianError=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/mendelianError", subcategoryId=NULL, action="run", params=params, @@ -354,8 +395,8 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @section Endpoint /{apiVersion}/analysis/variant/metadata: #' . - #' @param project Project [user@]project where project can be either the ID or the alias. - #' @param study Filter variants from the given studies, these can be either the numeric ID or the alias with the format user@project:study. + #' @param project Project [organization@]project where project can be either the ID or the alias. + #' @param study Filter variants from the given studies, these can be either the numeric ID or the alias with the format organization@project:study. #' @param file Filter variants from the files specified. This will set includeFile parameter when not provided. #' @param sample Filter variants by sample genotype. This will automatically set 'includeSample' parameter when not provided. This filter accepts multiple 3 forms: 1) List of samples: Samples that contain the main variant. Accepts AND (;) and OR (,) operators. e.g. HG0097,HG0098 . 2) List of samples with genotypes: {sample}:{gt1},{gt2}. Accepts AND (;) and OR (,) operators. e.g. HG0097:0/0;HG0098:0/1,1/1 . Unphased genotypes (e.g. 0/1, 1/1) will also include phased genotypes (e.g. 0|1, 1|0, 1|1), but not vice versa. When filtering by multi-allelic genotypes, any secondary allele will match, regardless of its position e.g. 1/2 will match with genotypes 1/2, 1/3, 1/4, .... Genotype aliases accepted: HOM_REF, HOM_ALT, HET, HET_REF, HET_ALT, HET_MISS and MISS e.g. HG0097:HOM_REF;HG0098:HET_REF,HOM_ALT . 3) Sample with segregation mode: {sample}:{segregation}. Only one sample accepted.Accepted segregation modes: [ autosomalDominant, autosomalRecessive, XLinkedDominant, XLinkedRecessive, YLinked, mitochondrial, deNovo, deNovoStrict, mendelianError, compoundHeterozygous ]. Value is case insensitive. e.g. HG0097:DeNovo Sample must have parents defined and indexed. . #' @param includeStudy List of studies to include in the result. Accepts 'all' and 'none'. @@ -368,7 +409,7 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @section Endpoint /{apiVersion}/analysis/variant/mutationalSignature/query: #' Run mutational signature analysis for a given sample. Use context index. - #' @param study Filter variants from the given studies, these can be either the numeric ID or the alias with the format user@project:study. + #' @param study Filter variants from the given studies, these can be either the numeric ID or the alias with the format organization@project:study. #' @param sample Sample name. #' @param type Variant type. Valid values: SNV, SV. #' @param ct List of SO consequence types, e.g. missense_variant,stop_lost or SO:0001583,SO:0001578. Accepts aliases 'loss_of_function' and 'protein_altering'. @@ -392,11 +433,14 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @section Endpoint /{apiVersion}/analysis/variant/mutationalSignature/run: #' Run mutational signature analysis for a given sample. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Mutational signature analysis parameters to index the genome context for that sample, and to compute both catalogue counts and signature fitting. In order to skip one of them, , use the following keywords: , catalogue, fitting. runMutationalSignature=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/mutationalSignature", subcategoryId=NULL, action="run", params=params, @@ -409,6 +453,9 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Plink params. runPlink=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/plink", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -430,8 +477,8 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @param type List of types, accepted values are SNV, MNV, INDEL, SV, COPY_NUMBER, COPY_NUMBER_LOSS, COPY_NUMBER_GAIN, INSERTION, DELETION, DUPLICATION, TANDEM_DUPLICATION, BREAKEND, e.g. SNV,INDEL. #' @param reference Reference allele. #' @param alternate Main alternate allele. - #' @param project Project [user@]project where project can be either the ID or the alias. - #' @param study Filter variants from the given studies, these can be either the numeric ID or the alias with the format user@project:study. + #' @param project Project [organization@]project where project can be either the ID or the alias. + #' @param study Filter variants from the given studies, these can be either the numeric ID or the alias with the format organization@project:study. #' @param file Filter variants from the files specified. This will set includeFile parameter when not provided. #' @param filter Specify the FILTER for any of the files. If 'file' filter is provided, will match the file and the filter. e.g.: PASS,LowGQX. #' @param qual Specify the QUAL for any of the files. If 'file' filter is provided, will match the file and the qual. e.g.: >123.4. @@ -497,11 +544,14 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @section Endpoint /{apiVersion}/analysis/variant/relatedness/run: #' Compute a score to quantify relatedness between samples. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Relatedness analysis params. runRelatedness=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/relatedness", subcategoryId=NULL, action="run", params=params, httpMethod="POST", @@ -514,6 +564,9 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data RvTests parameters. Supported RvTests commands: rvtest, vcf2kinship. runRvtests=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/rvtests", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -523,8 +576,8 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @param savedFilter Use a saved filter at User level. #' @param region List of regions, these can be just a single chromosome name or regions in the format chr:start-end, e.g.: 2,3:100000-200000. #' @param type List of types, accepted values are SNV, MNV, INDEL, SV, COPY_NUMBER, COPY_NUMBER_LOSS, COPY_NUMBER_GAIN, INSERTION, DELETION, DUPLICATION, TANDEM_DUPLICATION, BREAKEND, e.g. SNV,INDEL. - #' @param project Project [user@]project where project can be either the ID or the alias. - #' @param study Filter variants from the given studies, these can be either the numeric ID or the alias with the format user@project:study. + #' @param project Project [organization@]project where project can be either the ID or the alias. + #' @param study Filter variants from the given studies, these can be either the numeric ID or the alias with the format organization@project:study. #' @param file Filter variants from the files specified. This will set includeFile parameter when not provided. #' @param filter Specify the FILTER for any of the files. If 'file' filter is provided, will match the file and the filter. e.g.: PASS,LowGQX. #' @param sample Filter variants by sample genotype. This will automatically set 'includeSample' parameter when not provided. This filter accepts multiple 3 forms: 1) List of samples: Samples that contain the main variant. Accepts AND (;) and OR (,) operators. e.g. HG0097,HG0098 . 2) List of samples with genotypes: {sample}:{gt1},{gt2}. Accepts AND (;) and OR (,) operators. e.g. HG0097:0/0;HG0098:0/1,1/1 . Unphased genotypes (e.g. 0/1, 1/1) will also include phased genotypes (e.g. 0|1, 1|0, 1|1), but not vice versa. When filtering by multi-allelic genotypes, any secondary allele will match, regardless of its position e.g. 1/2 will match with genotypes 1/2, 1/3, 1/4, .... Genotype aliases accepted: HOM_REF, HOM_ALT, HET, HET_REF, HET_ALT, HET_MISS and MISS e.g. HG0097:HOM_REF;HG0098:HET_REF,HOM_ALT . 3) Sample with segregation mode: {sample}:{segregation}. Only one sample accepted.Accepted segregation modes: [ autosomalDominant, autosomalRecessive, XLinkedDominant, XLinkedRecessive, YLinked, mitochondrial, deNovo, deNovoStrict, mendelianError, compoundHeterozygous ]. Value is case insensitive. e.g. HG0097:DeNovo Sample must have parents defined and indexed. . @@ -548,11 +601,14 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @section Endpoint /{apiVersion}/analysis/variant/sample/eligibility/run: #' Filter samples by a complex query involving metadata and variants data. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data . runSampleEligibility=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/sample/eligibility", subcategoryId=NULL, action="run", params=params, @@ -560,11 +616,14 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @section Endpoint /{apiVersion}/analysis/variant/sample/qc/run: #' Run quality control (QC) for a given sample. It includes variant stats, and if the sample is somatic, mutational signature and genome plot are calculated. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Sample QC analysis params. Mutational signature and genome plot are calculated for somatic samples only. In order to skip some metrics, use the following keywords (separated by commas): variant-stats, signature, signature-catalogue, signature-fitting, genome-plot. runSampleQc=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/sample/qc", subcategoryId=NULL, action="run", params=params, httpMethod="POST", @@ -582,11 +641,14 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @section Endpoint /{apiVersion}/analysis/variant/sample/run: #' Get samples given a set of variants. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Sample variant filter params. runSample=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/sample", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -595,7 +657,7 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' Obtain sample variant stats from a sample. #' @param region List of regions, these can be just a single chromosome name or regions in the format chr:start-end, e.g.: 2,3:100000-200000. #' @param type List of types, accepted values are SNV, MNV, INDEL, SV, COPY_NUMBER, COPY_NUMBER_LOSS, COPY_NUMBER_GAIN, INSERTION, DELETION, DUPLICATION, TANDEM_DUPLICATION, BREAKEND, e.g. SNV,INDEL. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param file Filter variants from the files specified. This will set includeFile parameter when not provided. #' @param filter Specify the FILTER for any of the files. If 'file' filter is provided, will match the file and the filter. e.g.: PASS,LowGQX. #' @param sampleData Filter by any SampleData field from samples. [{sample}:]{key}{op}{value}[,;]* . If no sample is specified, will use all samples from "sample" or "genotype" filter. e.g. DP>200 or HG0097:DP>200,HG0098:DP<10 . Many FORMAT fields can be combined. e.g. HG0097:DP>200;GT=1/1,0/1,HG0098:DP<10. @@ -614,11 +676,14 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @section Endpoint /{apiVersion}/analysis/variant/sample/stats/run: #' Compute sample variant stats for the selected list of samples. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Sample variant stats params. Use index=true and indexId='' to store the result in catalog sample QC. indexId=ALL requires an empty query. Use sample=all to compute sample stats of all samples in the variant storage. runSampleStats=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/sample/stats", subcategoryId=NULL, action="run", params=params, httpMethod="POST", @@ -626,12 +691,15 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @section Endpoint /{apiVersion}/analysis/variant/stats/export/run: #' Export calculated variant stats and frequencies. - #' @param project Project [user@]project where project can be either the ID or the alias. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param project Project [organization@]project where project can be either the ID or the alias. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Variant stats export params. runStatsExport=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/stats/export", subcategoryId=NULL, action="run", params=params, httpMethod="POST", @@ -639,11 +707,14 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @section Endpoint /{apiVersion}/analysis/variant/stats/run: #' Compute variant stats for any cohort and any set of variants. - #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Variant stats params. runStats=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/stats", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), diff --git a/opencga-client/src/main/R/build.sh b/opencga-client/src/main/R/build.sh index 9f4533c8a9f..636d910a343 100755 --- a/opencga-client/src/main/R/build.sh +++ b/opencga-client/src/main/R/build.sh @@ -1,3 +1,7 @@ #!/bin/sh -docker run --mount type=bind,source=$1,target=/opt/opencga/R --mount type=bind,source=/tmp,target=/opt/opencga opencb/opencga-r-builder:1.0.0 R CMD build /opt/opencga/R +docker run --rm \ + --mount type=bind,source=$1,target=/opt/opencga/R \ + --mount type=bind,source=/tmp,target=/opt/opencga \ + opencb/opencga-r-builder:1.0.0 \ + R CMD build /opt/opencga/R diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/OpenCGAClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/OpenCGAClient.java index 81e201be6ea..263bb992771 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/OpenCGAClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/OpenCGAClient.java @@ -50,8 +50,9 @@ public OpenCGAClient(ClientConfiguration clientConfiguration) { this.init(null, clientConfiguration); } - public OpenCGAClient(String user, String password, ClientConfiguration clientConfiguration) throws ClientException { - AuthenticationResponse login = login(user, password); + public OpenCGAClient(String organizationId, String user, String password, ClientConfiguration clientConfiguration) + throws ClientException { + AuthenticationResponse login = login(organizationId, user, password); this.init(login, clientConfiguration); } @@ -85,6 +86,10 @@ private void init(AuthenticationResponse tokens, ClientConfiguration clientConfi } + public OrganizationClient getOrganizationClient() { + return getClient(OrganizationClient.class, () -> new OrganizationClient(token, clientConfiguration)); + } + public UserClient getUserClient() { return getClient(UserClient.class, () -> new UserClient(token, clientConfiguration)); } @@ -176,13 +181,14 @@ public AuthenticationResponse refresh() throws ClientException { /** * Logs in the user. * + * @param organizationId Organization id. * @param user userId. * @param password Password. * @return AuthenticationResponse object. * @throws ClientException when it is not possible logging in. */ - public AuthenticationResponse login(String user, String password) throws ClientException { - RestResponse login = getUserClient().login(new LoginParams(user, password), null); + public AuthenticationResponse login(String organizationId, String user, String password) throws ClientException { + RestResponse login = getUserClient().login(new LoginParams(organizationId, user, password), null); updateTokenFromClients(login); this.userId = user; return login.firstResult(); diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AdminClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AdminClient.java index 5601a4e5329..928fd361341 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AdminClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AdminClient.java @@ -20,15 +20,16 @@ import org.opencb.opencga.client.config.ClientConfiguration; import org.opencb.opencga.client.exceptions.ClientException; import org.opencb.opencga.client.rest.*; +import org.opencb.opencga.core.models.Acl; import org.opencb.opencga.core.models.admin.GroupSyncParams; import org.opencb.opencga.core.models.admin.InstallationParams; import org.opencb.opencga.core.models.admin.JWTParams; -import org.opencb.opencga.core.models.admin.UserCreateParams; import org.opencb.opencga.core.models.admin.UserImportParams; import org.opencb.opencga.core.models.admin.UserUpdateGroup; import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.study.Group; import org.opencb.opencga.core.models.user.User; +import org.opencb.opencga.core.models.user.UserCreateParams; import org.opencb.opencga.core.response.RestResponse; @@ -36,7 +37,6 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-04-25 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -45,7 +45,6 @@ /** * This class contains methods for the Admin webservices. - * Client version: 2.12.4-SNAPSHOT * PATH: admin */ public class AdminClient extends AbstractParentClient { @@ -75,19 +74,6 @@ public RestResponse groupByAudit(String fields, String entity, Object return execute("admin", null, "audit", null, "groupBy", params, GET, ObjectMap.class); } - /** - * Sync Catalog into the Solr. - * @param params Map containing any of the following optional parameters. - * collection: Collection to be indexed (file, sample, individual, family, cohort and/or job). If not provided, all of them will - * be indexed. - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse indexStatsCatalog(ObjectMap params) throws ClientException { - params = params != null ? params : new ObjectMap(); - return execute("admin", null, "catalog", null, "indexStats", params, POST, Boolean.class); - } - /** * Install OpenCGA database. * @param data JSON containing the mandatory parameters. @@ -103,11 +89,13 @@ public RestResponse installCatalog(InstallationParams data) throws Cl /** * Change JWT secret key. * @param data JSON containing the parameters. + * @param params Map containing any of the following optional parameters. + * organization: Organization id. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ - public RestResponse jwtCatalog(JWTParams data) throws ClientException { - ObjectMap params = new ObjectMap(); + public RestResponse jwtCatalog(JWTParams data, ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); params.put("body", data); return execute("admin", null, "catalog", null, "jwt", params, POST, ObjectMap.class); } @@ -127,15 +115,32 @@ public RestResponse createUsers(UserCreateParams data) throws ClientExcept /** * Import users or a group of users from LDAP or AAD. * @param data JSON containing the parameters. + * @param params Map containing any of the following optional parameters. + * organization: Organization id. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ - public RestResponse importUsers(UserImportParams data) throws ClientException { - ObjectMap params = new ObjectMap(); + public RestResponse importUsers(UserImportParams data, ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); params.put("body", data); return execute("admin", null, "users", null, "import", params, POST, User.class); } + /** + * User permissions. + * @param params Map containing any of the following optional parameters. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. + * entryIds: Comma separated list of entry ids. + * permissions: Comma separated list of permissions to be retrieved. + * category: Category corresponding to the id's provided. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse permissionsUsers(ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); + return execute("admin", null, "users", null, "permissions", params, GET, Acl.class); + } + /** * User search method. * @param params Map containing any of the following optional parameters. @@ -144,8 +149,8 @@ public RestResponse importUsers(UserImportParams data) throws ClientExcept * limit: Number of results to be returned. * skip: Number of results to skip. * count: Get the total number of results matching the query. Deactivated by default. + * organization: Organization id. * user: User ID. - * account: Account type [GUEST, FULL, ADMINISTRATOR]. * authenticationId: Authentication origin ID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -158,11 +163,13 @@ public RestResponse searchUsers(ObjectMap params) throws ClientException /** * Synchronise a group of users from an authentication origin with a group in a study from catalog. * @param data JSON containing the parameters. + * @param params Map containing any of the following optional parameters. + * organization: Organization id. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ - public RestResponse syncUsers(GroupSyncParams data) throws ClientException { - ObjectMap params = new ObjectMap(); + public RestResponse syncUsers(GroupSyncParams data, ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); params.put("body", data); return execute("admin", null, "users", null, "sync", params, POST, Group.class); } @@ -172,6 +179,7 @@ public RestResponse syncUsers(GroupSyncParams data) throws ClientExceptio * @param user User ID. * @param data JSON containing the parameters. * @param params Map containing any of the following optional parameters. + * organization: Organization id. * action: Action to be performed: ADD or REMOVE user to/from groups. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AlignmentClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AlignmentClient.java index ea517320b92..238df94066d 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AlignmentClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AlignmentClient.java @@ -40,7 +40,6 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-04-25 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -49,7 +48,6 @@ /** * This class contains methods for the Alignment webservices. - * Client version: 2.12.4-SNAPSHOT * PATH: analysis/alignment */ public class AlignmentClient extends AbstractParentClient { @@ -67,6 +65,10 @@ public AlignmentClient(String token, ClientConfiguration configuration) { * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobDescription: Job description. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -85,6 +87,10 @@ public RestResponse runBwa(BwaWrapperParams data, ObjectMap params) throws * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobDescription: Job description. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -103,6 +109,10 @@ public RestResponse runCoverageIndex(CoverageIndexParams data, ObjectMap pa * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobDescription: Job description. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -117,7 +127,7 @@ public RestResponse coverageQcGeneCoverageStatsRun(AlignmentGeneCoverageSta * Query the coverage of an alignment file for regions or genes. * @param file File ID. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * region: Comma separated list of regions 'chr:start-end, e.g.: 2,3:63500-65000. * gene: Comma separated list of genes, e.g.: BCRA2,TP53. * offset: Offset to extend the region, gene or exon at up and downstream. @@ -141,7 +151,7 @@ public RestResponse queryCoverage(String file, ObjectMap params) * @param file1 Input file #1 (e.g. somatic file). * @param file2 Input file #2 (e.g. germline file). * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * skipLog2: Do not apply Log2 to normalise the coverage ratio. * region: Comma separated list of regions 'chr:start-end, e.g.: 2,3:63500-65000. * gene: Comma separated list of genes, e.g.: BCRA2,TP53. @@ -164,7 +174,7 @@ public RestResponse ratioCoverage(String file1, String file2, Ob * @param file File ID. * @param gene Comma separated list of genes, e.g.: BCRA2,TP53. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * threshold: Only regions whose coverage depth is under this threshold will be reported. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -186,6 +196,10 @@ public RestResponse statsCoverage(String file, String gene, O * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobDescription: Job description. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -204,6 +218,10 @@ public RestResponse runDeeptools(DeeptoolsWrapperParams data, ObjectMap par * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobDescription: Job description. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -222,6 +240,10 @@ public RestResponse runFastqc(FastqcWrapperParams data, ObjectMap params) t * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobDescription: Job description. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -241,6 +263,10 @@ public RestResponse runIndex(AlignmentIndexParams data, ObjectMap params) t * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobDescription: Job description. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -260,6 +286,10 @@ public RestResponse runPicard(PicardWrapperParams data, ObjectMap params) t * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobDescription: Job description. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -276,7 +306,7 @@ public RestResponse runQc(AlignmentQcParams data, ObjectMap params) throws * limit: Number of results to be returned. * skip: Number of results to skip. * count: Get the total number of results matching the query. Deactivated by default. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * region: Comma separated list of regions 'chr:start-end, e.g.: 2,3:63500-65000. * gene: Comma separated list of genes, e.g.: BCRA2,TP53. * offset: Offset to extend the region, gene or exon at up and downstream. @@ -311,6 +341,10 @@ public RestResponse query(String file, ObjectMap params) throws C * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobDescription: Job description. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java index e80fc6b059f..7e94ff82723 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java @@ -33,6 +33,7 @@ import org.opencb.opencga.core.models.clinical.ClinicalAnalysisAclEntryList; import org.opencb.opencga.core.models.clinical.ClinicalAnalysisAclUpdateParams; import org.opencb.opencga.core.models.clinical.ClinicalAnalysisCreateParams; +import org.opencb.opencga.core.models.clinical.ClinicalAnalysisLoadParams; import org.opencb.opencga.core.models.clinical.ClinicalAnalysisUpdateParams; import org.opencb.opencga.core.models.clinical.ClinicalReport; import org.opencb.opencga.core.models.clinical.ExomiserInterpretationAnalysisParams; @@ -54,7 +55,6 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-04-25 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -63,7 +63,6 @@ /** * This class contains methods for the ClinicalAnalysis webservices. - * Client version: 2.12.4-SNAPSHOT * PATH: analysis/clinical */ public class ClinicalAnalysisClient extends AbstractParentClient { @@ -78,7 +77,7 @@ public ClinicalAnalysisClient(String token, ClientConfiguration configuration) { * @param action Action to be performed [ADD, SET, REMOVE or RESET]. * @param data JSON containing the parameters to add ACLs. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * propagate: Propagate permissions to related families, individuals, samples and files. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -97,7 +96,7 @@ public RestResponse updateAcl(String members, Stri * @param path Path where the TSV file is located in OpenCGA or where it should be located. * @param data JSON containing the 'content' of the TSV file if this has not yet been registered into OpenCGA. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * parents: Flag indicating whether to create parent directories if they don't exist (only when TSV file was not previously * associated). * annotationSetId: Annotation set id. If not provided, variableSetId will be used. @@ -117,7 +116,7 @@ public RestResponse loadAnnotationSets(String variableSetId, String path, T * Update Clinical Analysis configuration. * @param data Configuration params to update. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -134,7 +133,7 @@ public RestResponse updateClinicalConfiguration(ClinicalAnalysisStudy * @param params Map containing any of the following optional parameters. * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * skipCreateDefaultInterpretation: Flag to skip creating and initialise an empty default primary interpretation (Id will be * '{clinicalAnalysisId}.1'). This flag is only considered if no Interpretation object is passed. * includeResult: Flag indicating to include the created or updated document result in the response. @@ -151,7 +150,7 @@ public RestResponse create(ClinicalAnalysisCreateParams data, * Clinical Analysis distinct method. * @param field Comma separated list of fields for which to obtain the distinct values. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * id: Comma separated list of Clinical Analysis IDs up to a maximum of 100. Also admits basic regular expressions using the * operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * uuid: Comma separated list of Clinical Analysis UUIDs up to a maximum of 100. @@ -176,6 +175,7 @@ public RestResponse create(ClinicalAnalysisCreateParams data, * dueDate: Clinical Analysis due date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. * qualityControlSummary: Clinical Analysis quality control summary. * release: Release when it was created. + * snapshot: Snapshot value (Latest version of the entry in the specified release). * status: Filter by status. * internalStatus: Filter by internal status. * annotation: Annotation filters. Example: age>30;gender=FEMALE. For more information, please visit @@ -194,10 +194,11 @@ public RestResponse distinct(String field, ObjectMap params) throws C * Interpretation distinct method. * @param field Comma separated list of fields for which to obtain the distinct values. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * id: Comma separated list of Interpretation IDs up to a maximum of 100. Also admits basic regular expressions using the * operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * uuid: Comma separated list of Interpretation UUIDs up to a maximum of 100. + * name: Comma separated list of Interpretation names up to a maximum of 100. * clinicalAnalysisId: Clinical Analysis id. * analystId: Analyst ID. * methodName: Interpretation method name. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' @@ -227,10 +228,11 @@ public RestResponse distinctInterpretation(String field, ObjectMap pa * limit: Number of results to be returned. * skip: Number of results to skip. * sort: Sort the results. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * id: Comma separated list of Interpretation IDs up to a maximum of 100. Also admits basic regular expressions using the * operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * uuid: Comma separated list of Interpretation UUIDs up to a maximum of 100. + * name: Comma separated list of Interpretation names up to a maximum of 100. * clinicalAnalysisId: Clinical Analysis id. * analystId: Analyst ID. * methodName: Interpretation method name. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' @@ -257,7 +259,7 @@ public RestResponse searchInterpretation(ObjectMap params) throw * @param params Map containing any of the following optional parameters. * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * version: Comma separated list of interpretation versions. 'all' to get all the interpretation versions. Not supported if * multiple interpretation ids are provided. * deleted: Boolean to retrieve deleted entries. @@ -273,11 +275,15 @@ public RestResponse infoInterpretation(String interpretations, O * Run cancer tiering interpretation analysis. * @param data Cancer tiering interpretation analysis params. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -290,13 +296,17 @@ public RestResponse runInterpreterCancerTiering(CancerTieringInterpretation /** * Run exomiser interpretation analysis. - * @param data Exomizer interpretation analysis params. + * @param data Exomiser interpretation analysis params. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -310,11 +320,15 @@ public RestResponse runInterpreterExomiser(ExomiserInterpretationAnalysisPa * Run TEAM interpretation analysis. * @param data TEAM interpretation analysis params. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -328,11 +342,15 @@ public RestResponse runInterpreterTeam(TeamInterpretationAnalysisParams dat * Run tiering interpretation analysis. * @param data Tiering interpretation analysis params. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -346,11 +364,15 @@ public RestResponse runInterpreterTiering(TieringInterpretationAnalysisPara * Run Zetta interpretation analysis. * @param data Zetta interpretation analysis params. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -360,6 +382,28 @@ public RestResponse runInterpreterZetta(ZettaInterpretationAnalysisParams d return execute("analysis", null, "clinical/interpreter/zetta", null, "run", params, POST, Job.class); } + /** + * Load clinical analyses from a file. + * @param data Parameters to load clinical analysis in OpenCGA catalog from a file. + * @param params Map containing any of the following optional parameters. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. + * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. + * jobDescription: Job description. + * jobDependsOn: Comma separated list of existing job IDs the job will depend on. + * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse load(ClinicalAnalysisLoadParams data, ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); + params.put("body", data); + return execute("analysis", null, "clinical", null, "load", params, POST, Job.class); + } + /** * RGA aggregation stats. * @param field List of fields separated by semicolons, e.g.: clinicalSignificances;type. For nested fields use >>, e.g.: @@ -387,7 +431,7 @@ public RestResponse runInterpreterZetta(ZettaInterpretationAnalysisParams d * clinicalSignificance: Filter by clinical significance. * populationFrequency: Filter by population frequency. * consequenceType: Filter by consequence type. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -428,7 +472,7 @@ public RestResponse aggregationStatsRga(String field, ObjectMap para * clinicalSignificance: Filter by clinical significance. * populationFrequency: Filter by population frequency. * consequenceType: Filter by consequence type. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -463,7 +507,7 @@ public RestResponse queryRgaGene(ObjectMap params) throws Cli * clinicalSignificance: Filter by clinical significance. * populationFrequency: Filter by population frequency. * consequenceType: Filter by consequence type. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -476,11 +520,15 @@ public RestResponse summaryRgaGene(ObjectMap params) thro * Generate Recessive Gene Analysis secondary index. * @param data Recessive Gene Analysis index params. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * auxiliarIndex: Index auxiliar collection to improve performance assuming RGA is completely indexed. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -519,7 +567,7 @@ public RestResponse runRgaIndex(RgaAnalysisParams data, ObjectMap params) t * clinicalSignificance: Filter by clinical significance. * populationFrequency: Filter by population frequency. * consequenceType: Filter by consequence type. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -554,7 +602,7 @@ public RestResponse queryRgaIndividual(ObjectMap params) t * clinicalSignificance: Filter by clinical significance. * populationFrequency: Filter by population frequency. * consequenceType: Filter by consequence type. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -594,7 +642,7 @@ public RestResponse summaryRgaIndividual(ObjectMap * clinicalSignificance: Filter by clinical significance. * populationFrequency: Filter by population frequency. * consequenceType: Filter by consequence type. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -629,7 +677,7 @@ public RestResponse queryRgaVariant(ObjectMap params) throws * clinicalSignificance: Filter by clinical significance. * populationFrequency: Filter by population frequency. * consequenceType: Filter by consequence type. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -647,7 +695,7 @@ public RestResponse summaryRgaVariant(ObjectMap params * skip: Number of results to skip. * count: Get the total number of results matching the query. Deactivated by default. * flattenAnnotations: Flatten the annotations?. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * id: Comma separated list of Clinical Analysis IDs up to a maximum of 100. Also admits basic regular expressions using the * operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * uuid: Comma separated list of Clinical Analysis UUIDs up to a maximum of 100. @@ -672,6 +720,7 @@ public RestResponse summaryRgaVariant(ObjectMap params * dueDate: Clinical Analysis due date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. * qualityControlSummary: Clinical Analysis quality control summary. * release: Release when it was created. + * snapshot: Snapshot value (Latest version of the entry in the specified release). * status: Filter by status. * internalStatus: Filter by internal status. * annotation: Annotation filters. Example: age>30;gender=FEMALE. For more information, please visit @@ -704,7 +753,7 @@ public RestResponse search(ObjectMap params) throws ClientExce * type: List of types, accepted values are SNV, MNV, INDEL, SV, COPY_NUMBER, COPY_NUMBER_LOSS, COPY_NUMBER_GAIN, INSERTION, * DELETION, DUPLICATION, TANDEM_DUPLICATION, BREAKEND, e.g. SNV,INDEL. * study: Filter variants from the given studies, these can be either the numeric ID or the alias with the format - * user@project:study. + * organization@project:study. * file: Filter variants from the files specified. This will set includeFile parameter when not provided. * filter: Specify the FILTER for any of the files. If 'file' filter is provided, will match the file and the filter. e.g.: * PASS,LowGQX. @@ -790,7 +839,7 @@ public RestResponse queryVariant(ObjectMap params) throws Clien * Returns the acl of the clinical analyses. If member is provided, it will only return the acl for the member. * @param clinicalAnalyses Comma separated list of clinical analysis IDs or names up to a maximum of 100. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * member: User or group ID. * silent: Boolean to retrieve all possible entries that are queried for, false to raise an exception whenever one of the entries * looked for cannot be shown for whichever reason. @@ -806,7 +855,7 @@ public RestResponse acl(String clinicalAnalyses, O * Delete clinical analyses. * @param clinicalAnalyses Comma separated list of clinical analysis IDs or names up to a maximum of 100. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * force: Force deletion if the ClinicalAnalysis contains interpretations or is locked. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -823,7 +872,7 @@ public RestResponse delete(String clinicalAnalyses, ObjectMap * @param params Map containing any of the following optional parameters. * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * commentsAction: Action to be performed if the array of comments is being updated. * flagsAction: Action to be performed if the array of flags is being updated. * analystsAction: Action to be performed if the array of analysts is being updated. @@ -849,7 +898,7 @@ public RestResponse update(String clinicalAnalyses, ClinicalAn * containing the comma separated variables to be removed as a value when the action is REMOVE or a json with only the key 'reset' * containing the comma separated variables that will be set to the default value when the action is RESET. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * action: Action to be performed: ADD to add new annotations; REPLACE to replace the value of an already existing annotation; * SET to set the new list of annotations removing any possible old annotations; REMOVE to remove some annotations; RESET to * set some annotations to the default value configured in the corresponding variables of the VariableSet if any. @@ -871,7 +920,9 @@ public RestResponse updateAnnotationSetsAnnotations(String clinicalAnaly * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. * flattenAnnotations: Flatten the annotations?. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. + * version: Comma separated list of clinical versions. 'all' to get all the clinical versions. Not supported if multiple clinical + * ids are provided. * deleted: Boolean to retrieve deleted entries. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -888,7 +939,7 @@ public RestResponse info(String clinicalAnalysis, ObjectMap pa * @param params Map containing any of the following optional parameters. * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. - * study: [[user@]project:]study id. + * study: [[organization@]project:]study id. * setAs: Set interpretation as. * includeResult: Flag indicating to include the created or updated document result in the response. * @return a RestResponse object. @@ -906,7 +957,7 @@ public RestResponse createInterpretation(String clinicalAnalysis * @param clinicalAnalysis Clinical analysis ID. * @param interpretations Interpretation IDs of the Clinical Analysis. * @param params Map containing any of the following optional parameters. - * study: [[user@]project:]study ID. + * study: [[organization@]project:]study ID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -922,7 +973,7 @@ public RestResponse clearInterpretation(String clinicalAnalysis, * @param clinicalAnalysis Clinical analysis ID. * @param interpretations Interpretation IDs of the Clinical Analysis. * @param params Map containing any of the following optional parameters. - * study: [[user@]project:]study ID. + * study: [[organization@]project:]study ID. * setAsPrimary: Interpretation id to set as primary from the list of secondaries in case of deleting the actual primary one. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -940,7 +991,7 @@ public RestResponse deleteInterpretation(String clinicalAnalysis * @param interpretation Interpretation ID. * @param version Version to revert to. * @param params Map containing any of the following optional parameters. - * study: [[user@]project:]study ID. + * study: [[organization@]project:]study ID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -960,7 +1011,7 @@ public RestResponse revertInterpretation(String clinicalAnalysis * @param params Map containing any of the following optional parameters. * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. - * study: [[user@]project:]study ID. + * study: [[organization@]project:]study ID. * primaryFindingsAction: Action to be performed if the array of primary findings is being updated. * methodsAction: Action to be performed if the array of methods is being updated. * secondaryFindingsAction: Action to be performed if the array of secondary findings is being updated. @@ -987,7 +1038,7 @@ public RestResponse updateInterpretation(String clinicalAnalysis * @param params Map containing any of the following optional parameters. * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * commentsAction: Action to be performed if the array of comments is being updated. * supportingEvidencesAction: Action to be performed if the array of supporting evidences is being updated. * filesAction: Action to be performed if the array of files is being updated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalClient.java deleted file mode 100644 index cf6b1f8d70f..00000000000 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalClient.java +++ /dev/null @@ -1,914 +0,0 @@ -/* -* Copyright 2015-2022 OpenCB -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -package org.opencb.opencga.client.rest.clients; - -import org.opencb.biodata.models.clinical.interpretation.ClinicalVariant; -import org.opencb.commons.datastore.core.FacetField; -import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.opencga.client.config.ClientConfiguration; -import org.opencb.opencga.client.exceptions.ClientException; -import org.opencb.opencga.client.rest.AbstractParentClient; -import org.opencb.opencga.core.models.analysis.knockout.KnockoutByGeneSummary; -import org.opencb.opencga.core.models.analysis.knockout.KnockoutByIndividual; -import org.opencb.opencga.core.models.analysis.knockout.KnockoutByIndividualSummary; -import org.opencb.opencga.core.models.analysis.knockout.KnockoutByVariant; -import org.opencb.opencga.core.models.analysis.knockout.KnockoutByVariantSummary; -import org.opencb.opencga.core.models.analysis.knockout.RgaKnockoutByGene; -import org.opencb.opencga.core.models.clinical.CancerTieringInterpretationAnalysisParams; -import org.opencb.opencga.core.models.clinical.ClinicalAnalysis; -import org.opencb.opencga.core.models.clinical.ClinicalAnalysisAclUpdateParams; -import org.opencb.opencga.core.models.clinical.ClinicalAnalysisCreateParams; -import org.opencb.opencga.core.models.clinical.ClinicalAnalysisUpdateParams; -import org.opencb.opencga.core.models.clinical.Interpretation; -import org.opencb.opencga.core.models.clinical.InterpretationCreateParams; -import org.opencb.opencga.core.models.clinical.InterpretationUpdateParams; -import org.opencb.opencga.core.models.clinical.RgaAnalysisParams; -import org.opencb.opencga.core.models.clinical.TeamInterpretationAnalysisParams; -import org.opencb.opencga.core.models.clinical.TieringInterpretationAnalysisParams; -import org.opencb.opencga.core.models.clinical.ZettaInterpretationAnalysisParams; -import org.opencb.opencga.core.models.job.Job; -import org.opencb.opencga.core.models.study.configuration.ClinicalAnalysisStudyConfiguration; -import org.opencb.opencga.core.response.RestResponse; - - -/* -* WARNING: AUTOGENERATED CODE -* -* This code was generated by a tool. -* Autogenerated on: 2022-08-02 08:25:32 ->>>>>>> release-2.4.x ->>>>>>> release-2.4.x -* -* Manual changes to this file may cause unexpected behavior in your application. -* Manual changes to this file will be overwritten if the code is regenerated. -*/ - - -/** - * This class contains methods for the Clinical webservices. - * Client version: 2.3.3-SNAPSHOT [fb5e7ed17448243b10ddd619bdf973e4b20b0a74] ->>>>>>> release-2.4.x ->>>>>>> release-2.4.x - * PATH: analysis/clinical - */ -public class ClinicalClient extends AbstractParentClient { - - public ClinicalClient(String token, ClientConfiguration configuration) { - super(token, configuration); - } - - /** - * Update the set of permissions granted for the member. - * @param members Comma separated list of user or group IDs. - * @param action Action to be performed [ADD, SET, REMOVE or RESET]. - * @param data JSON containing the parameters to add ACLs. - * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. - * propagate: Propagate permissions to related families, individuals, samples and files. - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse updateAcl(String members, String action, ClinicalAnalysisAclUpdateParams data, ObjectMap params) - throws ClientException { - params = params != null ? params : new ObjectMap(); - params.putIfNotNull("action", action); - params.put("body", data); - return execute("analysis", null, "clinical/acl", members, "update", params, POST, ObjectMap.class); - } - - /** - * Update Clinical Analysis configuration. - * @param data Configuration params to update. - * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse updateClinicalConfiguration(ClinicalAnalysisStudyConfiguration data, ObjectMap params) - throws ClientException { - params = params != null ? params : new ObjectMap(); - params.put("body", data); - return execute("analysis", null, "clinical/clinical/configuration", null, "update", params, POST, ObjectMap.class); - } - - /** - * Create a new clinical analysis. - * @param data JSON containing clinical analysis information. - * @param params Map containing any of the following optional parameters. - * include: Fields included in the response, whole JSON path must be provided. - * exclude: Fields excluded in the response, whole JSON path must be provided. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. - * skipCreateDefaultInterpretation: Flag to skip creating and initialise an empty default primary interpretation (Id will be - * '{clinicalAnalysisId}.1'). This flag is only considered if no Interpretation object is passed. - * includeResult: Flag indicating to include the created or updated document result in the response. - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse create(ClinicalAnalysisCreateParams data, ObjectMap params) throws ClientException { - params = params != null ? params : new ObjectMap(); - params.put("body", data); - return execute("analysis", null, "clinical", null, "create", params, POST, ClinicalAnalysis.class); - } - - /** - * Clinical Analysis distinct method. - * @param field Field for which to obtain the distinct values. - * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. - * id: Comma separated list of Clinical Analysis IDs up to a maximum of 100. - * uuid: Comma separated list of Clinical Analysis UUIDs up to a maximum of 100. - * type: Clinical Analysis type. - * disorder: Clinical Analysis disorder. - * files: Clinical Analysis files. - * sample: Sample associated to the proband or any member of a family. - * individual: Proband or any member of a family. - * proband: Clinical Analysis proband. - * probandSamples: Clinical Analysis proband samples. - * family: Clinical Analysis family. - * familyMembers: Clinical Analysis family members. - * familyMemberSamples: Clinical Analysis family members samples. - * panels: Clinical Analysis panels. - * locked: Locked Clinical Analyses. - * analystId: Clinical Analysis analyst id. - * priority: Clinical Analysis priority. - * flags: Clinical Analysis flags. - * creationDate: Clinical Analysis Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. - * modificationDate: Clinical Analysis Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. - * dueDate: Clinical Analysis due date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. - * qualityControlSummary: Clinical Analysis quality control summary. - * release: Release when it was created. - * status: Filter by status. - * internalStatus: Filter by internal status. - * deleted: Boolean to retrieve deleted entries. - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse distinct(String field, ObjectMap params) throws ClientException { - params = params != null ? params : new ObjectMap(); - params.putIfNotNull("field", field); - return execute("analysis", null, "clinical", null, "distinct", params, GET, ObjectMap.class); - } - - /** - * Interpretation distinct method. - * @param field Field for which to obtain the distinct values. - * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. - * id: Comma separated list of Interpretation IDs up to a maximum of 100. - * uuid: Comma separated list of Interpretation UUIDs up to a maximum of 100. - * clinicalAnalysisId: Clinical Analysis id. - * analystId: Analyst ID. - * methodName: Interpretation method name. - * panels: Interpretation panels. - * primaryFindings: Interpretation primary findings. - * secondaryFindings: Interpretation secondary findings. - * creationDate: Interpretation Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. - * modificationDate: Interpretation Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. - * status: Filter by status. - * internalStatus: Filter by internal status. - * release: Release when it was created. - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse distinctInterpretation(String field, ObjectMap params) throws ClientException { - params = params != null ? params : new ObjectMap(); - params.putIfNotNull("field", field); - return execute("analysis", null, "clinical/interpretation", null, "distinct", params, GET, ObjectMap.class); - } - - /** - * Search clinical interpretations. - * @param params Map containing any of the following optional parameters. - * include: Fields included in the response, whole JSON path must be provided. - * exclude: Fields excluded in the response, whole JSON path must be provided. - * limit: Number of results to be returned. - * skip: Number of results to skip. - * sort: Sort the results. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. - * id: Comma separated list of Interpretation IDs up to a maximum of 100. - * uuid: Comma separated list of Interpretation UUIDs up to a maximum of 100. - * clinicalAnalysisId: Clinical Analysis id. - * analystId: Analyst ID. - * methodName: Interpretation method name. - * panels: Interpretation panels. - * primaryFindings: Interpretation primary findings. - * secondaryFindings: Interpretation secondary findings. - * creationDate: Interpretation Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. - * modificationDate: Interpretation Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. - * status: Filter by status. - * internalStatus: Filter by internal status. - * release: Release when it was created. - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse searchInterpretation(ObjectMap params) throws ClientException { - params = params != null ? params : new ObjectMap(); - return execute("analysis", null, "clinical/interpretation", null, "search", params, GET, Interpretation.class); - } - - /** - * Clinical interpretation information. - * @param interpretations Comma separated list of clinical interpretation IDs up to a maximum of 100. - * @param params Map containing any of the following optional parameters. - * include: Fields included in the response, whole JSON path must be provided. - * exclude: Fields excluded in the response, whole JSON path must be provided. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. - * version: Comma separated list of interpretation versions. 'all' to get all the interpretation versions. Not supported if - * multiple interpretation ids are provided. - * deleted: Boolean to retrieve deleted entries. - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse infoInterpretation(String interpretations, ObjectMap params) throws ClientException { - params = params != null ? params : new ObjectMap(); - return execute("analysis", null, "clinical/interpretation", interpretations, "info", params, GET, Interpretation.class); - } - - /** - * Run cancer tiering interpretation analysis. - * @param data Cancer tiering interpretation analysis params. - * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. - * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. - * jobDescription: Job description. - * jobDependsOn: Comma separated list of existing job IDs the job will depend on. - * jobTags: Job tags. - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse runInterpreterCancerTiering(CancerTieringInterpretationAnalysisParams data, ObjectMap params) - throws ClientException { - params = params != null ? params : new ObjectMap(); - params.put("body", data); - return execute("analysis", null, "clinical/interpreter/cancerTiering", null, "run", params, POST, Job.class); - } - - /** - * Run TEAM interpretation analysis. - * @param data TEAM interpretation analysis params. - * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. - * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. - * jobDescription: Job description. - * jobDependsOn: Comma separated list of existing job IDs the job will depend on. - * jobTags: Job tags. - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse runInterpreterTeam(TeamInterpretationAnalysisParams data, ObjectMap params) throws ClientException { - params = params != null ? params : new ObjectMap(); - params.put("body", data); - return execute("analysis", null, "clinical/interpreter/team", null, "run", params, POST, Job.class); - } - - /** - * Run tiering interpretation analysis. - * @param data Tiering interpretation analysis params. - * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. - * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. - * jobDescription: Job description. - * jobDependsOn: Comma separated list of existing job IDs the job will depend on. - * jobTags: Job tags. - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse runInterpreterTiering(TieringInterpretationAnalysisParams data, ObjectMap params) throws ClientException { - params = params != null ? params : new ObjectMap(); - params.put("body", data); - return execute("analysis", null, "clinical/interpreter/tiering", null, "run", params, POST, Job.class); - } - - /** - * Run Zetta interpretation analysis. - * @param data Zetta interpretation analysis params. - * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. - * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. - * jobDescription: Job description. - * jobDependsOn: Comma separated list of existing job IDs the job will depend on. - * jobTags: Job tags. - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse runInterpreterZetta(ZettaInterpretationAnalysisParams data, ObjectMap params) throws ClientException { - params = params != null ? params : new ObjectMap(); - params.put("body", data); - return execute("analysis", null, "clinical/interpreter/zetta", null, "run", params, POST, Job.class); - } - - /** - * RGA aggregation stats. - * @param field List of fields separated by semicolons, e.g.: clinicalSignificances;type. For nested fields use >>, e.g.: - * type>>clinicalSignificances;knockoutType. - * @param params Map containing any of the following optional parameters. - * limit: Number of results to be returned. - * skip: Number of results to skip. - * sampleId: Filter by sample id. - * individualId: Filter by individual id. - * sex: Filter by sex. - * phenotypes: Filter by phenotypes. - * disorders: Filter by disorders. - * numParents: Filter by the number of parents registered. - * geneId: Filter by gene id. - * geneName: Filter by gene name. - * chromosome: Filter by chromosome. - * start: Filter by start position. - * end: Filter by end position. - * transcriptId: Filter by transcript id. - * variants: Filter by variant id. - * dbSnps: Filter by DB_SNP id. - * knockoutType: Filter by knockout type. - * filter: Filter by filter (PASS, NOT_PASS). - * type: Filter by variant type. - * clinicalSignificance: Filter by clinical significance. - * populationFrequency: Filter by population frequency. - * consequenceType: Filter by consequence type. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse aggregationStatsRga(String field, ObjectMap params) throws ClientException { - params = params != null ? params : new ObjectMap(); - params.putIfNotNull("field", field); - return execute("analysis", null, "clinical/rga", null, "aggregationStats", params, GET, FacetField.class); - } - - /** - * Query gene RGA. - * @param params Map containing any of the following optional parameters. - * include: Fields included in the response, whole JSON path must be provided. - * exclude: Fields excluded in the response, whole JSON path must be provided. - * limit: Number of results to be returned. - * skip: Number of results to skip. - * count: Get the total number of results matching the query. Deactivated by default. - * includeIndividual: Include only the comma separated list of individuals to the response. - * skipIndividual: Number of individuals to skip. - * limitIndividual: Limit number of individuals returned (default: 1000). - * sampleId: Filter by sample id. - * individualId: Filter by individual id. - * sex: Filter by sex. - * phenotypes: Filter by phenotypes. - * disorders: Filter by disorders. - * numParents: Filter by the number of parents registered. - * geneId: Filter by gene id. - * geneName: Filter by gene name. - * chromosome: Filter by chromosome. - * start: Filter by start position. - * end: Filter by end position. - * transcriptId: Filter by transcript id. - * variants: Filter by variant id. - * dbSnps: Filter by DB_SNP id. - * knockoutType: Filter by knockout type. - * filter: Filter by filter (PASS, NOT_PASS). - * type: Filter by variant type. - * clinicalSignificance: Filter by clinical significance. - * populationFrequency: Filter by population frequency. - * consequenceType: Filter by consequence type. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse queryRgaGene(ObjectMap params) throws ClientException { - params = params != null ? params : new ObjectMap(); - return execute("analysis", null, "clinical/rga/gene", null, "query", params, GET, RgaKnockoutByGene.class); - } - - /** - * RGA gene summary stats. - * @param params Map containing any of the following optional parameters. - * limit: Number of results to be returned. - * skip: Number of results to skip. - * count: Get the total number of results matching the query. Deactivated by default. - * sampleId: Filter by sample id. - * individualId: Filter by individual id. - * sex: Filter by sex. - * phenotypes: Filter by phenotypes. - * disorders: Filter by disorders. - * numParents: Filter by the number of parents registered. - * geneId: Filter by gene id. - * geneName: Filter by gene name. - * chromosome: Filter by chromosome. - * start: Filter by start position. - * end: Filter by end position. - * transcriptId: Filter by transcript id. - * variants: Filter by variant id. - * dbSnps: Filter by DB_SNP id. - * knockoutType: Filter by knockout type. - * filter: Filter by filter (PASS, NOT_PASS). - * type: Filter by variant type. - * clinicalSignificance: Filter by clinical significance. - * populationFrequency: Filter by population frequency. - * consequenceType: Filter by consequence type. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse summaryRgaGene(ObjectMap params) throws ClientException { - params = params != null ? params : new ObjectMap(); - return execute("analysis", null, "clinical/rga/gene", null, "summary", params, GET, KnockoutByGeneSummary.class); - } - - /** - * Generate Recessive Gene Analysis secondary index. - * @param data Recessive Gene Analysis index params. - * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. - * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. - * jobDescription: Job description. - * jobDependsOn: Comma separated list of existing job IDs the job will depend on. - * jobTags: Job tags. - * auxiliarIndex: Index auxiliar collection to improve performance assuming RGA is completely indexed. - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse runRgaIndex(RgaAnalysisParams data, ObjectMap params) throws ClientException { - params = params != null ? params : new ObjectMap(); - params.put("body", data); - return execute("analysis", null, "clinical/rga/index", null, "run", params, POST, Job.class); - } - - /** - * Query individual RGA. - * @param params Map containing any of the following optional parameters. - * include: Fields included in the response, whole JSON path must be provided. - * exclude: Fields excluded in the response, whole JSON path must be provided. - * limit: Number of results to be returned. - * skip: Number of results to skip. - * count: Get the total number of results matching the query. Deactivated by default. - * sampleId: Filter by sample id. - * individualId: Filter by individual id. - * sex: Filter by sex. - * phenotypes: Filter by phenotypes. - * disorders: Filter by disorders. - * numParents: Filter by the number of parents registered. - * geneId: Filter by gene id. - * geneName: Filter by gene name. - * chromosome: Filter by chromosome. - * start: Filter by start position. - * end: Filter by end position. - * transcriptId: Filter by transcript id. - * variants: Filter by variant id. - * dbSnps: Filter by DB_SNP id. - * knockoutType: Filter by knockout type. - * filter: Filter by filter (PASS, NOT_PASS). - * type: Filter by variant type. - * clinicalSignificance: Filter by clinical significance. - * populationFrequency: Filter by population frequency. - * consequenceType: Filter by consequence type. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse queryRgaIndividual(ObjectMap params) throws ClientException { - params = params != null ? params : new ObjectMap(); - return execute("analysis", null, "clinical/rga/individual", null, "query", params, GET, KnockoutByIndividual.class); - } - - /** - * RGA individual summary stats. - * @param params Map containing any of the following optional parameters. - * limit: Number of results to be returned. - * skip: Number of results to skip. - * count: Get the total number of results matching the query. Deactivated by default. - * sampleId: Filter by sample id. - * individualId: Filter by individual id. - * sex: Filter by sex. - * phenotypes: Filter by phenotypes. - * disorders: Filter by disorders. - * numParents: Filter by the number of parents registered. - * geneId: Filter by gene id. - * geneName: Filter by gene name. - * chromosome: Filter by chromosome. - * start: Filter by start position. - * end: Filter by end position. - * transcriptId: Filter by transcript id. - * variants: Filter by variant id. - * dbSnps: Filter by DB_SNP id. - * knockoutType: Filter by knockout type. - * filter: Filter by filter (PASS, NOT_PASS). - * type: Filter by variant type. - * clinicalSignificance: Filter by clinical significance. - * populationFrequency: Filter by population frequency. - * consequenceType: Filter by consequence type. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse summaryRgaIndividual(ObjectMap params) throws ClientException { - params = params != null ? params : new ObjectMap(); - return execute("analysis", null, "clinical/rga/individual", null, "summary", params, GET, KnockoutByIndividualSummary.class); - } - - /** - * Query variant RGA. - * @param params Map containing any of the following optional parameters. - * include: Fields included in the response, whole JSON path must be provided. - * exclude: Fields excluded in the response, whole JSON path must be provided. - * limit: Number of results to be returned. - * skip: Number of results to skip. - * count: Get the total number of results matching the query. Deactivated by default. - * includeIndividual: Include only the comma separated list of individuals to the response. - * skipIndividual: Number of individuals to skip. - * limitIndividual: Limit number of individuals returned (default: 1000). - * sampleId: Filter by sample id. - * individualId: Filter by individual id. - * sex: Filter by sex. - * phenotypes: Filter by phenotypes. - * disorders: Filter by disorders. - * numParents: Filter by the number of parents registered. - * geneId: Filter by gene id. - * geneName: Filter by gene name. - * chromosome: Filter by chromosome. - * start: Filter by start position. - * end: Filter by end position. - * transcriptId: Filter by transcript id. - * variants: Filter by variant id. - * dbSnps: Filter by DB_SNP id. - * knockoutType: Filter by knockout type. - * filter: Filter by filter (PASS, NOT_PASS). - * type: Filter by variant type. - * clinicalSignificance: Filter by clinical significance. - * populationFrequency: Filter by population frequency. - * consequenceType: Filter by consequence type. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse queryRgaVariant(ObjectMap params) throws ClientException { - params = params != null ? params : new ObjectMap(); - return execute("analysis", null, "clinical/rga/variant", null, "query", params, GET, KnockoutByVariant.class); - } - - /** - * RGA variant summary stats. - * @param params Map containing any of the following optional parameters. - * limit: Number of results to be returned. - * skip: Number of results to skip. - * count: Get the total number of results matching the query. Deactivated by default. - * sampleId: Filter by sample id. - * individualId: Filter by individual id. - * sex: Filter by sex. - * phenotypes: Filter by phenotypes. - * disorders: Filter by disorders. - * numParents: Filter by the number of parents registered. - * geneId: Filter by gene id. - * geneName: Filter by gene name. - * chromosome: Filter by chromosome. - * start: Filter by start position. - * end: Filter by end position. - * transcriptId: Filter by transcript id. - * variants: Filter by variant id. - * dbSnps: Filter by DB_SNP id. - * knockoutType: Filter by knockout type. - * filter: Filter by filter (PASS, NOT_PASS). - * type: Filter by variant type. - * clinicalSignificance: Filter by clinical significance. - * populationFrequency: Filter by population frequency. - * consequenceType: Filter by consequence type. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse summaryRgaVariant(ObjectMap params) throws ClientException { - params = params != null ? params : new ObjectMap(); - return execute("analysis", null, "clinical/rga/variant", null, "summary", params, GET, KnockoutByVariantSummary.class); - } - - /** - * Clinical analysis search. - * @param params Map containing any of the following optional parameters. - * include: Fields included in the response, whole JSON path must be provided. - * exclude: Fields excluded in the response, whole JSON path must be provided. - * limit: Number of results to be returned. - * skip: Number of results to skip. - * count: Get the total number of results matching the query. Deactivated by default. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. - * id: Comma separated list of Clinical Analysis IDs up to a maximum of 100. - * uuid: Comma separated list of Clinical Analysis UUIDs up to a maximum of 100. - * type: Clinical Analysis type. - * disorder: Clinical Analysis disorder. - * files: Clinical Analysis files. - * sample: Sample associated to the proband or any member of a family. - * individual: Proband or any member of a family. - * proband: Clinical Analysis proband. - * probandSamples: Clinical Analysis proband samples. - * family: Clinical Analysis family. - * familyMembers: Clinical Analysis family members. - * familyMemberSamples: Clinical Analysis family members samples. - * panels: Clinical Analysis panels. - * locked: Locked Clinical Analyses. - * analystId: Clinical Analysis analyst id. - * priority: Clinical Analysis priority. - * flags: Clinical Analysis flags. - * creationDate: Clinical Analysis Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. - * modificationDate: Clinical Analysis Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. - * dueDate: Clinical Analysis due date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. - * qualityControlSummary: Clinical Analysis quality control summary. - * release: Release when it was created. - * status: Filter by status. - * internalStatus: Filter by internal status. - * deleted: Boolean to retrieve deleted entries. - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse search(ObjectMap params) throws ClientException { - params = params != null ? params : new ObjectMap(); - return execute("analysis", null, "clinical", null, "search", params, GET, ClinicalAnalysis.class); - } - - /** - * Fetch actionable clinical variants. - * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. - * sample: Sample ID. - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse actionableVariant(ObjectMap params) throws ClientException { - params = params != null ? params : new ObjectMap(); - return execute("analysis", null, "clinical/variant", null, "actionable", params, GET, ClinicalVariant.class); - } - - /** - * Fetch clinical variants. - * @param params Map containing any of the following optional parameters. - * include: Fields included in the response, whole JSON path must be provided. - * exclude: Fields excluded in the response, whole JSON path must be provided. - * limit: Number of results to be returned. - * skip: Number of results to skip. - * count: Get the total number of results matching the query. Deactivated by default. - * approximateCount: Get an approximate count, instead of an exact total count. Reduces execution time. - * approximateCountSamplingSize: Sampling size to get the approximate count. Larger values increase accuracy but also increase - * execution time. - * savedFilter: Use a saved filter at User level. - * id: List of IDs, these can be rs IDs (dbSNP) or variants in the format chrom:start:ref:alt, e.g. rs116600158,19:7177679:C:T. - * region: List of regions, these can be just a single chromosome name or regions in the format chr:start-end, e.g.: - * 2,3:100000-200000. - * type: List of types, accepted values are SNV, MNV, INDEL, SV, COPY_NUMBER, COPY_NUMBER_LOSS, COPY_NUMBER_GAIN, INSERTION, - * DELETION, DUPLICATION, TANDEM_DUPLICATION, BREAKEND, e.g. SNV,INDEL. - * study: Filter variants from the given studies, these can be either the numeric ID or the alias with the format - * user@project:study. - * file: Filter variants from the files specified. This will set includeFile parameter when not provided. - * filter: Specify the FILTER for any of the files. If 'file' filter is provided, will match the file and the filter. e.g.: - * PASS,LowGQX. - * qual: Specify the QUAL for any of the files. If 'file' filter is provided, will match the file and the qual. e.g.: >123.4. - * fileData: Filter by file data (i.e. FILTER, QUAL and INFO columns from VCF file). [{file}:]{key}{op}{value}[,;]* . If no file - * is specified, will use all files from "file" filter. e.g. AN>200 or file_1.vcf:AN>200;file_2.vcf:AN<10 . Many fields can - * be combined. e.g. file_1.vcf:AN>200;DB=true;file_2.vcf:AN<10,FILTER=PASS,LowDP. - * sample: Filter variants by sample genotype. This will automatically set 'includeSample' parameter when not provided. This - * filter accepts multiple 3 forms: 1) List of samples: Samples that contain the main variant. Accepts AND (;) and OR (,) - * operators. e.g. HG0097,HG0098 . 2) List of samples with genotypes: {sample}:{gt1},{gt2}. Accepts AND (;) and OR (,) - * operators. e.g. HG0097:0/0;HG0098:0/1,1/1 . Unphased genotypes (e.g. 0/1, 1/1) will also include phased genotypes (e.g. - * 0|1, 1|0, 1|1), but not vice versa. When filtering by multi-allelic genotypes, any secondary allele will match, - * regardless of its position e.g. 1/2 will match with genotypes 1/2, 1/3, 1/4, .... Genotype aliases accepted: HOM_REF, - * HOM_ALT, HET, HET_REF, HET_ALT, HET_MISS and MISS e.g. HG0097:HOM_REF;HG0098:HET_REF,HOM_ALT . 3) Sample with - * segregation mode: {sample}:{segregation}. Only one sample accepted.Accepted segregation modes: [ autosomalDominant, - * autosomalRecessive, XLinkedDominant, XLinkedRecessive, YLinked, mitochondrial, deNovo, mendelianError, - * compoundHeterozygous ]. Value is case insensitive. e.g. HG0097:DeNovo Sample must have parents defined and indexed. . - * sampleData: Filter by any SampleData field from samples. [{sample}:]{key}{op}{value}[,;]* . If no sample is specified, will - * use all samples from "sample" or "genotype" filter. e.g. DP>200 or HG0097:DP>200,HG0098:DP<10 . Many FORMAT fields can be - * combined. e.g. HG0097:DP>200;GT=1/1,0/1,HG0098:DP<10. - * sampleAnnotation: Selects some samples using metadata information from Catalog. e.g. - * age>20;phenotype=hpo:123,hpo:456;name=smith. - * cohort: Select variants with calculated stats for the selected cohorts. - * cohortStatsRef: Reference Allele Frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL<=0.4. - * cohortStatsAlt: Alternate Allele Frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL<=0.4. - * cohortStatsMaf: Minor Allele Frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL<=0.4. - * cohortStatsMgf: Minor Genotype Frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL<=0.4. - * cohortStatsPass: Filter PASS frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL>0.8. - * missingAlleles: Number of missing alleles: [{study:}]{cohort}[<|>|<=|>=]{number}. - * missingGenotypes: Number of missing genotypes: [{study:}]{cohort}[<|>|<=|>=]{number}. - * score: Filter by variant score: [{study:}]{score}[<|>|<=|>=]{number}. - * family: Filter variants where any of the samples from the given family contains the variant (HET or HOM_ALT). - * familyDisorder: Specify the disorder to use for the family segregation. - * familySegregation: Filter by segregation mode from a given family. Accepted values: [ autosomalDominant, autosomalRecessive, - * XLinkedDominant, XLinkedRecessive, YLinked, mitochondrial, deNovo, mendelianError, compoundHeterozygous ]. - * familyMembers: Sub set of the members of a given family. - * familyProband: Specify the proband child to use for the family segregation. - * gene: List of genes, most gene IDs are accepted (HGNC, Ensembl gene, ...). This is an alias to 'xref' parameter. - * ct: List of SO consequence types, e.g. missense_variant,stop_lost or SO:0001583,SO:0001578. Accepts aliases 'loss_of_function' - * and 'protein_altering'. - * xref: List of any external reference, these can be genes, proteins or variants. Accepted IDs include HGNC, Ensembl genes, - * dbSNP, ClinVar, HPO, Cosmic, ... - * biotype: List of biotypes, e.g. protein_coding. - * proteinSubstitution: Protein substitution scores include SIFT and PolyPhen. You can query using the score - * {protein_score}[<|>|<=|>=]{number} or the description {protein_score}[~=|=]{description} e.g. polyphen>0.1,sift=tolerant. - * conservation: Filter by conservation score: {conservation_score}[<|>|<=|>=]{number} e.g. phastCons>0.5,phylop<0.1,gerp>0.1. - * populationFrequencyAlt: Alternate Population Frequency: {study}:{population}[<|>|<=|>=]{number}. e.g. 1000G:ALL<0.01. - * populationFrequencyRef: Reference Population Frequency: {study}:{population}[<|>|<=|>=]{number}. e.g. 1000G:ALL<0.01. - * populationFrequencyMaf: Population minor allele frequency: {study}:{population}[<|>|<=|>=]{number}. e.g. 1000G:ALL<0.01. - * transcriptFlag: List of transcript flags. e.g. canonical, CCDS, basic, LRG, MANE Select, MANE Plus Clinical, EGLH_HaemOnc, - * TSO500. - * geneTraitId: List of gene trait association id. e.g. "umls:C0007222" , "OMIM:269600". - * go: List of GO (Gene Ontology) terms. e.g. "GO:0002020". - * expression: List of tissues of interest. e.g. "lung". - * proteinKeyword: List of Uniprot protein variant annotation keywords. - * drug: List of drug names. - * functionalScore: Functional score: {functional_score}[<|>|<=|>=]{number} e.g. cadd_scaled>5.2 , cadd_raw<=0.3. - * clinical: Clinical source: clinvar, cosmic. - * clinicalSignificance: Clinical significance: benign, likely_benign, likely_pathogenic, pathogenic. - * clinicalConfirmedStatus: Clinical confirmed status. - * customAnnotation: Custom annotation: {key}[<|>|<=|>=]{number} or {key}[~=|=]{text}. - * panel: Filter by genes from the given disease panel. - * panelModeOfInheritance: Filter genes from specific panels that match certain mode of inheritance. Accepted values : [ - * autosomalDominant, autosomalRecessive, XLinkedDominant, XLinkedRecessive, YLinked, mitochondrial, deNovo, mendelianError, - * compoundHeterozygous ]. - * panelConfidence: Filter genes from specific panels that match certain confidence. Accepted values : [ high, medium, low, - * rejected ]. - * panelRoleInCancer: Filter genes from specific panels that match certain role in cancer. Accepted values : [ both, oncogene, - * tumorSuppressorGene, fusion ]. - * panelFeatureType: Filter elements from specific panels by type. Accepted values : [ gene, region, str, variant ]. - * panelIntersection: Intersect panel genes and regions with given genes and regions from que input query. This will prevent - * returning variants from regions out of the panel. - * trait: List of traits, based on ClinVar, HPO, COSMIC, i.e.: IDs, histologies, descriptions,... - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse queryVariant(ObjectMap params) throws ClientException { - params = params != null ? params : new ObjectMap(); - return execute("analysis", null, "clinical/variant", null, "query", params, GET, ClinicalVariant.class); - } - - /** - * Returns the acl of the clinical analyses. If member is provided, it will only return the acl for the member. - * @param clinicalAnalyses Comma separated list of clinical analysis IDs or names up to a maximum of 100. - * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. - * member: User or group ID. - * silent: Boolean to retrieve all possible entries that are queried for, false to raise an exception whenever one of the entries - * looked for cannot be shown for whichever reason. - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse acl(String clinicalAnalyses, ObjectMap params) throws ClientException { - params = params != null ? params : new ObjectMap(); - return execute("analysis", null, "clinical", clinicalAnalyses, "acl", params, GET, ObjectMap.class); - } - - /** - * Delete clinical analyses. - * @param clinicalAnalyses Comma separated list of clinical analysis IDs or names up to a maximum of 100. - * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. - * force: Force deletion if the ClinicalAnalysis contains interpretations or is locked. - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse delete(String clinicalAnalyses, ObjectMap params) throws ClientException { - params = params != null ? params : new ObjectMap(); - return execute("analysis", null, "clinical", clinicalAnalyses, "delete", params, DELETE, ClinicalAnalysis.class); - } - - /** - * Update clinical analysis attributes. - * @param clinicalAnalyses Comma separated list of clinical analysis IDs. - * @param data JSON containing clinical analysis information. - * @param params Map containing any of the following optional parameters. - * include: Fields included in the response, whole JSON path must be provided. - * exclude: Fields excluded in the response, whole JSON path must be provided. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. - * commentsAction: Action to be performed if the array of comments is being updated. - * flagsAction: Action to be performed if the array of flags is being updated. - * filesAction: Action to be performed if the array of files is being updated. - * panelsAction: Action to be performed if the array of panels is being updated. - * includeResult: Flag indicating to include the created or updated document result in the response. - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse update(String clinicalAnalyses, ClinicalAnalysisUpdateParams data, ObjectMap params) - throws ClientException { - params = params != null ? params : new ObjectMap(); - params.put("body", data); - return execute("analysis", null, "clinical", clinicalAnalyses, "update", params, POST, ClinicalAnalysis.class); - } - - /** - * Clinical analysis info. - * @param clinicalAnalysis Comma separated list of clinical analysis IDs or names up to a maximum of 100. - * @param params Map containing any of the following optional parameters. - * include: Fields included in the response, whole JSON path must be provided. - * exclude: Fields excluded in the response, whole JSON path must be provided. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. - * deleted: Boolean to retrieve deleted entries. - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse info(String clinicalAnalysis, ObjectMap params) throws ClientException { - params = params != null ? params : new ObjectMap(); - return execute("analysis", null, "clinical", clinicalAnalysis, "info", params, GET, ClinicalAnalysis.class); - } - - /** - * Create a new Interpretation. - * @param clinicalAnalysis Clinical analysis ID. - * @param data JSON containing clinical interpretation information. - * @param params Map containing any of the following optional parameters. - * include: Fields included in the response, whole JSON path must be provided. - * exclude: Fields excluded in the response, whole JSON path must be provided. - * study: [[user@]project:]study id. - * setAs: Set interpretation as. - * includeResult: Flag indicating to include the created or updated document result in the response. - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse createInterpretation(String clinicalAnalysis, InterpretationCreateParams data, ObjectMap params) - throws ClientException { - params = params != null ? params : new ObjectMap(); - params.put("body", data); - return execute("analysis/clinical", clinicalAnalysis, "interpretation", null, "create", params, POST, Interpretation.class); - } - - /** - * Clear the fields of the main interpretation of the Clinical Analysis. - * @param clinicalAnalysis Clinical analysis ID. - * @param interpretations Interpretation IDs of the Clinical Analysis. - * @param params Map containing any of the following optional parameters. - * study: [[user@]project:]study ID. - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse clearInterpretation(String clinicalAnalysis, String interpretations, ObjectMap params) - throws ClientException { - params = params != null ? params : new ObjectMap(); - return execute("analysis/clinical", clinicalAnalysis, "interpretation", interpretations, "clear", params, POST, - Interpretation.class); - } - - /** - * Delete interpretation. - * @param clinicalAnalysis Clinical analysis ID. - * @param interpretations Interpretation IDs of the Clinical Analysis. - * @param params Map containing any of the following optional parameters. - * study: [[user@]project:]study ID. - * setAsPrimary: Interpretation id to set as primary from the list of secondaries in case of deleting the actual primary one. - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse deleteInterpretation(String clinicalAnalysis, String interpretations, ObjectMap params) - throws ClientException { - params = params != null ? params : new ObjectMap(); - return execute("analysis/clinical", clinicalAnalysis, "interpretation", interpretations, "delete", params, DELETE, - Interpretation.class); - } - - /** - * Revert to a previous interpretation version. - * @param clinicalAnalysis Clinical analysis ID. - * @param interpretation Interpretation ID. - * @param version Version to revert to. - * @param params Map containing any of the following optional parameters. - * study: [[user@]project:]study ID. - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse revertInterpretation(String clinicalAnalysis, String interpretation, int version, ObjectMap params) - throws ClientException { - params = params != null ? params : new ObjectMap(); - params.putIfNotNull("version", version); - return execute("analysis/clinical", clinicalAnalysis, "interpretation", interpretation, "revert", params, POST, - Interpretation.class); - } - - /** - * Update interpretation fields. - * @param clinicalAnalysis Clinical analysis ID. - * @param interpretation Interpretation ID. - * @param data JSON containing clinical interpretation information. - * @param params Map containing any of the following optional parameters. - * include: Fields included in the response, whole JSON path must be provided. - * exclude: Fields excluded in the response, whole JSON path must be provided. - * study: [[user@]project:]study ID. - * primaryFindingsAction: Action to be performed if the array of primary findings is being updated. - * methodsAction: Action to be performed if the array of methods is being updated. - * secondaryFindingsAction: Action to be performed if the array of secondary findings is being updated. - * commentsAction: Action to be performed if the array of comments is being updated. To REMOVE or REPLACE, the date will need to - * be provided to identify the comment. - * panelsAction: Action to be performed if the array of panels is being updated. - * setAs: Set interpretation as. - * includeResult: Flag indicating to include the created or updated document result in the response. - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse updateInterpretation(String clinicalAnalysis, String interpretation, InterpretationUpdateParams - data, ObjectMap params) throws ClientException { - params = params != null ? params : new ObjectMap(); - params.put("body", data); - return execute("analysis/clinical", clinicalAnalysis, "interpretation", interpretation, "update", params, POST, - Interpretation.class); - } -} diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/CohortClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/CohortClient.java index 64dfe81be6b..0bcb27158a8 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/CohortClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/CohortClient.java @@ -17,7 +17,6 @@ package org.opencb.opencga.client.rest.clients; import java.lang.Object; -import org.opencb.commons.datastore.core.FacetField; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.client.config.ClientConfiguration; import org.opencb.opencga.client.exceptions.ClientException; @@ -37,7 +36,6 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-04-25 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -46,7 +44,6 @@ /** * This class contains methods for the Cohort webservices. - * Client version: 2.12.4-SNAPSHOT * PATH: cohorts */ public class CohortClient extends AbstractParentClient { @@ -61,7 +58,7 @@ public CohortClient(String token, ClientConfiguration configuration) { * @param action Action to be performed [ADD, SET, REMOVE or RESET]. * @param data JSON containing the parameters to add ACLs. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -73,38 +70,13 @@ public RestResponse updateAcl(String members, String action, return execute("cohorts", null, "acl", members, "update", params, POST, CohortAclEntryList.class); } - /** - * Fetch catalog cohort stats. - * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. - * type: Type. - * creationYear: Creation year. - * creationMonth: Creation month (JANUARY, FEBRUARY...). - * creationDay: Creation day. - * creationDayOfWeek: Creation day of week (MONDAY, TUESDAY...). - * numSamples: Number of samples. - * status: Status. - * release: Release. - * annotation: Annotation filters. Example: age>30;gender=FEMALE. For more information, please visit - * http://docs.opencb.org/display/opencga/AnnotationSets+1.4.0. - * default: Calculate default stats. - * field: List of fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: - * studies>>biotype;type;numSamples[0..10]:1. - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse aggregationStats(ObjectMap params) throws ClientException { - params = params != null ? params : new ObjectMap(); - return execute("cohorts", null, null, null, "aggregationStats", params, GET, FacetField.class); - } - /** * Load annotation sets from a TSV file. * @param variableSetId Variable set ID or name. * @param path Path where the TSV file is located in OpenCGA or where it should be located. * @param data JSON containing the 'content' of the TSV file if this has not yet been registered into OpenCGA. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * parents: Flag indicating whether to create parent directories if they don't exist (only when TSV file was not previously * associated). * annotationSetId: Annotation set id. If not provided, variableSetId will be used. @@ -126,7 +98,7 @@ public RestResponse loadAnnotationSets(String variableSetId, String path, T * @param params Map containing any of the following optional parameters. * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * variableSet: Deprecated: Use /generate web service and filter by annotation. * variable: Deprecated: Use /generate web service and filter by annotation. * includeResult: Flag indicating to include the created or updated document result in the response. @@ -143,7 +115,7 @@ public RestResponse create(CohortCreateParams data, ObjectMap params) th * Cohort distinct method. * @param field Comma separated list of fields for which to obtain the distinct values. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * id: Comma separated list of cohort IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', * i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * name: Comma separated list of cohort names up to a maximum of 100. Also admits basic regular expressions using the operator @@ -175,7 +147,7 @@ public RestResponse distinct(String field, ObjectMap params) throws Clie * @param params Map containing any of the following optional parameters. * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * id: Comma separated list sample IDs or UUIDs up to a maximum of 100. * somatic: Somatic sample. * individualId: Individual ID or UUID. @@ -212,7 +184,7 @@ public RestResponse generate(CohortGenerateParams data, ObjectMap params * skip: Number of results to skip. * count: Get the total number of results matching the query. Deactivated by default. * flattenAnnotations: Flatten the annotations?. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * id: Comma separated list of cohort IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', * i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * name: Comma separated list of cohort names up to a maximum of 100. Also admits basic regular expressions using the operator @@ -241,7 +213,7 @@ public RestResponse search(ObjectMap params) throws ClientException { * Return the acl of the cohort. If member is provided, it will only return the acl for the member. * @param cohorts Comma separated list of cohort IDs or UUIDs up to a maximum of 100. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * member: User or group id. * silent: Boolean to retrieve all possible entries that are queried for, false to raise an exception whenever one of the entries * looked for cannot be shown for whichever reason. @@ -257,7 +229,7 @@ public RestResponse acl(String cohorts, ObjectMap params) th * Delete cohorts. * @param cohorts Comma separated list of cohort ids. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -273,7 +245,7 @@ public RestResponse delete(String cohorts, ObjectMap params) throws Clie * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. * flattenAnnotations: Flatten the annotations?. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * deleted: Boolean to retrieve deleted cohorts. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -290,7 +262,7 @@ public RestResponse info(String cohorts, ObjectMap params) throws Client * @param params Map containing any of the following optional parameters. * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * samplesAction: Action to be performed if the array of samples is being updated. * annotationSetsAction: Action to be performed if the array of annotationSets is being updated. * includeResult: Flag indicating to include the created or updated document result in the response. @@ -311,7 +283,7 @@ public RestResponse update(String cohorts, CohortUpdateParams data, Obje * containing the comma separated variables to be removed as a value when the action is REMOVE or a json with only the key 'reset' * containing the comma separated variables that will be set to the default value when the action is RESET. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * action: Action to be performed: ADD to add new annotations; REPLACE to replace the value of an already existing annotation; * SET to set the new list of annotations removing any possible old annotations; REMOVE to remove some annotations; RESET to * set some annotations to the default value configured in the corresponding variables of the VariableSet if any. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/DiseasePanelClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/DiseasePanelClient.java index ba0285bd561..15d71d98597 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/DiseasePanelClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/DiseasePanelClient.java @@ -35,7 +35,6 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-04-25 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -44,7 +43,6 @@ /** * This class contains methods for the DiseasePanel webservices. - * Client version: 2.12.4-SNAPSHOT * PATH: panels */ public class DiseasePanelClient extends AbstractParentClient { @@ -59,7 +57,7 @@ public DiseasePanelClient(String token, ClientConfiguration configuration) { * @param action Action to be performed [ADD, SET, REMOVE or RESET]. * @param data JSON containing the parameters to update the permissions. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -77,7 +75,7 @@ public RestResponse updateAcl(String members, String action, * @param params Map containing any of the following optional parameters. * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * includeResult: Flag indicating to include the created or updated document result in the response. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -92,7 +90,7 @@ public RestResponse create(PanelCreateParams data, ObjectMap params) thro * Panel distinct method. * @param field Comma separated list of fields for which to obtain the distinct values. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * id: Comma separated list of panel IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', * i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * uuid: Comma separated list of panel UUIDs up to a maximum of 100. @@ -136,11 +134,15 @@ public RestResponse distinct(String field, ObjectMap params) throws Clie * Import panels. * @param data Panel parameters. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobDescription: Job description. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -158,7 +160,7 @@ public RestResponse importPanels(PanelImportParams data, ObjectMap params) * limit: Number of results to be returned. * skip: Number of results to skip. * count: Get the total number of results matching the query. Deactivated by default. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * id: Comma separated list of panel IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', * i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * uuid: Comma separated list of panel UUIDs up to a maximum of 100. @@ -201,7 +203,7 @@ public RestResponse search(ObjectMap params) throws ClientException { * Returns the acl of the panels. If member is provided, it will only return the acl for the member. * @param panels Comma separated list of panel IDs up to a maximum of 100. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * member: User or group id. * silent: Boolean to retrieve all possible entries that are queried for, false to raise an exception whenever one of the entries * looked for cannot be shown for whichever reason. @@ -217,7 +219,7 @@ public RestResponse acl(String panels, ObjectMap params) thro * Delete existing panels. * @param panels Comma separated list of panel ids. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -232,7 +234,7 @@ public RestResponse delete(String panels, ObjectMap params) throws Client * @param params Map containing any of the following optional parameters. * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * version: Comma separated list of panel versions. 'all' to get all the panel versions. Not supported if multiple panel ids are * provided. * deleted: Boolean to retrieve deleted panels. @@ -251,7 +253,7 @@ public RestResponse info(String panels, ObjectMap params) throws ClientEx * @param params Map containing any of the following optional parameters. * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * includeResult: Flag indicating to include the created or updated document result in the response. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FamilyClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FamilyClient.java index e7c9399f9d6..368cf2eb4a5 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FamilyClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FamilyClient.java @@ -17,7 +17,6 @@ package org.opencb.opencga.client.rest.clients; import java.lang.Object; -import org.opencb.commons.datastore.core.FacetField; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.client.config.ClientConfiguration; import org.opencb.opencga.client.exceptions.ClientException; @@ -36,7 +35,6 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-04-25 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -45,7 +43,6 @@ /** * This class contains methods for the Family webservices. - * Client version: 2.12.4-SNAPSHOT * PATH: families */ public class FamilyClient extends AbstractParentClient { @@ -60,7 +57,7 @@ public FamilyClient(String token, ClientConfiguration configuration) { * @param action Action to be performed [ADD, SET, REMOVE or RESET]. * @param data JSON containing the parameters to add ACLs. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * propagate: Propagate family permissions to related individuals and samples. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -73,40 +70,13 @@ public RestResponse updateAcl(String members, String action, return execute("families", null, "acl", members, "update", params, POST, FamilyAclEntryList.class); } - /** - * Fetch catalog family stats. - * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. - * creationYear: Creation year. - * creationMonth: Creation month (JANUARY, FEBRUARY...). - * creationDay: Creation day. - * creationDayOfWeek: Creation day of week (MONDAY, TUESDAY...). - * status: Status. - * phenotypes: Phenotypes. - * release: Release. - * version: Version. - * numMembers: Number of members. - * expectedSize: Expected size. - * annotation: Annotation filters. Example: age>30;gender=FEMALE. For more information, please visit - * http://docs.opencb.org/display/opencga/AnnotationSets+1.4.0. - * default: Calculate default stats. - * field: List of fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: - * studies>>biotype;type;numSamples[0..10]:1. - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse aggregationStats(ObjectMap params) throws ClientException { - params = params != null ? params : new ObjectMap(); - return execute("families", null, null, null, "aggregationStats", params, GET, FacetField.class); - } - /** * Load annotation sets from a TSV file. * @param variableSetId Variable set ID or name. * @param path Path where the TSV file is located in OpenCGA or where it should be located. * @param data JSON containing the 'content' of the TSV file if this has not yet been registered into OpenCGA. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * parents: Flag indicating whether to create parent directories if they don't exist (only when TSV file was not previously * associated). * annotationSetId: Annotation set id. If not provided, variableSetId will be used. @@ -128,7 +98,7 @@ public RestResponse loadAnnotationSets(String variableSetId, String path, T * @param params Map containing any of the following optional parameters. * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * members: Comma separated list of member ids to be associated to the created family. * includeResult: Flag indicating to include the created or updated document result in the response. * @return a RestResponse object. @@ -144,7 +114,7 @@ public RestResponse create(FamilyCreateParams data, ObjectMap params) th * Family distinct method. * @param field Comma separated list of fields for which to obtain the distinct values. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * id: Comma separated list family IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. * '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * name: Comma separated list family names up to a maximum of 100. Also admits basic regular expressions using the operator '~', @@ -187,7 +157,7 @@ public RestResponse distinct(String field, ObjectMap params) throws Clie * skip: Number of results to skip. * count: Get the total number of results matching the query. Deactivated by default. * flattenAnnotations: Flatten the annotations?. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * id: Comma separated list family IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. * '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * name: Comma separated list family names up to a maximum of 100. Also admits basic regular expressions using the operator '~', @@ -224,7 +194,7 @@ public RestResponse search(ObjectMap params) throws ClientException { * Returns the acl of the families. If member is provided, it will only return the acl for the member. * @param families Comma separated list of family IDs or names up to a maximum of 100. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * member: User or group id. * silent: Boolean to retrieve all possible entries that are queried for, false to raise an exception whenever one of the entries * looked for cannot be shown for whichever reason. @@ -240,7 +210,7 @@ public RestResponse acl(String families, ObjectMap params) t * Delete existing families. * @param families Comma separated list of family ids. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -256,7 +226,7 @@ public RestResponse delete(String families, ObjectMap params) throws Cli * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. * flattenAnnotations: Flatten the annotations?. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * version: Comma separated list of family versions. 'all' to get all the family versions. Not supported if multiple family ids * are provided. * deleted: Boolean to retrieve deleted families. @@ -275,7 +245,7 @@ public RestResponse info(String families, ObjectMap params) throws Clien * @param params Map containing any of the following optional parameters. * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * updateRoles: Update the member roles within the family. * updatePedigreeGraph: Update the family pedigree graph. * annotationSetsAction: Action to be performed if the array of annotationSets is being updated. @@ -297,7 +267,7 @@ public RestResponse update(String families, FamilyUpdateParams data, Obj * containing the comma separated variables to be removed as a value when the action is REMOVE or a json with only the key 'reset' * containing the comma separated variables that will be set to the default value when the action is RESET. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * action: Action to be performed: ADD to add new annotations; REPLACE to replace the value of an already existing annotation; * SET to set the new list of annotations removing any possible old annotations; REMOVE to remove some annotations; RESET to * set some annotations to the default value configured in the corresponding variables of the VariableSet if any. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FileClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FileClient.java index 43814c27514..dcf3ae18ef8 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FileClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FileClient.java @@ -18,7 +18,6 @@ import java.io.DataInputStream; import java.lang.Object; -import org.opencb.commons.datastore.core.FacetField; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.client.config.ClientConfiguration; import org.opencb.opencga.client.exceptions.ClientException; @@ -44,7 +43,6 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-04-25 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -53,7 +51,6 @@ /** * This class contains methods for the File webservices. - * Client version: 2.12.4-SNAPSHOT * PATH: files */ public class FileClient extends AbstractParentClient { @@ -68,7 +65,7 @@ public FileClient(String token, ClientConfiguration configuration) { * @param action Action to be performed [ADD, SET, REMOVE or RESET]. * @param data JSON containing the parameters to add ACLs. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -80,46 +77,13 @@ public RestResponse updateAcl(String members, String action, F return execute("files", null, "acl", members, "update", params, POST, FileAclEntryList.class); } - /** - * Fetch catalog file stats. - * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. - * name: Name. - * type: Type. - * format: Format. - * bioformat: Bioformat. - * creationYear: Creation year. - * creationMonth: Creation month (JANUARY, FEBRUARY...). - * creationDay: Creation day. - * creationDayOfWeek: Creation day of week (MONDAY, TUESDAY...). - * status: Status. - * release: Release. - * external: External. - * size: Size. - * software: Software. - * experiment: Experiment. - * numSamples: Number of samples. - * numRelatedFiles: Number of related files. - * annotation: Annotation filters. Example: age>30;gender=FEMALE. For more information, please visit - * http://docs.opencb.org/display/opencga/AnnotationSets+1.4.0. - * default: Calculate default stats. - * field: List of fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: - * studies>>biotype;type;numSamples[0..10]:1. - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse aggregationStats(ObjectMap params) throws ClientException { - params = params != null ? params : new ObjectMap(); - return execute("files", null, null, null, "aggregationStats", params, GET, FacetField.class); - } - /** * Load annotation sets from a TSV file. * @param variableSetId Variable set ID or name. * @param path Path where the TSV file is located in OpenCGA or where it should be located. * @param data JSON containing the 'content' of the TSV file if this has not yet been registered into OpenCGA. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * parents: Flag indicating whether to create parent directories if they don't exist (only when TSV file was not previously * associated). * annotationSetId: Annotation set id. If not provided, variableSetId will be used. @@ -149,7 +113,7 @@ public RestResponse bioformats() throws ClientException { * Create file or folder. * @param data File parameters. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * parents: Create the parent directories if they do not exist. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -164,7 +128,7 @@ public RestResponse create(FileCreateParams data, ObjectMap params) throws * File distinct method. * @param field Comma separated list of fields for which to obtain the distinct values. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * id: Comma separated list of file IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', * i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * uuid: Comma separated list file UUIDs up to a maximum of 100. @@ -215,7 +179,11 @@ public RestResponse distinct(String field, ObjectMap params) throws Clie * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -239,7 +207,7 @@ public RestResponse formats() throws ClientException { * Link an external file into catalog. * @param data File parameters. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * parents: Create the parent directories if they do not exist. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -254,11 +222,15 @@ public RestResponse link(FileLinkParams data, ObjectMap params) throws Cli * Link an external file into catalog asynchronously. * @param data File parameters. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobDescription: Job description. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -272,11 +244,15 @@ public RestResponse runLink(FileLinkToolParams data, ObjectMap params) thro * Associate non-registered samples for files with high volumes of samples. * @param data File parameters. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobDescription: Job description. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -295,7 +271,7 @@ public RestResponse runPostlink(PostLinkToolParams data, ObjectMap params) * skip: Number of results to skip. * count: Get the total number of results matching the query. Deactivated by default. * flattenAnnotations: Boolean indicating to flatten the annotations. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * id: Comma separated list of file IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', * i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * uuid: Comma separated list file UUIDs up to a maximum of 100. @@ -345,7 +321,7 @@ public RestResponse search(ObjectMap params) throws ClientException { * fileFormat: File format. * bioformat: File bioformat. * checksum: Expected MD5 file checksum. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * relativeFilePath: Path within catalog where the file will be located (default: root folder). * description: description. * parents: Create the parent directories if they do not exist. @@ -361,8 +337,8 @@ public RestResponse upload(ObjectMap params) throws ClientException { * Return the acl defined for the file or folder. If member is provided, it will only return the acl for the member. * @param files Comma separated list of file IDs or names up to a maximum of 100. * @param params Map containing any of the following optional parameters. - * study: Comma separated list of Studies [[user@]project:]study where study and project can be either the ID or UUID up to a - * maximum of 100. + * study: Comma separated list of Studies [[organization@]project:]study where study and project can be either the ID or UUID up + * to a maximum of 100. * member: User or group id. * silent: Boolean to retrieve all possible entries that are queried for, false to raise an exception whenever one of the entries * looked for cannot be shown for whichever reason. @@ -378,7 +354,7 @@ public RestResponse acl(String files, ObjectMap params) throws * Delete existing files and folders. * @param files Comma separated list of file ids, names or paths. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * skipTrash: Skip trash and delete the files/folders from disk directly (CANNOT BE RECOVERED). * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -395,7 +371,7 @@ public RestResponse delete(String files, ObjectMap params) throws ClientExc * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. * flattenAnnotations: Flatten the annotations?. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * deleted: Boolean to retrieve deleted files. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -409,7 +385,7 @@ public RestResponse info(String files, ObjectMap params) throws ClientExce * Unlink linked files and folders. * @param files Comma separated list of file ids, names or paths. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -425,7 +401,7 @@ public RestResponse unlink(String files, ObjectMap params) throws ClientExc * @param params Map containing any of the following optional parameters. * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * sampleIdsAction: Action to be performed if the array of samples is being updated. * annotationSetsAction: Action to be performed if the array of annotationSets is being updated. * relatedFilesAction: Action to be performed if the array of relatedFiles is being updated. @@ -447,7 +423,7 @@ public RestResponse update(String files, FileUpdateParams data, ObjectMap * containing the comma separated variables to be removed as a value when the action is REMOVE or a json with only the key 'reset' * containing the comma separated variables that will be set to the default value when the action is RESET. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * action: Action to be performed: ADD to add new annotations; REPLACE to replace the value of an already existing annotation; * SET to set the new list of annotations removing any possible old annotations; REMOVE to remove some annotations; RESET to * set some annotations to the default value configured in the corresponding variables of the VariableSet if any. @@ -465,7 +441,7 @@ public RestResponse updateAnnotationSetsAnnotations(String file, String an * Download file. * @param file File id, name or path. Paths must be separated by : instead of /. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -478,7 +454,7 @@ public RestResponse download(String file, ObjectMap params) thr * Filter lines of the file containing the pattern. * @param file File uuid, id, or name. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * pattern: String pattern. * ignoreCase: Flag to perform a case insensitive search. * maxCount: Stop reading a file after 'n' matching lines. 0 means no limit. @@ -494,7 +470,7 @@ public RestResponse grep(String file, ObjectMap params) throws Clie * Show the first lines of a file (up to a limit). * @param file File uuid, id, or name. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * offset: Starting byte from which the file will be read. * lines: Maximum number of lines to be returned up to a maximum of 1000. * @return a RestResponse object. @@ -509,7 +485,7 @@ public RestResponse head(String file, ObjectMap params) throws Clie * Obtain the base64 content of an image. * @param file File ID. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -525,7 +501,7 @@ public RestResponse image(String file, ObjectMap params) throws Cli * @param params Map containing any of the following optional parameters. * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -539,7 +515,7 @@ public RestResponse move(String file, FileMoveParams data, ObjectMap param * Refresh metadata from the selected file or folder. Return updated files. * @param file File id, name or path. Paths must be separated by : instead of /. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -552,7 +528,7 @@ public RestResponse refresh(String file, ObjectMap params) throws ClientEx * Show the last lines of a file (up to a limit). * @param file File uuid, id, or name. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * lines: Maximum number of lines to be returned up to a maximum of 1000. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -571,7 +547,7 @@ public RestResponse tail(String file, ObjectMap params) throws Clie * limit: Number of results to be returned. * skip: Number of results to skip. * count: Get the total number of results matching the query. Deactivated by default. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -586,7 +562,7 @@ public RestResponse list(String folder, ObjectMap params) throws ClientExc * @param params Map containing any of the following optional parameters. * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * maxDepth: Maximum depth to get files from. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/GA4GHClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/GA4GHClient.java index 2959295e3a0..e1eb10bd8a1 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/GA4GHClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/GA4GHClient.java @@ -27,7 +27,6 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-04-25 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -36,7 +35,6 @@ /** * This class contains methods for the GA4GH webservices. - * Client version: 2.12.4-SNAPSHOT * PATH: ga4gh */ public class GA4GHClient extends AbstractParentClient { @@ -57,7 +55,7 @@ public RestResponse searchReads() throws ClientException { /** * Fetch alignment files using HTSget protocol. - * @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param file File id, name or path. * @param params Map containing any of the following optional parameters. * referenceName: Reference sequence name (Example: 'chr1', '1' or 'chrX'. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/IndividualClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/IndividualClient.java index 66f78beaf7e..571f7b8953f 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/IndividualClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/IndividualClient.java @@ -17,7 +17,6 @@ package org.opencb.opencga.client.rest.clients; import java.lang.Object; -import org.opencb.commons.datastore.core.FacetField; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.client.config.ClientConfiguration; import org.opencb.opencga.client.exceptions.ClientException; @@ -36,7 +35,6 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-04-25 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -45,7 +43,6 @@ /** * This class contains methods for the Individual webservices. - * Client version: 2.12.4-SNAPSHOT * PATH: individuals */ public class IndividualClient extends AbstractParentClient { @@ -61,7 +58,7 @@ public IndividualClient(String token, ClientConfiguration configuration) { * @param data JSON containing the parameters to update the permissions. If propagate flag is set to true, it will propagate the * permissions defined to the samples that are associated to the matching individuals. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * propagate: Propagate individual permissions to related samples. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -74,47 +71,13 @@ public RestResponse updateAcl(String members, String act return execute("individuals", null, "acl", members, "update", params, POST, IndividualAclEntryList.class); } - /** - * Fetch catalog individual stats. - * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. - * hasFather: Has father. - * hasMother: Has mother. - * sex: Sex. - * karyotypicSex: Karyotypic sex. - * ethnicity: Ethnicity. - * population: Population. - * creationYear: Creation year. - * creationMonth: Creation month (JANUARY, FEBRUARY...). - * creationDay: Creation day. - * creationDayOfWeek: Creation day of week (MONDAY, TUESDAY...). - * status: Status. - * lifeStatus: Life status. - * phenotypes: Phenotypes. - * numSamples: Number of samples. - * parentalConsanguinity: Parental consanguinity. - * release: Release. - * version: Version. - * annotation: Annotation filters. Example: age>30;gender=FEMALE. For more information, please visit - * http://docs.opencb.org/display/opencga/AnnotationSets+1.4.0. - * default: Calculate default stats. - * field: List of fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: - * studies>>biotype;type;numSamples[0..10]:1. - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse aggregationStats(ObjectMap params) throws ClientException { - params = params != null ? params : new ObjectMap(); - return execute("individuals", null, null, null, "aggregationStats", params, GET, FacetField.class); - } - /** * Load annotation sets from a TSV file. * @param variableSetId Variable set ID or name. * @param path Path where the TSV file is located in OpenCGA or where it should be located. * @param data JSON containing the 'content' of the TSV file if this has not yet been registered into OpenCGA. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * parents: Flag indicating whether to create parent directories if they don't exist (only when TSV file was not previously * associated). * annotationSetId: Annotation set id. If not provided, variableSetId will be used. @@ -136,7 +99,7 @@ public RestResponse loadAnnotationSets(String variableSetId, String path, T * @param params Map containing any of the following optional parameters. * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * samples: Comma separated list of sample ids to be associated to the created individual. * includeResult: Flag indicating to include the created or updated document result in the response. * @return a RestResponse object. @@ -152,7 +115,7 @@ public RestResponse create(IndividualCreateParams data, ObjectMap pa * Individual distinct method. * @param field Comma separated list of fields for which to obtain the distinct values. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * id: Comma separated list individual IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', * i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * uuid: Comma separated list individual UUIDs up to a maximum of 100. @@ -206,7 +169,7 @@ public RestResponse distinct(String field, ObjectMap params) throws Clie * skip: Number of results to skip. * count: Get the total number of results matching the query. Deactivated by default. * flattenAnnotations: Flatten the annotations?. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * id: Comma separated list individual IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', * i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * uuid: Comma separated list individual UUIDs up to a maximum of 100. @@ -254,7 +217,7 @@ public RestResponse search(ObjectMap params) throws ClientException * Return the acl of the individual. If member is provided, it will only return the acl for the member. * @param individuals Comma separated list of individual IDs, names or UUIDs up to a maximum of 100. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * member: User or group id. * silent: Boolean to retrieve all possible entries that are queried for, false to raise an exception whenever one of the entries * looked for cannot be shown for whichever reason. @@ -271,7 +234,7 @@ public RestResponse acl(String individuals, ObjectMap pa * @param individuals Comma separated list of individual ids. * @param params Map containing any of the following optional parameters. * force: Force the deletion of individuals that already belong to families. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -287,7 +250,7 @@ public RestResponse delete(String individuals, ObjectMap params) thr * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. * flattenAnnotations: Flatten the annotations?. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * version: Comma separated list of individual versions. 'all' to get all the individual versions. Not supported if multiple * individual ids are provided. * deleted: Boolean to retrieve deleted individuals. @@ -306,7 +269,7 @@ public RestResponse info(String individuals, ObjectMap params) throw * @param params Map containing any of the following optional parameters. * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * samplesAction: Action to be performed if the array of samples is being updated. * phenotypesAction: Action to be performed if the array of phenotypes is being updated [SET, ADD, REMOVE]. * disordersAction: Action to be performed if the array of disorders is being updated [SET, ADD, REMOVE]. @@ -329,7 +292,7 @@ public RestResponse update(String individuals, IndividualUpdateParam * containing the comma separated variables to be removed as a value when the action is REMOVE or a json with only the key 'reset' * containing the comma separated variables that will be set to the default value when the action is RESET. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * action: Action to be performed: ADD to add new annotations; REPLACE to replace the value of an already existing annotation; * SET to set the new list of annotations removing any possible old annotations; REMOVE to remove some annotations; RESET to * set some annotations to the default value configured in the corresponding variables of the VariableSet if any. @@ -350,7 +313,7 @@ public RestResponse updateAnnotationSetsAnnotations(String individua * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. * flattenAnnotations: Flatten the annotations?. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * degree: Pedigree degree. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java index f34b38d60f6..77c04be5053 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java @@ -17,7 +17,6 @@ package org.opencb.opencga.client.rest.clients; import java.lang.Object; -import org.opencb.commons.datastore.core.FacetField; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.client.config.ClientConfiguration; import org.opencb.opencga.client.exceptions.ClientException; @@ -37,7 +36,6 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-04-25 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -46,7 +44,6 @@ /** * This class contains methods for the Job webservices. - * Client version: 2.12.4-SNAPSHOT * PATH: jobs */ public class JobClient extends AbstractParentClient { @@ -70,41 +67,11 @@ public RestResponse updateAcl(String members, String action, Jo return execute("jobs", null, "acl", members, "update", params, POST, JobAclEntryList.class); } - /** - * Fetch catalog job stats. - * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. - * toolId: Tool id. - * toolScope: Tool scope. - * toolType: Tool type. - * toolResource: Tool resource. - * userId: User id. - * priority: Priority. - * tags: Tags. - * executorId: Executor id. - * executorFramework: Executor framework. - * creationYear: Creation year. - * creationMonth: Creation month (JANUARY, FEBRUARY...). - * creationDay: Creation day. - * creationDayOfWeek: Creation day of week (MONDAY, TUESDAY...). - * status: Status. - * release: Release. - * default: Calculate default stats. - * field: List of fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: - * studies>>biotype;type;numSamples[0..10]:1. - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse aggregationStats(ObjectMap params) throws ClientException { - params = params != null ? params : new ObjectMap(); - return execute("jobs", null, null, null, "aggregationStats", params, GET, FacetField.class); - } - /** * Register an executed job with POST method. * @param data job. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -118,7 +85,7 @@ public RestResponse create(JobCreateParams data, ObjectMap params) throws C * Job distinct method. * @param field Comma separated list of fields for which to obtain the distinct values. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * otherStudies: Flag indicating the entries being queried can belong to any related study, not just the primary one. * id: Comma separated list of job IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. * '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. @@ -158,7 +125,8 @@ public RestResponse distinct(String field, ObjectMap params) throws Clie * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * jobScheduledStartTime: Time when the job is scheduled to start. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -176,7 +144,7 @@ public RestResponse retry(JobRetryParams data, ObjectMap params) throws Cli * limit: Number of results to be returned. * skip: Number of results to skip. * count: Get the total number of results matching the query. Deactivated by default. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * otherStudies: Flag indicating the entries being queried can belong to any related study, not just the primary one. * id: Comma separated list of job IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. * '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. @@ -211,7 +179,7 @@ public RestResponse search(ObjectMap params) throws ClientException { * Provide a summary of the running jobs. * @param params Map containing any of the following optional parameters. * limit: Maximum number of jobs to be returned. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * internalStatus: Filter by internal status. * priority: Priority of the job. * userId: User that created the job. @@ -244,7 +212,7 @@ public RestResponse acl(String jobs, ObjectMap params) throws C * Delete existing jobs. * @param jobs Comma separated list of job ids. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -259,7 +227,7 @@ public RestResponse delete(String jobs, ObjectMap params) throws ClientExce * @param params Map containing any of the following optional parameters. * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * deleted: Boolean to retrieve deleted jobs. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -276,7 +244,7 @@ public RestResponse info(String jobs, ObjectMap params) throws ClientExcept * @param params Map containing any of the following optional parameters. * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * includeResult: Flag indicating to include the created or updated document result in the response. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -287,11 +255,24 @@ public RestResponse update(String jobs, JobUpdateParams data, ObjectMap par return execute("jobs", jobs, null, null, "update", params, POST, Job.class); } + /** + * Send a signal to kill a pending or running job. + * @param job Job ID or UUID. + * @param params Map containing any of the following optional parameters. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse kill(String job, ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); + return execute("jobs", job, null, null, "kill", params, POST, Job.class); + } + /** * Show the first lines of a log file (up to a limit). * @param job Job ID or UUID. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * offset: Starting byte from which the file will be read. * lines: Maximum number of lines to be returned up to a maximum of 1000. * type: Log file to be shown (stdout or stderr). @@ -307,7 +288,7 @@ public RestResponse headLog(String job, ObjectMap params) throws Cl * Show the last lines of a log file (up to a limit). * @param job Job ID or UUID. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * lines: Maximum number of lines to be returned up to a maximum of 1000. * type: Log file to be shown (stdout or stderr). * @return a RestResponse object. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/MetaClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/MetaClient.java index 3d4dddbf928..8401614fec4 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/MetaClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/MetaClient.java @@ -28,7 +28,6 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-04-25 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -37,7 +36,6 @@ /** * This class contains methods for the Meta webservices. - * Client version: 2.12.4-SNAPSHOT * PATH: meta */ public class MetaClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/OrganizationClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/OrganizationClient.java new file mode 100644 index 00000000000..b5d5a3b7fa1 --- /dev/null +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/OrganizationClient.java @@ -0,0 +1,228 @@ +/* +* Copyright 2015-2024 OpenCB +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.opencb.opencga.client.rest.clients; + +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.opencga.client.config.ClientConfiguration; +import org.opencb.opencga.client.exceptions.ClientException; +import org.opencb.opencga.client.rest.*; +import org.opencb.opencga.core.models.notes.Note; +import org.opencb.opencga.core.models.notes.NoteCreateParams; +import org.opencb.opencga.core.models.notes.NoteUpdateParams; +import org.opencb.opencga.core.models.organizations.Organization; +import org.opencb.opencga.core.models.organizations.OrganizationConfiguration; +import org.opencb.opencga.core.models.organizations.OrganizationCreateParams; +import org.opencb.opencga.core.models.organizations.OrganizationUpdateParams; +import org.opencb.opencga.core.models.user.OrganizationUserUpdateParams; +import org.opencb.opencga.core.models.user.User; +import org.opencb.opencga.core.models.user.UserStatusUpdateParams; +import org.opencb.opencga.core.response.RestResponse; + + +/* +* WARNING: AUTOGENERATED CODE +* +* This code was generated by a tool. +* +* Manual changes to this file may cause unexpected behavior in your application. +* Manual changes to this file will be overwritten if the code is regenerated. +*/ + + +/** + * This class contains methods for the Organization webservices. + * PATH: organizations + */ +public class OrganizationClient extends AbstractParentClient { + + public OrganizationClient(String token, ClientConfiguration configuration) { + super(token, configuration); + } + + /** + * Create a new organization. + * @param data JSON containing the organization to be created. + * @param params Map containing any of the following optional parameters. + * include: Fields included in the response, whole JSON path must be provided. + * exclude: Fields excluded in the response, whole JSON path must be provided. + * includeResult: Flag indicating to include the created or updated document result in the response. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse create(OrganizationCreateParams data, ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); + params.put("body", data); + return execute("organizations", null, null, null, "create", params, POST, Organization.class); + } + + /** + * Create a new note. + * @param data JSON containing the Note to be added. + * @param params Map containing any of the following optional parameters. + * include: Fields included in the response, whole JSON path must be provided. + * exclude: Fields excluded in the response, whole JSON path must be provided. + * includeResult: Flag indicating to include the created or updated document result in the response. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse createNotes(NoteCreateParams data, ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); + params.put("body", data); + return execute("organizations", null, "notes", null, "create", params, POST, Note.class); + } + + /** + * Search for notes of scope ORGANIZATION. + * @param params Map containing any of the following optional parameters. + * include: Fields included in the response, whole JSON path must be provided. + * exclude: Fields excluded in the response, whole JSON path must be provided. + * creationDate: Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + * modificationDate: Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + * id: Note unique identifier. + * scope: Scope of the Note. + * visibility: Visibility of the Note. + * uuid: Unique 32-character identifier assigned automatically by OpenCGA. + * userId: User that wrote that Note. + * tags: Note tags. + * version: Autoincremental version assigned to the registered entry. By default, updates does not create new versions. To enable + * versioning, users must set the `incVersion` flag from the /update web service when updating the document. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse searchNotes(ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); + return execute("organizations", null, "notes", null, "search", params, GET, Note.class); + } + + /** + * Delete note. + * @param id Note unique identifier. + * @param params Map containing any of the following optional parameters. + * includeResult: Flag indicating to include the created or updated document result in the response. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse deleteNotes(String id, ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); + return execute("organizations", null, "notes", id, "delete", params, DELETE, Note.class); + } + + /** + * Update a note. + * @param id Note unique identifier. + * @param data JSON containing the Note fields to be updated. + * @param params Map containing any of the following optional parameters. + * include: Fields included in the response, whole JSON path must be provided. + * exclude: Fields excluded in the response, whole JSON path must be provided. + * tagsAction: Action to be performed if the array of tags is being updated. + * includeResult: Flag indicating to include the created or updated document result in the response. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse updateNotes(String id, NoteUpdateParams data, ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); + params.put("body", data); + return execute("organizations", null, "notes", id, "update", params, POST, Note.class); + } + + /** + * Update the user status. + * @param user User ID. + * @param data JSON containing the User fields to be updated. + * @param params Map containing any of the following optional parameters. + * include: Fields included in the response, whole JSON path must be provided. + * exclude: Fields excluded in the response, whole JSON path must be provided. + * organization: Organization id. + * includeResult: Flag indicating to include the created or updated document result in the response. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse userUpdateStatus(String user, UserStatusUpdateParams data, ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); + params.put("body", data); + return execute("organizations/user", user, "status", null, "update", params, POST, User.class); + } + + /** + * Update the user information. + * @param user User ID. + * @param data JSON containing the User fields to be updated. + * @param params Map containing any of the following optional parameters. + * include: Fields included in the response, whole JSON path must be provided. + * exclude: Fields excluded in the response, whole JSON path must be provided. + * organization: Organization id. + * includeResult: Flag indicating to include the created or updated document result in the response. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse updateUser(String user, OrganizationUserUpdateParams data, ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); + params.put("body", data); + return execute("organizations", null, "user", user, "update", params, POST, User.class); + } + + /** + * Update the Organization configuration attributes. + * @param organization Organization id. + * @param data JSON containing the params to be updated. + * @param params Map containing any of the following optional parameters. + * include: Fields included in the response, whole JSON path must be provided. + * exclude: Fields excluded in the response, whole JSON path must be provided. + * includeResult: Flag indicating to include the created or updated document result in the response. + * authenticationOriginsAction: Action to be performed if the array of authenticationOrigins is being updated. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse updateConfiguration(String organization, OrganizationConfiguration data, ObjectMap + params) throws ClientException { + params = params != null ? params : new ObjectMap(); + params.put("body", data); + return execute("organizations", organization, "configuration", null, "update", params, POST, OrganizationConfiguration.class); + } + + /** + * Return the organization information. + * @param organization Organization id. + * @param params Map containing any of the following optional parameters. + * include: Fields included in the response, whole JSON path must be provided. + * exclude: Fields excluded in the response, whole JSON path must be provided. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse info(String organization, ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); + return execute("organizations", organization, null, null, "info", params, GET, Organization.class); + } + + /** + * Update some organization attributes. + * @param organization Organization id. + * @param data JSON containing the params to be updated. + * @param params Map containing any of the following optional parameters. + * include: Fields included in the response, whole JSON path must be provided. + * exclude: Fields excluded in the response, whole JSON path must be provided. + * includeResult: Flag indicating to include the created or updated document result in the response. + * adminsAction: Action to be performed if the array of admins is being updated. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse update(String organization, OrganizationUpdateParams data, ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); + params.put("body", data); + return execute("organizations", organization, null, null, "update", params, POST, Organization.class); + } +} diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ProjectClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ProjectClient.java index 8931d46d3c2..53ecaea4610 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ProjectClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ProjectClient.java @@ -16,7 +16,6 @@ package org.opencb.opencga.client.rest.clients; -import org.opencb.commons.datastore.core.FacetField; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.client.config.ClientConfiguration; import org.opencb.opencga.client.exceptions.ClientException; @@ -32,7 +31,6 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-04-25 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -41,7 +39,6 @@ /** * This class contains methods for the Project webservices. - * Client version: 2.12.4-SNAPSHOT * PATH: projects */ public class ProjectClient extends AbstractParentClient { @@ -73,8 +70,8 @@ public RestResponse create(ProjectCreateParams data, ObjectMap params) * exclude: Fields excluded in the response, whole JSON path must be provided. * limit: Number of results to be returned. * skip: Number of results to skip. - * owner: Owner of the project. - * id: Project [user@]project where project can be either the ID or the alias. + * organization: Project organization. + * id: Project [organization@]project where project can be either the ID or the alias. * name: Project name. * fqn: Project fqn. * organization: Project organization. @@ -92,34 +89,9 @@ public RestResponse search(ObjectMap params) throws ClientException { return execute("projects", null, null, null, "search", params, GET, Project.class); } - /** - * Fetch catalog project stats. - * @param projects Comma separated list of projects [user@]project up to a maximum of 100. - * @param params Map containing any of the following optional parameters. - * default: Calculate default stats. - * fileFields: List of file fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: - * studies>>biotype;type. - * individualFields: List of individual fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: - * studies>>biotype;type. - * familyFields: List of family fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: - * studies>>biotype;type. - * sampleFields: List of sample fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: - * studies>>biotype;type. - * cohortFields: List of cohort fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: - * studies>>biotype;type. - * jobFields: List of job fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: - * studies>>biotype;type. - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse aggregationStats(String projects, ObjectMap params) throws ClientException { - params = params != null ? params : new ObjectMap(); - return execute("projects", projects, null, null, "aggregationStats", params, GET, FacetField.class); - } - /** * Fetch project information. - * @param projects Comma separated list of projects [user@]project up to a maximum of 100. + * @param projects Comma separated list of projects [organization@]project up to a maximum of 100. * @param params Map containing any of the following optional parameters. * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. @@ -133,7 +105,7 @@ public RestResponse info(String projects, ObjectMap params) throws Clie /** * Increment current release number in the project. - * @param project Project [user@]project where project can be either the ID or the alias. + * @param project Project [organization@]project where project can be either the ID or the alias. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -144,7 +116,7 @@ public RestResponse incRelease(String project) throws ClientException { /** * Fetch all the studies contained in the project. - * @param project Project [user@]project where project can be either the ID or the alias. + * @param project Project [organization@]project where project can be either the ID or the alias. * @param params Map containing any of the following optional parameters. * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. @@ -160,7 +132,7 @@ public RestResponse studies(String project, ObjectMap params) throws Clie /** * Update some project attributes. - * @param project Project [user@]project where project can be either the ID or the alias. + * @param project Project [organization@]project where project can be either the ID or the alias. * @param data JSON containing the params to be updated. It will be only possible to update organism fields not previously defined. * @param params Map containing any of the following optional parameters. * include: Fields included in the response, whole JSON path must be provided. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/SampleClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/SampleClient.java index 5de5d3488bd..811dcbfe08c 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/SampleClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/SampleClient.java @@ -17,7 +17,6 @@ package org.opencb.opencga.client.rest.clients; import java.lang.Object; -import org.opencb.commons.datastore.core.FacetField; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.client.config.ClientConfiguration; import org.opencb.opencga.client.exceptions.ClientException; @@ -36,7 +35,6 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-04-25 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -45,7 +43,6 @@ /** * This class contains methods for the Sample webservices. - * Client version: 2.12.4-SNAPSHOT * PATH: samples */ public class SampleClient extends AbstractParentClient { @@ -61,7 +58,7 @@ public SampleClient(String token, ClientConfiguration configuration) { * @param data JSON containing the parameters to update the permissions. If propagate flag is set to true, it will propagate the * permissions defined to the individuals that are associated to the matching samples. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -73,41 +70,13 @@ public RestResponse updateAcl(String members, String action, return execute("samples", null, "acl", members, "update", params, POST, SampleAclEntryList.class); } - /** - * Fetch catalog sample stats. - * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. - * source: Source. - * creationYear: Creation year. - * creationMonth: Creation month (JANUARY, FEBRUARY...). - * creationDay: Creation day. - * creationDayOfWeek: Creation day of week (MONDAY, TUESDAY...). - * status: Status. - * type: Type. - * phenotypes: Phenotypes. - * release: Release. - * version: Version. - * somatic: Somatic. - * annotation: Annotation filters. Example: age>30;gender=FEMALE. For more information, please visit - * http://docs.opencb.org/display/opencga/AnnotationSets+1.4.0. - * default: Calculate default stats. - * field: List of fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: - * studies>>biotype;type;numSamples[0..10]:1. - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse aggregationStats(ObjectMap params) throws ClientException { - params = params != null ? params : new ObjectMap(); - return execute("samples", null, null, null, "aggregationStats", params, GET, FacetField.class); - } - /** * Load annotation sets from a TSV file. * @param variableSetId Variable set ID or name. * @param path Path where the TSV file is located in OpenCGA or where it should be located. * @param data JSON containing the 'content' of the TSV file if this has not yet been registered into OpenCGA. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * parents: Flag indicating whether to create parent directories if they don't exist (only when TSV file was not previously * associated). * annotationSetId: Annotation set id. If not provided, variableSetId will be used. @@ -129,7 +98,7 @@ public RestResponse loadAnnotationSets(String variableSetId, String path, T * @param params Map containing any of the following optional parameters. * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * includeResult: Flag indicating to include the created or updated document result in the response. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -144,7 +113,7 @@ public RestResponse create(SampleCreateParams data, ObjectMap params) th * Sample distinct method. * @param field Comma separated list of fields for which to obtain the distinct values. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * id: Comma separated list sample IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. * '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * uuid: Comma separated list sample UUIDs up to a maximum of 100. @@ -201,7 +170,7 @@ public RestResponse distinct(String field, ObjectMap params) throws Clie * Load samples from a ped file [EXPERIMENTAL]. * @param file file. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * variableSet: variableSet. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -222,7 +191,7 @@ public RestResponse load(String file, ObjectMap params) throws ClientExc * count: Get the total number of results matching the query. Deactivated by default. * includeIndividual: Include Individual object as an attribute. * flattenAnnotations: Flatten the annotations?. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * id: Comma separated list sample IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. * '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * uuid: Comma separated list sample UUIDs up to a maximum of 100. @@ -278,7 +247,7 @@ public RestResponse search(ObjectMap params) throws ClientException { * Returns the acl of the samples. If member is provided, it will only return the acl for the member. * @param samples Comma separated list sample IDs or UUIDs up to a maximum of 100. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * member: User or group id. * silent: Boolean to retrieve all possible entries that are queried for, false to raise an exception whenever one of the entries * looked for cannot be shown for whichever reason. @@ -298,7 +267,7 @@ public RestResponse acl(String samples, ObjectMap params) th * emptyFilesAction: Action to be performed over files that were associated only to the sample to be deleted. Possible actions * are NONE, TRASH, DELETE. * deleteEmptyCohorts: Boolean indicating if the cohorts associated only to the sample to be deleted should be also deleted. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -315,7 +284,7 @@ public RestResponse delete(String samples, ObjectMap params) throws Clie * exclude: Fields excluded in the response, whole JSON path must be provided. * includeIndividual: Include Individual object as an attribute. * flattenAnnotations: Flatten the annotations?. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * version: Comma separated list of sample versions. 'all' to get all the sample versions. Not supported if multiple sample ids * are provided. * deleted: Boolean to retrieve deleted entries. @@ -334,7 +303,7 @@ public RestResponse info(String samples, ObjectMap params) throws Client * @param params Map containing any of the following optional parameters. * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * annotationSetsAction: Action to be performed if the array of annotationSets is being updated. * phenotypesAction: Action to be performed if the array of phenotypes is being updated [SET, ADD, REMOVE]. * includeResult: Flag indicating to include the created or updated document result in the response. @@ -355,7 +324,7 @@ public RestResponse update(String samples, SampleUpdateParams data, Obje * containing the comma separated variables to be removed as a value when the action is REMOVE or a json with only the key 'reset' * containing the comma separated variables that will be set to the default value when the action is RESET. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * action: Action to be performed: ADD to add new annotations; REPLACE to replace the value of an already existing annotation; * SET to set the new list of annotations removing any possible old annotations; REMOVE to remove some annotations; RESET to * set some annotations to the default value configured in the corresponding variables of the VariableSet if any. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java index 3f598c842a3..b1a8bce6959 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java @@ -16,7 +16,6 @@ package org.opencb.opencga.client.rest.clients; -import org.opencb.commons.datastore.core.FacetField; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.client.config.ClientConfiguration; import org.opencb.opencga.client.exceptions.ClientException; @@ -24,6 +23,9 @@ import org.opencb.opencga.core.models.AclEntryList; import org.opencb.opencga.core.models.audit.AuditRecord; import org.opencb.opencga.core.models.job.Job; +import org.opencb.opencga.core.models.notes.Note; +import org.opencb.opencga.core.models.notes.NoteCreateParams; +import org.opencb.opencga.core.models.notes.NoteUpdateParams; import org.opencb.opencga.core.models.study.CustomGroup; import org.opencb.opencga.core.models.study.Group; import org.opencb.opencga.core.models.study.GroupCreateParams; @@ -45,7 +47,6 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-04-25 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -54,7 +55,6 @@ /** * This class contains methods for the Study webservices. - * Client version: 2.12.4-SNAPSHOT * PATH: studies */ public class StudyClient extends AbstractParentClient { @@ -84,7 +84,7 @@ public RestResponse updateAcl(String members, String action, * @param params Map containing any of the following optional parameters. * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. - * project: Project [user@]project where project can be either the ID or the alias. + * project: Project [organization@]project where project can be either the ID or the alias. * includeResult: Flag indicating to include the created or updated document result in the response. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -97,7 +97,7 @@ public RestResponse create(StudyCreateParams data, ObjectMap params) thro /** * Search studies. - * @param project Project [user@]project where project can be either the ID or the alias. + * @param project Project [organization@]project where project can be either the ID or the alias. * @param params Map containing any of the following optional parameters. * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. @@ -125,8 +125,8 @@ public RestResponse search(String project, ObjectMap params) throws Clien /** * Return the acl of the study. If member is provided, it will only return the acl for the member. - * @param studies Comma separated list of Studies [[user@]project:]study where study and project can be either the ID or UUID up to a - * maximum of 100. + * @param studies Comma separated list of Studies [[organization@]project:]study where study and project can be either the ID or UUID + * up to a maximum of 100. * @param params Map containing any of the following optional parameters. * member: User or group id. * silent: Boolean to retrieve all possible entries that are queried for, false to raise an exception whenever one of the entries @@ -139,35 +139,10 @@ public RestResponse acl(String studies, ObjectMap params) throws C return execute("studies", studies, null, null, "acl", params, GET, AclEntryList.class); } - /** - * Fetch catalog study stats. - * @param studies Comma separated list of studies [[user@]project:]study up to a maximum of 100. - * @param params Map containing any of the following optional parameters. - * default: Calculate default stats. - * fileFields: List of file fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: - * studies>>biotype;type. - * individualFields: List of individual fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: - * studies>>biotype;type. - * familyFields: List of family fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: - * studies>>biotype;type. - * sampleFields: List of sample fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: - * studies>>biotype;type. - * cohortFields: List of cohort fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: - * studies>>biotype;type. - * jobFields: List of job fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: - * studies>>biotype;type. - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse aggregationStats(String studies, ObjectMap params) throws ClientException { - params = params != null ? params : new ObjectMap(); - return execute("studies", studies, null, null, "aggregationStats", params, GET, FacetField.class); - } - /** * Fetch study information. - * @param studies Comma separated list of Studies [[user@]project:]study where study and project can be either the ID or UUID up to a - * maximum of 100. + * @param studies Comma separated list of Studies [[organization@]project:]study where study and project can be either the ID or UUID + * up to a maximum of 100. * @param params Map containing any of the following optional parameters. * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. @@ -206,7 +181,7 @@ public RestResponse searchAudit(String study, ObjectMap params) thr /** * Return the groups present in the study. For owners and administrators only. - * @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param params Map containing any of the following optional parameters. * id: Group id. If provided, it will only fetch information for the provided group. * silent: Boolean to retrieve all possible entries that are queried for, false to raise an exception whenever one of the entries @@ -221,7 +196,7 @@ public RestResponse groups(String study, ObjectMap params) throws C /** * Add or remove a group. - * @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param data JSON containing the parameters. * @param params Map containing any of the following optional parameters. * action: Action to be performed: ADD or REMOVE a group. @@ -236,7 +211,7 @@ public RestResponse updateGroups(String study, GroupCreateParams data, Ob /** * Add, set or remove users from an existing group. - * @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param group Group name. * @param data JSON containing the parameters. * @param params Map containing any of the following optional parameters. @@ -251,9 +226,82 @@ public RestResponse updateGroupsUsers(String study, String group, GroupUp return execute("studies", study, "groups", group, "users/update", params, POST, Group.class); } + /** + * Create a new note. + * @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @param data JSON containing the Note to be added. + * @param params Map containing any of the following optional parameters. + * include: Fields included in the response, whole JSON path must be provided. + * exclude: Fields excluded in the response, whole JSON path must be provided. + * includeResult: Flag indicating to include the created or updated document result in the response. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse createNotes(String study, NoteCreateParams data, ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); + params.put("body", data); + return execute("studies", study, "notes", null, "create", params, POST, Note.class); + } + + /** + * Search for notes of scope STUDY. + * @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @param params Map containing any of the following optional parameters. + * include: Fields included in the response, whole JSON path must be provided. + * exclude: Fields excluded in the response, whole JSON path must be provided. + * creationDate: Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + * modificationDate: Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + * id: Note unique identifier. + * uuid: Unique 32-character identifier assigned automatically by OpenCGA. + * userId: User that wrote that Note. + * tags: Note tags. + * visibility: Visibility of the Note. + * version: Autoincremental version assigned to the registered entry. By default, updates does not create new versions. To enable + * versioning, users must set the `incVersion` flag from the /update web service when updating the document. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse searchNotes(String study, ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); + return execute("studies", study, "notes", null, "search", params, GET, Note.class); + } + + /** + * Delete note. + * @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @param id Note unique identifier. + * @param params Map containing any of the following optional parameters. + * includeResult: Flag indicating to include the created or updated document result in the response. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse deleteNotes(String study, String id, ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); + return execute("studies", study, "notes", id, "delete", params, DELETE, Note.class); + } + + /** + * Update a note. + * @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @param id Note unique identifier. + * @param data JSON containing the Note fields to be updated. + * @param params Map containing any of the following optional parameters. + * include: Fields included in the response, whole JSON path must be provided. + * exclude: Fields excluded in the response, whole JSON path must be provided. + * tagsAction: Action to be performed if the array of tags is being updated. + * includeResult: Flag indicating to include the created or updated document result in the response. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse updateNotes(String study, String id, NoteUpdateParams data, ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); + params.put("body", data); + return execute("studies", study, "notes", id, "update", params, POST, Note.class); + } + /** * Fetch permission rules. - * @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param entity Entity where the permission rules should be applied to. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -266,7 +314,7 @@ public RestResponse permissionRules(String study, String entity) /** * Add or remove a permission rule. - * @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param entity Entity where the permission rules should be applied to. * @param data JSON containing the permission rule to be created or removed. * @param params Map containing any of the following optional parameters. @@ -287,13 +335,17 @@ public RestResponse updatePermissionRules(String study, String e /** * Execute template. - * @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param data Template loader parameters. * @param params Map containing any of the following optional parameters. * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobDescription: Job description. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -305,7 +357,7 @@ public RestResponse runTemplates(String study, TemplateParams data, ObjectM /** * Resource to upload a zipped template. - * @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param params Map containing any of the following optional parameters. * file: File to upload. * @return a RestResponse object. @@ -318,22 +370,19 @@ public RestResponse uploadTemplates(String study, ObjectMap params) thro /** * Delete template. - * @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param templateId Template id. - * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ - public RestResponse deleteTemplates(String study, String templateId, ObjectMap params) throws ClientException { - params = params != null ? params : new ObjectMap(); - params.putIfNotNull("study", study); + public RestResponse deleteTemplates(String study, String templateId) throws ClientException { + ObjectMap params = new ObjectMap(); return execute("studies", study, "templates", templateId, "delete", params, DELETE, Boolean.class); } /** * Update some study attributes. - * @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param data JSON containing the params to be updated. * @param params Map containing any of the following optional parameters. * include: Fields included in the response, whole JSON path must be provided. @@ -350,7 +399,7 @@ public RestResponse update(String study, StudyUpdateParams data, ObjectMa /** * Fetch variableSets from a study. - * @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param params Map containing any of the following optional parameters. * id: Id of the variableSet to be retrieved. If no id is passed, it will show all the variableSets of the study. * @return a RestResponse object. @@ -363,7 +412,7 @@ public RestResponse variableSets(String study, ObjectMap params) th /** * Add or remove a variableSet. - * @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param data JSON containing the VariableSet to be created or removed. * @param params Map containing any of the following optional parameters. * action: Action to be performed: ADD, REMOVE or FORCE_REMOVE a variableSet. @@ -379,7 +428,7 @@ public RestResponse updateVariableSets(String study, VariableSetCre /** * Add or remove variables to a VariableSet. - * @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param variableSet VariableSet id of the VariableSet to be updated. * @param data JSON containing the variable to be added or removed. For removing, only the variable id will be needed. * @param params Map containing any of the following optional parameters. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/UserClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/UserClient.java index 0ecdcb5b517..ee290b40d21 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/UserClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/UserClient.java @@ -20,13 +20,13 @@ import org.opencb.opencga.client.config.ClientConfiguration; import org.opencb.opencga.client.exceptions.ClientException; import org.opencb.opencga.client.rest.*; -import org.opencb.opencga.core.models.project.Project; import org.opencb.opencga.core.models.user.AuthenticationResponse; import org.opencb.opencga.core.models.user.ConfigUpdateParams; import org.opencb.opencga.core.models.user.FilterUpdateParams; import org.opencb.opencga.core.models.user.LoginParams; import org.opencb.opencga.core.models.user.PasswordChangeParams; import org.opencb.opencga.core.models.user.User; +import org.opencb.opencga.core.models.user.UserCreateParams; import org.opencb.opencga.core.models.user.UserFilter; import org.opencb.opencga.core.models.user.UserUpdateParams; import org.opencb.opencga.core.response.RestResponse; @@ -36,7 +36,6 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-04-25 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -45,7 +44,6 @@ /** * This class contains methods for the User webservices. - * Client version: 2.12.4-SNAPSHOT * PATH: users */ public class UserClient extends AbstractParentClient { @@ -54,6 +52,30 @@ public UserClient(String token, ClientConfiguration configuration) { super(token, configuration); } + /** + * Get an anonymous token to gain access to the system. + * @param organization Organization id. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse anonymous(String organization) throws ClientException { + ObjectMap params = new ObjectMap(); + params.putIfNotNull("organization", organization); + return execute("users", null, null, null, "anonymous", params, POST, AuthenticationResponse.class); + } + + /** + * Create a new user. + * @param data JSON containing the parameters. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse create(UserCreateParams data) throws ClientException { + ObjectMap params = new ObjectMap(); + params.put("body", data); + return execute("users", null, null, null, "create", params, POST, User.class); + } + /** * Get identified and gain access to the system. * @param data JSON containing the authentication parameters. @@ -79,12 +101,33 @@ public RestResponse password(PasswordChangeParams data) throws ClientExcep return execute("users", null, null, null, "password", params, POST, User.class); } + /** + * User search method. + * @param params Map containing any of the following optional parameters. + * include: Fields included in the response, whole JSON path must be provided. + * exclude: Fields excluded in the response, whole JSON path must be provided. + * limit: Number of results to be returned. + * skip: Number of results to skip. + * count: Get the total number of results matching the query. Deactivated by default. + * organization: Organization id. + * id: Comma separated list user IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. + * '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + * authenticationId: Authentication origin ID. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse search(ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); + return execute("users", null, null, null, "search", params, GET, User.class); + } + /** * Return the user information including its projects and studies. * @param users Comma separated list of user IDs. * @param params Map containing any of the following optional parameters. * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. + * organization: Organization id. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -175,22 +218,6 @@ public RestResponse resetPassword(String user) throws ClientException { return execute("users", user, "password", null, "reset", params, GET, User.class); } - /** - * Retrieve the projects of the user. - * @param user User ID. - * @param params Map containing any of the following optional parameters. - * include: Fields included in the response, whole JSON path must be provided. - * exclude: Fields excluded in the response, whole JSON path must be provided. - * limit: Number of results to be returned. - * skip: Number of results to skip. - * @return a RestResponse object. - * @throws ClientException ClientException if there is any server error. - */ - public RestResponse projects(String user, ObjectMap params) throws ClientException { - params = params != null ? params : new ObjectMap(); - return execute("users", user, null, null, "projects", params, GET, Project.class); - } - /** * Update some user attributes. * @param user User ID. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantClient.java index 810091d8275..6a68ae8ea82 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantClient.java @@ -62,7 +62,6 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-04-25 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -71,7 +70,6 @@ /** * This class contains methods for the Variant webservices. - * Client version: 2.12.4-SNAPSHOT * PATH: analysis/variant */ public class VariantClient extends AbstractParentClient { @@ -88,9 +86,9 @@ public VariantClient(String token, ClientConfiguration configuration) { * 2,3:100000-200000. * type: List of types, accepted values are SNV, MNV, INDEL, SV, COPY_NUMBER, COPY_NUMBER_LOSS, COPY_NUMBER_GAIN, INSERTION, * DELETION, DUPLICATION, TANDEM_DUPLICATION, BREAKEND, e.g. SNV,INDEL. - * project: Project [user@]project where project can be either the ID or the alias. + * project: Project [organization@]project where project can be either the ID or the alias. * study: Filter variants from the given studies, these can be either the numeric ID or the alias with the format - * user@project:study. + * organization@project:study. * cohort: Select variants with calculated stats for the selected cohorts. * cohortStatsRef: Reference Allele Frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL<=0.4. * cohortStatsAlt: Alternate Allele Frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL<=0.4. @@ -140,7 +138,7 @@ public RestResponse aggregationStats(ObjectMap params) throws Cli * Read variant annotations metadata from any saved versions. * @param params Map containing any of the following optional parameters. * annotationId: Annotation identifier. - * project: Project [user@]project where project can be either the ID or the alias. + * project: Project [organization@]project where project can be either the ID or the alias. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -215,11 +213,15 @@ public RestResponse infoCohortStats(String cohort, ObjectMap pa * Compute cohort variant stats for the selected list of samples. * @param data Cohort variant stats params. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -238,6 +240,10 @@ public RestResponse runCohortStats(CohortVariantStatsAnalysisParams data, O * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobDescription: Job description. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -253,12 +259,16 @@ public RestResponse runExomiser(ExomiserWrapperParams data, ObjectMap param * @param params Map containing any of the following optional parameters. * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. - * project: Project [user@]project where project can be either the ID or the alias. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * project: Project [organization@]project where project can be either the ID or the alias. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -272,7 +282,7 @@ public RestResponse runExport(VariantExportParams data, ObjectMap params) t * Calculate the possible genotypes for the members of a family. * @param modeOfInheritance Mode of inheritance. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * family: Family id. * clinicalAnalysis: Clinical analysis id. * penetrance: Penetrance. @@ -291,11 +301,15 @@ public RestResponse genotypesFamily(String modeOfInheritance, ObjectM * @param data Family QC analysis params. Family ID. Relatedness method, by default 'PLINK/IBD'. Minor allele frequence (MAF) is used * to filter variants before computing relatedness, e.g.: 1000G:CEU>0.35 or cohort:ALL>0.05. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -312,7 +326,11 @@ public RestResponse runFamilyQc(FamilyQcAnalysisParams data, ObjectMap para * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * file: Files to remove. * resume: Resume a previously failed indexation. * @return a RestResponse object. @@ -332,6 +350,10 @@ public RestResponse deleteFile(ObjectMap params) throws ClientException { * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -347,11 +369,15 @@ public RestResponse runGatk(GatkWrapperParams data, ObjectMap params) throw * the number of points to display), the general query and the list of tracks. Currently, the supported track types are: * COPY-NUMBER, INDEL, REARRANGEMENT and SNV. In addition, each track can contain a specific query. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -365,11 +391,15 @@ public RestResponse runGenomePlot(GenomePlotAnalysisParams data, ObjectMap * Run a Genome Wide Association Study between two cohorts. * @param data Gwas analysis params. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -383,11 +413,15 @@ public RestResponse runGwas(GwasAnalysisParams data, ObjectMap params) thro * Run HRDetect analysis for a given somatic sample. * @param data HRDetect analysis parameters. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -401,11 +435,15 @@ public RestResponse runHrDetect(HRDetectAnalysisParams data, ObjectMap para * [DEPRECATED] Use operation/variant/index. * @param data Variant index params. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobDescription: Job description. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -419,11 +457,15 @@ public RestResponse runIndex(VariantIndexParams data, ObjectMap params) thr * Run quality control (QC) for a given individual. It includes inferred sex and mendelian errors (UDP). * @param data Individual QC analysis params. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -437,11 +479,15 @@ public RestResponse runIndividualQc(IndividualQcAnalysisParams data, Object * Infer sex from chromosome mean coverages. * @param data Inferred sex analysis params. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -490,6 +536,10 @@ public RestResponse queryKnockoutIndividual(ObjectMap para * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -503,11 +553,15 @@ public RestResponse runKnockout(KnockoutAnalysisParams data, ObjectMap para * Run mendelian error analysis to infer uniparental disomy regions. * @param data Mendelian error analysis params. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -520,9 +574,9 @@ public RestResponse runMendelianError(MendelianErrorAnalysisParams data, Ob /** * . * @param params Map containing any of the following optional parameters. - * project: Project [user@]project where project can be either the ID or the alias. + * project: Project [organization@]project where project can be either the ID or the alias. * study: Filter variants from the given studies, these can be either the numeric ID or the alias with the format - * user@project:study. + * organization@project:study. * file: Filter variants from the files specified. This will set includeFile parameter when not provided. * sample: Filter variants by sample genotype. This will automatically set 'includeSample' parameter when not provided. This * filter accepts multiple 3 forms: 1) List of samples: Samples that contain the main variant. Accepts AND (;) and OR (,) @@ -553,7 +607,7 @@ public RestResponse metadata(ObjectMap params) throws ClientExc * Run mutational signature analysis for a given sample. Use context index. * @param params Map containing any of the following optional parameters. * study: Filter variants from the given studies, these can be either the numeric ID or the alias with the format - * user@project:study. + * organization@project:study. * sample: Sample name. * type: Variant type. Valid values: SNV, SV. * ct: List of SO consequence types, e.g. missense_variant,stop_lost or SO:0001583,SO:0001578. Accepts aliases 'loss_of_function' @@ -594,11 +648,15 @@ public RestResponse queryMutationalSignature(ObjectMap params) throws * @param data Mutational signature analysis parameters to index the genome context for that sample, and to compute both catalogue * counts and signature fitting. In order to skip one of them, , use the following keywords: , catalogue, fitting. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -617,6 +675,10 @@ public RestResponse runMutationalSignature(MutationalSignatureAnalysisParam * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -647,9 +709,9 @@ public RestResponse runPlink(PlinkWrapperParams data, ObjectMap params) thr * DELETION, DUPLICATION, TANDEM_DUPLICATION, BREAKEND, e.g. SNV,INDEL. * reference: Reference allele. * alternate: Main alternate allele. - * project: Project [user@]project where project can be either the ID or the alias. + * project: Project [organization@]project where project can be either the ID or the alias. * study: Filter variants from the given studies, these can be either the numeric ID or the alias with the format - * user@project:study. + * organization@project:study. * file: Filter variants from the files specified. This will set includeFile parameter when not provided. * filter: Specify the FILTER for any of the files. If 'file' filter is provided, will match the file and the filter. e.g.: * PASS,LowGQX. @@ -756,11 +818,15 @@ public RestResponse query(ObjectMap params) throws ClientException { * Compute a score to quantify relatedness between samples. * @param data Relatedness analysis params. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -779,6 +845,10 @@ public RestResponse runRelatedness(RelatednessAnalysisParams data, ObjectMa * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -796,9 +866,9 @@ public RestResponse runRvtests(RvtestsWrapperParams data, ObjectMap params) * 2,3:100000-200000. * type: List of types, accepted values are SNV, MNV, INDEL, SV, COPY_NUMBER, COPY_NUMBER_LOSS, COPY_NUMBER_GAIN, INSERTION, * DELETION, DUPLICATION, TANDEM_DUPLICATION, BREAKEND, e.g. SNV,INDEL. - * project: Project [user@]project where project can be either the ID or the alias. + * project: Project [organization@]project where project can be either the ID or the alias. * study: Filter variants from the given studies, these can be either the numeric ID or the alias with the format - * user@project:study. + * organization@project:study. * file: Filter variants from the files specified. This will set includeFile parameter when not provided. * filter: Specify the FILTER for any of the files. If 'file' filter is provided, will match the file and the filter. e.g.: * PASS,LowGQX. @@ -848,11 +918,15 @@ public RestResponse aggregationStatsSample(ObjectMap params) throws * Filter samples by a complex query involving metadata and variants data. * @param data . * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -869,11 +943,15 @@ public RestResponse runSampleEligibility(SampleEligibilityAnalysisParams da * skip some metrics, use the following keywords (separated by commas): variant-stats, signature, signature-catalogue, * signature-fitting, genome-plot. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -903,11 +981,15 @@ public RestResponse querySample(ObjectMap params) throws ClientExceptio * Get samples given a set of variants. * @param data Sample variant filter params. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -925,7 +1007,7 @@ public RestResponse runSample(SampleVariantFilterParams data, ObjectMap par * 2,3:100000-200000. * type: List of types, accepted values are SNV, MNV, INDEL, SV, COPY_NUMBER, COPY_NUMBER_LOSS, COPY_NUMBER_GAIN, INSERTION, * DELETION, DUPLICATION, TANDEM_DUPLICATION, BREAKEND, e.g. SNV,INDEL. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * file: Filter variants from the files specified. This will set includeFile parameter when not provided. * filter: Specify the FILTER for any of the files. If 'file' filter is provided, will match the file and the filter. e.g.: * PASS,LowGQX. @@ -941,7 +1023,7 @@ public RestResponse runSample(SampleVariantFilterParams data, ObjectMap par * clinical: Clinical source: clinvar, cosmic. * clinicalSignificance: Clinical significance: benign, likely_benign, likely_pathogenic, pathogenic. * clinicalConfirmedStatus: Clinical confirmed status. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * filterTranscript: Do filter transcripts when obtaining transcript counts. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -957,11 +1039,15 @@ public RestResponse querySampleStats(String sample, ObjectMa * @param data Sample variant stats params. Use index=true and indexId='' to store the result in catalog sample QC. indexId=ALL * requires an empty query. Use sample=all to compute sample stats of all samples in the variant storage. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -975,12 +1061,16 @@ public RestResponse runSampleStats(SampleVariantStatsAnalysisParams data, O * Export calculated variant stats and frequencies. * @param data Variant stats export params. * @param params Map containing any of the following optional parameters. - * project: Project [user@]project where project can be either the ID or the alias. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * project: Project [organization@]project where project can be either the ID or the alias. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -994,11 +1084,15 @@ public RestResponse runStatsExport(VariantStatsExportParams data, ObjectMap * Compute variant stats for any cohort and any set of variants. * @param data Variant stats params. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java index 14a89e79d08..97c999d192d 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java @@ -43,6 +43,8 @@ import org.opencb.opencga.core.models.operations.variant.VariantStorageMetadataRepairToolParams; import org.opencb.opencga.core.models.operations.variant.VariantStorageMetadataSynchronizeParams; import org.opencb.opencga.core.models.operations.variant.VariantStudyDeleteParams; +import org.opencb.opencga.core.models.study.VariantSetupResult; +import org.opencb.opencga.core.models.variant.VariantSetupParams; import org.opencb.opencga.core.response.RestResponse; @@ -50,7 +52,6 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2024-04-25 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -59,7 +60,6 @@ /** * This class contains methods for the VariantOperation webservices. - * Client version: 2.12.4-SNAPSHOT * PATH: operation */ public class VariantOperationClient extends AbstractParentClient { @@ -72,7 +72,7 @@ public VariantOperationClient(String token, ClientConfiguration configuration) { * Update Cellbase configuration. * @param data New cellbase configuration. * @param params Map containing any of the following optional parameters. - * project: Project [user@]project where project can be either the ID or the alias. + * project: Project [organization@]project where project can be either the ID or the alias. * annotationUpdate: Create and load variant annotations into the database. * annotationSaveId: Save a copy of the current variant annotation at the database. * @return a RestResponse object. @@ -92,7 +92,11 @@ public RestResponse configureCellbase(CellBaseConfiguration data, ObjectMap * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -109,7 +113,11 @@ public RestResponse aggregateVariant(VariantAggregateParams data, ObjectMap * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. - * project: Project [user@]project where project can be either the ID or the alias. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * project: Project [organization@]project where project can be either the ID or the alias. * annotationId: Annotation identifier. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -127,8 +135,12 @@ public RestResponse deleteVariantAnnotation(ObjectMap params) throws Client * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. - * project: Project [user@]project where project can be either the ID or the alias. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * project: Project [organization@]project where project can be either the ID or the alias. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -146,7 +158,11 @@ public RestResponse indexVariantAnnotation(VariantAnnotationIndexParams dat * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. - * project: Project [user@]project where project can be either the ID or the alias. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * project: Project [organization@]project where project can be either the ID or the alias. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -160,8 +176,8 @@ public RestResponse saveVariantAnnotation(VariantAnnotationSaveParams data, * Update Variant Storage Engine configuration. Can be updated at Project or Study level. * @param data Configuration params to update. * @param params Map containing any of the following optional parameters. - * project: Project [user@]project where project can be either the ID or the alias. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * project: Project [organization@]project where project can be either the ID or the alias. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -179,7 +195,11 @@ public RestResponse configureVariant(VariantConfigureParams data, Obj * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -197,7 +217,11 @@ public RestResponse deleteVariant(VariantFileDeleteParams data, ObjectMap p * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -215,7 +239,11 @@ public RestResponse aggregateVariantFamily(VariantAggregateFamilyParams dat * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -233,7 +261,11 @@ public RestResponse indexVariantFamily(VariantFamilyIndexParams data, Objec * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -251,7 +283,11 @@ public RestResponse indexVariant(VariantIndexParams data, ObjectMap params) * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -269,6 +305,10 @@ public RestResponse launcherVariantIndex(VariantFileIndexJobLauncherParams * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * project: project. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -287,6 +327,10 @@ public RestResponse runVariantJulie(JulieParams data, ObjectMap params) thr * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -304,7 +348,11 @@ public RestResponse repairVariantMetadata(VariantStorageMetadataRepairToolP * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -323,6 +371,10 @@ public RestResponse synchronizeVariantMetadata(VariantStorageMetadataSynchr * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -340,7 +392,11 @@ public RestResponse pruneVariant(VariantPruneParams data, ObjectMap params) * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -358,7 +414,11 @@ public RestResponse deleteVariantSample(VariantSampleDeleteParams data, Obj * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -372,7 +432,7 @@ public RestResponse indexVariantSample(VariantSecondarySampleIndexParams da * DEPRECATED You should use the new sample index configure method. * @param data New SampleIndexConfiguration. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * skipRebuild: Skip sample index re-build. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -390,7 +450,11 @@ public RestResponse variantSampleIndexConfigure(SampleIndexConfiguration da * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * name: Unique name of the score within the study. * resume: Resume a previously failed remove. * force: Force remove of partially indexed scores. @@ -417,7 +481,11 @@ public RestResponse deleteVariantScore(ObjectMap params) throws ClientExcep * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -435,8 +503,12 @@ public RestResponse indexVariantScore(VariantScoreIndexParams data, ObjectM * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. - * project: Project [user@]project where project can be either the ID or the alias. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * project: Project [organization@]project where project can be either the ID or the alias. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -455,7 +527,11 @@ public RestResponse variantSecondaryAnnotationIndex(VariantSecondaryAnnotat * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -469,7 +545,7 @@ public RestResponse variantSecondarySampleIndex(VariantSecondarySampleIndex * Update SampleIndex configuration (New!). * @param data New SampleIndexConfiguration. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * skipRebuild: Skip sample index re-build. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -488,8 +564,12 @@ public RestResponse configureVariantSecondarySampleIndex(SampleIndexConfigu * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. - * project: Project [user@]project where project can be either the ID or the alias. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * project: Project [organization@]project where project can be either the ID or the alias. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -506,7 +586,11 @@ public RestResponse secondaryIndexVariant(VariantSecondaryAnnotationIndexPa * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * samples: Samples to remove. Needs to provide all the samples in the secondary index. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -516,15 +600,33 @@ public RestResponse deleteVariantSecondaryIndex(ObjectMap params) throws Cl return execute("operation", null, "variant/secondaryIndex", null, "delete", params, DELETE, Job.class); } + /** + * Execute Variant Setup to allow using the variant engine. This setup is necessary before starting any variant operation. + * @param data Variant setup params. + * @param params Map containing any of the following optional parameters. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse setupVariant(VariantSetupParams data, ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); + params.put("body", data); + return execute("operation", null, "variant", null, "setup", params, POST, VariantSetupResult.class); + } + /** * Deletes the VariantStats of a cohort/s from the database. * @param data Variant stats delete params. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -538,11 +640,15 @@ public RestResponse deleteVariantStats(VariantStatsDeleteParams data, Objec * Compute variant stats for any cohort and any set of variants and index the result in the variant storage database. * @param data Variant stats params. * @param params Map containing any of the following optional parameters. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -560,7 +666,11 @@ public RestResponse indexVariantStats(VariantStatsIndexParams data, ObjectM * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. - * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ diff --git a/opencga-client/src/main/javascript/Admin.js b/opencga-client/src/main/javascript/Admin.js index 5772c11289c..3f6204d4165 100644 --- a/opencga-client/src/main/javascript/Admin.js +++ b/opencga-client/src/main/javascript/Admin.js @@ -1,9 +1,12 @@ /** - * Copyright 2015-2020 OpenCB + * Copyright 2015-2024 OpenCB + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -12,7 +15,6 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-04-25 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -34,8 +36,8 @@ export default class Admin extends OpenCGAParentClass { /** Group by operation * @param {String} fields - Comma separated list of fields by which to group by. - * @param {"AUDIT USER PROJECT STUDY FILE SAMPLE JOB INDIVIDUAL COHORT DISEASE_PANEL FAMILY CLINICAL_ANALYSIS INTERPRETATION VARIANT - * ALIGNMENT CLINICAL EXPRESSION RGA FUNCTIONAL"} entity - Entity to be grouped by. + * @param {"AUDIT NOTE ORGANIZATION USER PROJECT STUDY FILE SAMPLE JOB INDIVIDUAL COHORT DISEASE_PANEL FAMILY CLINICAL_ANALYSIS + * INTERPRETATION VARIANT ALIGNMENT CLINICAL EXPRESSION RGA FUNCTIONAL"} entity - Entity to be grouped by. * @param {Object} [params] - The Object containing the following optional parameters: * @param {Boolean} [params.count] - Count the number of elements matching the group. * @param {Number} [params.limit = "50"] - Maximum number of documents (groups) to be returned. The default value is 50. @@ -49,16 +51,6 @@ export default class Admin extends OpenCGAParentClass { return this._get("admin", null, "audit", null, "groupBy", {fields, entity, ...params}); } - /** Sync Catalog into the Solr - * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.collection] - Collection to be indexed (file, sample, individual, family, cohort and/or job). If not provided, - * all of them will be indexed. - * @returns {Promise} Promise object in the form of RestResponse instance. - */ - indexStatsCatalog(params) { - return this._post("admin", null, "catalog", null, "indexStats", params); - } - /** Install OpenCGA database * @param {Object} data - JSON containing the mandatory parameters. * @returns {Promise} Promise object in the form of RestResponse instance. @@ -69,10 +61,12 @@ export default class Admin extends OpenCGAParentClass { /** Change JWT secret key * @param {Object} data - JSON containing the parameters. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.organization] - Organization id. * @returns {Promise} Promise object in the form of RestResponse instance. */ - jwtCatalog(data) { - return this._post("admin", null, "catalog", null, "jwt", data); + jwtCatalog(data, params) { + return this._post("admin", null, "catalog", null, "jwt", data, params); } /** Create a new user @@ -85,10 +79,24 @@ export default class Admin extends OpenCGAParentClass { /** Import users or a group of users from LDAP or AAD * @param {Object} data - JSON containing the parameters. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.organization] - Organization id. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + importUsers(data, params) { + return this._post("admin", null, "users", null, "import", data, params); + } + + /** User permissions + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.entryIds] - Comma separated list of entry ids. + * @param {String} [params.permissions] - Comma separated list of permissions to be retrieved. + * @param {String} [params.category] - Category corresponding to the id's provided. * @returns {Promise} Promise object in the form of RestResponse instance. */ - importUsers(data) { - return this._post("admin", null, "users", null, "import", data); + permissionsUsers(params) { + return this._get("admin", null, "users", null, "permissions", params); } /** User search method @@ -99,8 +107,8 @@ export default class Admin extends OpenCGAParentClass { * @param {Number} [params.skip] - Number of results to skip. * @param {Boolean} [params.count = "false"] - Get the total number of results matching the query. Deactivated by default. The default * value is false. + * @param {String} [params.organization] - Organization id. * @param {String} [params.user] - User ID. - * @param {String} [params.account] - Account type [GUEST, FULL, ADMINISTRATOR]. * @param {String} [params.authenticationId] - Authentication origin ID. * @returns {Promise} Promise object in the form of RestResponse instance. */ @@ -110,16 +118,19 @@ export default class Admin extends OpenCGAParentClass { /** Synchronise a group of users from an authentication origin with a group in a study from catalog * @param {Object} data - JSON containing the parameters. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.organization] - Organization id. * @returns {Promise} Promise object in the form of RestResponse instance. */ - syncUsers(data) { - return this._post("admin", null, "users", null, "sync", data); + syncUsers(data, params) { + return this._post("admin", null, "users", null, "sync", data, params); } /** Add or remove users from existing groups * @param {String} user - User ID. * @param {Object} data - JSON containing the parameters. * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.organization] - Organization id. * @param {"ADD REMOVE"} [params.action = "ADD"] - Action to be performed: ADD or REMOVE user to/from groups. The default value is ADD. * @returns {Promise} Promise object in the form of RestResponse instance. */ diff --git a/opencga-client/src/main/javascript/Alignment.js b/opencga-client/src/main/javascript/Alignment.js index c6e95fac4a9..fc302177872 100644 --- a/opencga-client/src/main/javascript/Alignment.js +++ b/opencga-client/src/main/javascript/Alignment.js @@ -1,9 +1,12 @@ /** - * Copyright 2015-2020 OpenCB + * Copyright 2015-2024 OpenCB + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -12,7 +15,6 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-04-25 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -41,6 +43,10 @@ export default class Alignment extends OpenCGAParentClass { * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runBwa(data, params) { @@ -56,6 +62,10 @@ export default class Alignment extends OpenCGAParentClass { * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runCoverageIndex(data, params) { @@ -71,6 +81,10 @@ export default class Alignment extends OpenCGAParentClass { * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ coverageQcGeneCoverageStatsRun(data, params) { @@ -80,7 +94,7 @@ export default class Alignment extends OpenCGAParentClass { /** Query the coverage of an alignment file for regions or genes * @param {String} file - File ID. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.region] - Comma separated list of regions 'chr:start-end, e.g.: 2,3:63500-65000. * @param {String} [params.gene] - Comma separated list of genes, e.g.: BCRA2,TP53. * @param {Number} [params.offset] - Offset to extend the region, gene or exon at up and downstream. @@ -100,7 +114,7 @@ export default class Alignment extends OpenCGAParentClass { * @param {String} file1 - Input file #1 (e.g. somatic file). * @param {String} file2 - Input file #2 (e.g. germline file). * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Boolean} [params.skipLog2] - Do not apply Log2 to normalise the coverage ratio. * @param {String} [params.region] - Comma separated list of regions 'chr:start-end, e.g.: 2,3:63500-65000. * @param {String} [params.gene] - Comma separated list of genes, e.g.: BCRA2,TP53. @@ -118,7 +132,7 @@ export default class Alignment extends OpenCGAParentClass { * @param {String} file - File ID. * @param {String} gene - Comma separated list of genes, e.g.: BCRA2,TP53. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Number} [params.threshold] - Only regions whose coverage depth is under this threshold will be reported. * @returns {Promise} Promise object in the form of RestResponse instance. */ @@ -136,6 +150,10 @@ export default class Alignment extends OpenCGAParentClass { * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runDeeptools(data, params) { @@ -151,6 +169,10 @@ export default class Alignment extends OpenCGAParentClass { * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runFastqc(data, params) { @@ -166,6 +188,10 @@ export default class Alignment extends OpenCGAParentClass { * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runIndex(data, params) { @@ -182,6 +208,10 @@ export default class Alignment extends OpenCGAParentClass { * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runPicard(data, params) { @@ -198,6 +228,10 @@ export default class Alignment extends OpenCGAParentClass { * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runQc(data, params) { @@ -211,7 +245,7 @@ export default class Alignment extends OpenCGAParentClass { * @param {Number} [params.skip] - Number of results to skip. * @param {Boolean} [params.count = "false"] - Get the total number of results matching the query. Deactivated by default. The default * value is false. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.region] - Comma separated list of regions 'chr:start-end, e.g.: 2,3:63500-65000. * @param {String} [params.gene] - Comma separated list of genes, e.g.: BCRA2,TP53. * @param {Number} [params.offset] - Offset to extend the region, gene or exon at up and downstream. @@ -244,6 +278,10 @@ export default class Alignment extends OpenCGAParentClass { * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runSamtools(data, params) { diff --git a/opencga-client/src/main/javascript/ClinicalAnalysis.js b/opencga-client/src/main/javascript/ClinicalAnalysis.js index 78c02c0920b..931d5fe01ce 100644 --- a/opencga-client/src/main/javascript/ClinicalAnalysis.js +++ b/opencga-client/src/main/javascript/ClinicalAnalysis.js @@ -1,9 +1,12 @@ /** - * Copyright 2015-2020 OpenCB + * Copyright 2015-2024 OpenCB + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -12,7 +15,6 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-04-25 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -37,7 +39,7 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * @param {Object} data - JSON containing the parameters to add ACLs. * @param {"SET ADD REMOVE RESET"} action = "ADD" - Action to be performed [ADD, SET, REMOVE or RESET]. The default value is ADD. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Boolean} [params.propagate = "false"] - Propagate permissions to related families, individuals, samples and files. The default * value is false. * @returns {Promise} Promise object in the form of RestResponse instance. @@ -51,7 +53,7 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * @param {String} variableSetId - Variable set ID or name. * @param {String} path - Path where the TSV file is located in OpenCGA or where it should be located. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Boolean} [params.parents] - Flag indicating whether to create parent directories if they don't exist (only when TSV file was * not previously associated). * @param {String} [params.annotationSetId] - Annotation set id. If not provided, variableSetId will be used. @@ -64,7 +66,7 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { /** Update Clinical Analysis configuration. * @param {Object} [data] - Configuration params to update. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ updateClinicalConfiguration(data, params) { @@ -76,7 +78,7 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Boolean} [params.skipCreateDefaultInterpretation] - Flag to skip creating and initialise an empty default primary * interpretation (Id will be '{clinicalAnalysisId}.1'). This flag is only considered if no Interpretation object is passed. * @param {Boolean} [params.includeResult = "false"] - Flag indicating to include the created or updated document result in the response. @@ -90,7 +92,7 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { /** Clinical Analysis distinct method * @param {String} field - Comma separated list of fields for which to obtain the distinct values. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.id] - Comma separated list of Clinical Analysis IDs up to a maximum of 100. Also admits basic regular * expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * @param {String} [params.uuid] - Comma separated list of Clinical Analysis UUIDs up to a maximum of 100. @@ -116,6 +118,7 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * @param {String} [params.dueDate] - Clinical Analysis due date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. * @param {String} [params.qualityControlSummary] - Clinical Analysis quality control summary. * @param {String} [params.release] - Release when it was created. + * @param {Number} [params.snapshot] - Snapshot value (Latest version of the entry in the specified release). * @param {String} [params.status] - Filter by status. * @param {String} [params.internalStatus] - Filter by internal status. * @param {String} [params.annotation] - Annotation filters. Example: age>30;gender=FEMALE. For more information, please visit @@ -130,10 +133,11 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { /** Interpretation distinct method * @param {String} field - Comma separated list of fields for which to obtain the distinct values. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.id] - Comma separated list of Interpretation IDs up to a maximum of 100. Also admits basic regular expressions * using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * @param {String} [params.uuid] - Comma separated list of Interpretation UUIDs up to a maximum of 100. + * @param {String} [params.name] - Comma separated list of Interpretation names up to a maximum of 100. * @param {String} [params.clinicalAnalysisId] - Clinical Analysis id. * @param {String} [params.analystId] - Analyst ID. * @param {String} [params.methodName] - Interpretation method name. Also admits basic regular expressions using the operator '~', i.e. @@ -160,10 +164,11 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * @param {Number} [params.limit] - Number of results to be returned. * @param {Number} [params.skip] - Number of results to skip. * @param {Boolean} [params.sort] - Sort the results. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.id] - Comma separated list of Interpretation IDs up to a maximum of 100. Also admits basic regular expressions * using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * @param {String} [params.uuid] - Comma separated list of Interpretation UUIDs up to a maximum of 100. + * @param {String} [params.name] - Comma separated list of Interpretation names up to a maximum of 100. * @param {String} [params.clinicalAnalysisId] - Clinical Analysis id. * @param {String} [params.analystId] - Analyst ID. * @param {String} [params.methodName] - Interpretation method name. Also admits basic regular expressions using the operator '~', i.e. @@ -188,7 +193,7 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.version] - Comma separated list of interpretation versions. 'all' to get all the interpretation versions. Not * supported if multiple interpretation ids are provided. * @param {Boolean} [params.deleted = "false"] - Boolean to retrieve deleted entries. The default value is false. @@ -201,12 +206,16 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { /** Run cancer tiering interpretation analysis * @param {Object} data - Cancer tiering interpretation analysis params. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not * provided. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runInterpreterCancerTiering(data, params) { @@ -214,14 +223,18 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { } /** Run exomiser interpretation analysis - * @param {Object} data - Exomizer interpretation analysis params. + * @param {Object} data - Exomiser interpretation analysis params. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not * provided. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runInterpreterExomiser(data, params) { @@ -231,12 +244,16 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { /** Run TEAM interpretation analysis * @param {Object} data - TEAM interpretation analysis params. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not * provided. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runInterpreterTeam(data, params) { @@ -246,12 +263,16 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { /** Run tiering interpretation analysis * @param {Object} data - Tiering interpretation analysis params. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not * provided. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runInterpreterTiering(data, params) { @@ -261,18 +282,41 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { /** Run Zetta interpretation analysis * @param {Object} data - Zetta interpretation analysis params. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not * provided. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runInterpreterZetta(data, params) { return this._post("analysis", null, "clinical/interpreter/zetta", null, "run", data, params); } + /** Load clinical analyses from a file + * @param {Object} data - Parameters to load clinical analysis in OpenCGA catalog from a file. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not + * provided. + * @param {String} [params.jobDescription] - Job description. + * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. + * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + load(data, params) { + return this._post("analysis", null, "clinical", null, "load", data, params); + } + /** RGA aggregation stats * @param {String} field - List of fields separated by semicolons, e.g.: clinicalSignificances;type. For nested fields use >>, e.g.: * type>>clinicalSignificances;knockoutType. @@ -299,7 +343,7 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * @param {String} [params.clinicalSignificance] - Filter by clinical significance. * @param {String} [params.populationFrequency] - Filter by population frequency. * @param {String} [params.consequenceType] - Filter by consequence type. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ aggregationStatsRga(field, params) { @@ -336,7 +380,7 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * @param {String} [params.clinicalSignificance] - Filter by clinical significance. * @param {String} [params.populationFrequency] - Filter by population frequency. * @param {String} [params.consequenceType] - Filter by consequence type. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ queryRgaGene(params) { @@ -368,7 +412,7 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * @param {String} [params.clinicalSignificance] - Filter by clinical significance. * @param {String} [params.populationFrequency] - Filter by population frequency. * @param {String} [params.consequenceType] - Filter by consequence type. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ summaryRgaGene(params) { @@ -378,12 +422,16 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { /** Generate Recessive Gene Analysis secondary index * @param {Object} data - Recessive Gene Analysis index params. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not * provided. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @param {Boolean} [params.auxiliarIndex = "false"] - Index auxiliar collection to improve performance assuming RGA is completely * indexed. The default value is false. * @returns {Promise} Promise object in the form of RestResponse instance. @@ -419,7 +467,7 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * @param {String} [params.clinicalSignificance] - Filter by clinical significance. * @param {String} [params.populationFrequency] - Filter by population frequency. * @param {String} [params.consequenceType] - Filter by consequence type. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ queryRgaIndividual(params) { @@ -451,7 +499,7 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * @param {String} [params.clinicalSignificance] - Filter by clinical significance. * @param {String} [params.populationFrequency] - Filter by population frequency. * @param {String} [params.consequenceType] - Filter by consequence type. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ summaryRgaIndividual(params) { @@ -488,7 +536,7 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * @param {String} [params.clinicalSignificance] - Filter by clinical significance. * @param {String} [params.populationFrequency] - Filter by population frequency. * @param {String} [params.consequenceType] - Filter by consequence type. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ queryRgaVariant(params) { @@ -520,7 +568,7 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * @param {String} [params.clinicalSignificance] - Filter by clinical significance. * @param {String} [params.populationFrequency] - Filter by population frequency. * @param {String} [params.consequenceType] - Filter by consequence type. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ summaryRgaVariant(params) { @@ -536,7 +584,7 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * @param {Boolean} [params.count = "false"] - Get the total number of results matching the query. Deactivated by default. The default * value is false. * @param {Boolean} [params.flattenAnnotations = "false"] - Flatten the annotations?. The default value is false. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.id] - Comma separated list of Clinical Analysis IDs up to a maximum of 100. Also admits basic regular * expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * @param {String} [params.uuid] - Comma separated list of Clinical Analysis UUIDs up to a maximum of 100. @@ -562,6 +610,7 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * @param {String} [params.dueDate] - Clinical Analysis due date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. * @param {String} [params.qualityControlSummary] - Clinical Analysis quality control summary. * @param {String} [params.release] - Release when it was created. + * @param {Number} [params.snapshot] - Snapshot value (Latest version of the entry in the specified release). * @param {String} [params.status] - Filter by status. * @param {String} [params.internalStatus] - Filter by internal status. * @param {String} [params.annotation] - Annotation filters. Example: age>30;gender=FEMALE. For more information, please visit @@ -591,7 +640,7 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * @param {String} [params.type] - List of types, accepted values are SNV, MNV, INDEL, SV, COPY_NUMBER, COPY_NUMBER_LOSS, * COPY_NUMBER_GAIN, INSERTION, DELETION, DUPLICATION, TANDEM_DUPLICATION, BREAKEND, e.g. SNV,INDEL. * @param {String} [params.study] - Filter variants from the given studies, these can be either the numeric ID or the alias with the - * format user@project:study. + * format organization@project:study. * @param {String} [params.file] - Filter variants from the files specified. This will set includeFile parameter when not provided. * @param {String} [params.filter] - Specify the FILTER for any of the files. If 'file' filter is provided, will match the file and the * filter. e.g.: PASS,LowGQX. @@ -684,7 +733,7 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { /** Returns the acl of the clinical analyses. If member is provided, it will only return the acl for the member. * @param {String} clinicalAnalyses - Comma separated list of clinical analysis IDs or names up to a maximum of 100. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.member] - User or group ID. * @param {Boolean} [params.silent = "false"] - Boolean to retrieve all possible entries that are queried for, false to raise an * exception whenever one of the entries looked for cannot be shown for whichever reason. The default value is false. @@ -697,7 +746,7 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { /** Delete clinical analyses * @param {String} clinicalAnalyses - Comma separated list of clinical analysis IDs or names up to a maximum of 100. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Boolean} [params.force = "false"] - Force deletion if the ClinicalAnalysis contains interpretations or is locked. The default * value is false. * @returns {Promise} Promise object in the form of RestResponse instance. @@ -712,7 +761,7 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {"ADD REMOVE REPLACE"} [params.commentsAction = "ADD"] - Action to be performed if the array of comments is being updated. The * default value is ADD. * @param {"ADD SET REMOVE"} [params.flagsAction = "ADD"] - Action to be performed if the array of flags is being updated. The default @@ -740,7 +789,7 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * 'remove' containing the comma separated variables to be removed as a value when the action is REMOVE or a json with only the key * 'reset' containing the comma separated variables that will be set to the default value when the action is RESET. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {"ADD SET REMOVE RESET REPLACE"} [params.action = "ADD"] - Action to be performed: ADD to add new annotations; REPLACE to * replace the value of an already existing annotation; SET to set the new list of annotations removing any possible old annotations; * REMOVE to remove some annotations; RESET to set some annotations to the default value configured in the corresponding variables of the @@ -757,7 +806,9 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. * @param {Boolean} [params.flattenAnnotations = "false"] - Flatten the annotations?. The default value is false. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.version] - Comma separated list of clinical versions. 'all' to get all the clinical versions. Not supported if + * multiple clinical ids are provided. * @param {Boolean} [params.deleted = "false"] - Boolean to retrieve deleted entries. The default value is false. * @returns {Promise} Promise object in the form of RestResponse instance. */ @@ -771,7 +822,7 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. - * @param {String} [params.study] - [[user@]project:]study id. + * @param {String} [params.study] - [[organization@]project:]study id. * @param {"PRIMARY SECONDARY"} [params.setAs = "SECONDARY"] - Set interpretation as. The default value is SECONDARY. * @param {Boolean} [params.includeResult = "false"] - Flag indicating to include the created or updated document result in the response. * The default value is false. @@ -785,18 +836,18 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * @param {String} interpretations - Interpretation IDs of the Clinical Analysis. * @param {String} clinicalAnalysis - Clinical analysis ID. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - [[user@]project:]study ID. + * @param {String} [params.study] - [[organization@]project:]study ID. * @returns {Promise} Promise object in the form of RestResponse instance. */ clearInterpretation(clinicalAnalysis, interpretations, params) { - return this._post("analysis/clinical", clinicalAnalysis, "interpretation", interpretations, "clear", params); + return this._post("analysis/clinical", clinicalAnalysis, "interpretation", interpretations, "clear", null, params); } /** Delete interpretation * @param {String} clinicalAnalysis - Clinical analysis ID. * @param {String} interpretations - Interpretation IDs of the Clinical Analysis. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - [[user@]project:]study ID. + * @param {String} [params.study] - [[organization@]project:]study ID. * @param {String} [params.setAsPrimary] - Interpretation id to set as primary from the list of secondaries in case of deleting the * actual primary one. * @returns {Promise} Promise object in the form of RestResponse instance. @@ -810,11 +861,11 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * @param {String} interpretation - Interpretation ID. * @param {Number} version - Version to revert to. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - [[user@]project:]study ID. + * @param {String} [params.study] - [[organization@]project:]study ID. * @returns {Promise} Promise object in the form of RestResponse instance. */ revertInterpretation(clinicalAnalysis, interpretation, version, params) { - return this._post("analysis/clinical", clinicalAnalysis, "interpretation", interpretation, "revert", {version, ...params}); + return this._post("analysis/clinical", clinicalAnalysis, "interpretation", interpretation, "revert", null, {version, ...params}); } /** Update interpretation fields @@ -824,7 +875,7 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. - * @param {String} [params.study] - [[user@]project:]study ID. + * @param {String} [params.study] - [[organization@]project:]study ID. * @param {"ADD SET REMOVE REPLACE"} [params.primaryFindingsAction = "ADD"] - Action to be performed if the array of primary findings is * being updated. The default value is ADD. * @param {"ADD SET REMOVE"} [params.methodsAction = "ADD"] - Action to be performed if the array of methods is being updated. The @@ -850,7 +901,7 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {"ADD REMOVE REPLACE"} [params.commentsAction = "ADD"] - Action to be performed if the array of comments is being updated. The * default value is ADD. * @param {"ADD SET REMOVE"} [params.supportingEvidencesAction = "ADD"] - Action to be performed if the array of supporting evidences is diff --git a/opencga-client/src/main/javascript/Cohort.js b/opencga-client/src/main/javascript/Cohort.js index bc33d1c6e26..dcd44092cc6 100644 --- a/opencga-client/src/main/javascript/Cohort.js +++ b/opencga-client/src/main/javascript/Cohort.js @@ -1,9 +1,12 @@ /** - * Copyright 2015-2020 OpenCB + * Copyright 2015-2024 OpenCB + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -12,7 +15,6 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-04-25 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -37,41 +39,19 @@ export default class Cohort extends OpenCGAParentClass { * @param {Object} data - JSON containing the parameters to add ACLs. * @param {"SET ADD REMOVE RESET"} action = "ADD" - Action to be performed [ADD, SET, REMOVE or RESET]. The default value is ADD. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ updateAcl(members, action, data, params) { return this._post("cohorts", null, "acl", members, "update", data, {action, ...params}); } - /** Fetch catalog cohort stats - * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. - * @param {String} [params.type] - Type. - * @param {String} [params.creationYear] - Creation year. - * @param {String} [params.creationMonth] - Creation month (JANUARY, FEBRUARY...). - * @param {String} [params.creationDay] - Creation day. - * @param {String} [params.creationDayOfWeek] - Creation day of week (MONDAY, TUESDAY...). - * @param {String} [params.numSamples] - Number of samples. - * @param {String} [params.status] - Status. - * @param {String} [params.release] - Release. - * @param {String} [params.annotation] - Annotation filters. Example: age>30;gender=FEMALE. For more information, please visit - * http://docs.opencb.org/display/opencga/AnnotationSets+1.4.0. - * @param {Boolean} [params.default = "false"] - Calculate default stats. The default value is false. - * @param {String} [params.field] - List of fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: - * studies>>biotype;type;numSamples[0..10]:1. - * @returns {Promise} Promise object in the form of RestResponse instance. - */ - aggregationStats(params) { - return this._get("cohorts", null, null, null, "aggregationStats", params); - } - /** Load annotation sets from a TSV file * @param {Object} [data] - JSON containing the 'content' of the TSV file if this has not yet been registered into OpenCGA. * @param {String} variableSetId - Variable set ID or name. * @param {String} path - Path where the TSV file is located in OpenCGA or where it should be located. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Boolean} [params.parents] - Flag indicating whether to create parent directories if they don't exist (only when TSV file was * not previously associated). * @param {String} [params.annotationSetId] - Annotation set id. If not provided, variableSetId will be used. @@ -86,7 +66,7 @@ export default class Cohort extends OpenCGAParentClass { * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.variableSet] - Deprecated: Use /generate web service and filter by annotation. * @param {String} [params.variable] - Deprecated: Use /generate web service and filter by annotation. * @param {Boolean} [params.includeResult = "false"] - Flag indicating to include the created or updated document result in the response. @@ -100,7 +80,7 @@ export default class Cohort extends OpenCGAParentClass { /** Cohort distinct method * @param {String} field - Comma separated list of fields for which to obtain the distinct values. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.id] - Comma separated list of cohort IDs up to a maximum of 100. Also admits basic regular expressions using * the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * @param {String} [params.name] - Comma separated list of cohort names up to a maximum of 100. Also admits basic regular expressions @@ -128,7 +108,7 @@ export default class Cohort extends OpenCGAParentClass { * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.id] - Comma separated list sample IDs or UUIDs up to a maximum of 100. * @param {Boolean} [params.somatic] - Somatic sample. * @param {String} [params.individualId] - Individual ID or UUID. @@ -163,7 +143,7 @@ export default class Cohort extends OpenCGAParentClass { * @param {Boolean} [params.count = "false"] - Get the total number of results matching the query. Deactivated by default. The default * value is false. * @param {Boolean} [params.flattenAnnotations = "false"] - Flatten the annotations?. The default value is false. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.id] - Comma separated list of cohort IDs up to a maximum of 100. Also admits basic regular expressions using * the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * @param {String} [params.name] - Comma separated list of cohort names up to a maximum of 100. Also admits basic regular expressions @@ -189,7 +169,7 @@ export default class Cohort extends OpenCGAParentClass { /** Return the acl of the cohort. If member is provided, it will only return the acl for the member. * @param {String} cohorts - Comma separated list of cohort IDs or UUIDs up to a maximum of 100. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.member] - User or group id. * @param {Boolean} [params.silent = "false"] - Boolean to retrieve all possible entries that are queried for, false to raise an * exception whenever one of the entries looked for cannot be shown for whichever reason. The default value is false. @@ -202,7 +182,7 @@ export default class Cohort extends OpenCGAParentClass { /** Delete cohorts * @param {String} cohorts - Comma separated list of cohort ids. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ delete(cohorts, params) { @@ -215,7 +195,7 @@ export default class Cohort extends OpenCGAParentClass { * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. * @param {Boolean} [params.flattenAnnotations = "false"] - Flatten the annotations?. The default value is false. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Boolean} [params.deleted = "false"] - Boolean to retrieve deleted cohorts. The default value is false. * @returns {Promise} Promise object in the form of RestResponse instance. */ @@ -229,7 +209,7 @@ export default class Cohort extends OpenCGAParentClass { * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {"ADD SET REMOVE"} [params.samplesAction = "ADD"] - Action to be performed if the array of samples is being updated. The * default value is ADD. * @param {"ADD SET REMOVE"} [params.annotationSetsAction = "ADD"] - Action to be performed if the array of annotationSets is being @@ -249,7 +229,7 @@ export default class Cohort extends OpenCGAParentClass { * 'remove' containing the comma separated variables to be removed as a value when the action is REMOVE or a json with only the key * 'reset' containing the comma separated variables that will be set to the default value when the action is RESET. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {"ADD SET REMOVE RESET REPLACE"} [params.action = "ADD"] - Action to be performed: ADD to add new annotations; REPLACE to * replace the value of an already existing annotation; SET to set the new list of annotations removing any possible old annotations; * REMOVE to remove some annotations; RESET to set some annotations to the default value configured in the corresponding variables of the diff --git a/opencga-client/src/main/javascript/DiseasePanel.js b/opencga-client/src/main/javascript/DiseasePanel.js index c1f49cba7da..1aca2e4ef57 100644 --- a/opencga-client/src/main/javascript/DiseasePanel.js +++ b/opencga-client/src/main/javascript/DiseasePanel.js @@ -1,9 +1,12 @@ /** - * Copyright 2015-2020 OpenCB + * Copyright 2015-2024 OpenCB + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -12,7 +15,6 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-04-25 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -37,7 +39,7 @@ export default class DiseasePanel extends OpenCGAParentClass { * @param {Object} data - JSON containing the parameters to update the permissions. * @param {"SET ADD REMOVE RESET"} action = "ADD" - Action to be performed [ADD, SET, REMOVE or RESET]. The default value is ADD. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ updateAcl(members, action, data, params) { @@ -49,7 +51,7 @@ export default class DiseasePanel extends OpenCGAParentClass { * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Boolean} [params.includeResult = "false"] - Flag indicating to include the created or updated document result in the response. * The default value is false. * @returns {Promise} Promise object in the form of RestResponse instance. @@ -61,7 +63,7 @@ export default class DiseasePanel extends OpenCGAParentClass { /** Panel distinct method * @param {String} field - Comma separated list of fields for which to obtain the distinct values. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.id] - Comma separated list of panel IDs up to a maximum of 100. Also admits basic regular expressions using * the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * @param {String} [params.uuid] - Comma separated list of panel UUIDs up to a maximum of 100. @@ -101,12 +103,16 @@ export default class DiseasePanel extends OpenCGAParentClass { /** Import panels * @param {Object} [data] - Panel parameters. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not * provided. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ importPanels(data, params) { @@ -121,7 +127,7 @@ export default class DiseasePanel extends OpenCGAParentClass { * @param {Number} [params.skip] - Number of results to skip. * @param {Boolean} [params.count = "false"] - Get the total number of results matching the query. Deactivated by default. The default * value is false. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.id] - Comma separated list of panel IDs up to a maximum of 100. Also admits basic regular expressions using * the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * @param {String} [params.uuid] - Comma separated list of panel UUIDs up to a maximum of 100. @@ -161,7 +167,7 @@ export default class DiseasePanel extends OpenCGAParentClass { /** Returns the acl of the panels. If member is provided, it will only return the acl for the member. * @param {String} panels - Comma separated list of panel IDs up to a maximum of 100. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.member] - User or group id. * @param {Boolean} [params.silent = "false"] - Boolean to retrieve all possible entries that are queried for, false to raise an * exception whenever one of the entries looked for cannot be shown for whichever reason. The default value is false. @@ -174,7 +180,7 @@ export default class DiseasePanel extends OpenCGAParentClass { /** Delete existing panels * @param {String} panels - Comma separated list of panel ids. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ delete(panels, params) { @@ -186,7 +192,7 @@ export default class DiseasePanel extends OpenCGAParentClass { * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.version] - Comma separated list of panel versions. 'all' to get all the panel versions. Not supported if * multiple panel ids are provided. * @param {Boolean} [params.deleted = "false"] - Boolean to retrieve deleted panels. The default value is false. @@ -202,7 +208,7 @@ export default class DiseasePanel extends OpenCGAParentClass { * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Boolean} [params.includeResult = "false"] - Flag indicating to include the created or updated document result in the response. * The default value is false. * @returns {Promise} Promise object in the form of RestResponse instance. diff --git a/opencga-client/src/main/javascript/Family.js b/opencga-client/src/main/javascript/Family.js index 9bd5d316764..55a5ac0caf6 100644 --- a/opencga-client/src/main/javascript/Family.js +++ b/opencga-client/src/main/javascript/Family.js @@ -1,9 +1,12 @@ /** - * Copyright 2015-2020 OpenCB + * Copyright 2015-2024 OpenCB + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -12,7 +15,6 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-04-25 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -37,7 +39,7 @@ export default class Family extends OpenCGAParentClass { * @param {Object} data - JSON containing the parameters to add ACLs. * @param {"SET ADD REMOVE RESET"} action = "ADD" - Action to be performed [ADD, SET, REMOVE or RESET]. The default value is ADD. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {"NO YES YES_AND_VARIANT_VIEW"} [params.propagate = "NO"] - Propagate family permissions to related individuals and samples. * The default value is NO. * @returns {Promise} Promise object in the form of RestResponse instance. @@ -46,36 +48,12 @@ export default class Family extends OpenCGAParentClass { return this._post("families", null, "acl", members, "update", data, {action, ...params}); } - /** Fetch catalog family stats - * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. - * @param {String} [params.creationYear] - Creation year. - * @param {String} [params.creationMonth] - Creation month (JANUARY, FEBRUARY...). - * @param {String} [params.creationDay] - Creation day. - * @param {String} [params.creationDayOfWeek] - Creation day of week (MONDAY, TUESDAY...). - * @param {String} [params.status] - Status. - * @param {String} [params.phenotypes] - Phenotypes. - * @param {String} [params.release] - Release. - * @param {String} [params.version] - Version. - * @param {String} [params.numMembers] - Number of members. - * @param {String} [params.expectedSize] - Expected size. - * @param {String} [params.annotation] - Annotation filters. Example: age>30;gender=FEMALE. For more information, please visit - * http://docs.opencb.org/display/opencga/AnnotationSets+1.4.0. - * @param {Boolean} [params.default = "false"] - Calculate default stats. The default value is false. - * @param {String} [params.field] - List of fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: - * studies>>biotype;type;numSamples[0..10]:1. - * @returns {Promise} Promise object in the form of RestResponse instance. - */ - aggregationStats(params) { - return this._get("families", null, null, null, "aggregationStats", params); - } - /** Load annotation sets from a TSV file * @param {Object} [data] - JSON containing the 'content' of the TSV file if this has not yet been registered into OpenCGA. * @param {String} variableSetId - Variable set ID or name. * @param {String} path - Path where the TSV file is located in OpenCGA or where it should be located. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Boolean} [params.parents] - Flag indicating whether to create parent directories if they don't exist (only when TSV file was * not previously associated). * @param {String} [params.annotationSetId] - Annotation set id. If not provided, variableSetId will be used. @@ -90,7 +68,7 @@ export default class Family extends OpenCGAParentClass { * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.members] - Comma separated list of member ids to be associated to the created family. * @param {Boolean} [params.includeResult = "false"] - Flag indicating to include the created or updated document result in the response. * The default value is false. @@ -103,7 +81,7 @@ export default class Family extends OpenCGAParentClass { /** Family distinct method * @param {String} field - Comma separated list of fields for which to obtain the distinct values. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.id] - Comma separated list family IDs up to a maximum of 100. Also admits basic regular expressions using the * operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * @param {String} [params.name] - Comma separated list family names up to a maximum of 100. Also admits basic regular expressions using @@ -143,7 +121,7 @@ export default class Family extends OpenCGAParentClass { * @param {Boolean} [params.count = "false"] - Get the total number of results matching the query. Deactivated by default. The default * value is false. * @param {Boolean} [params.flattenAnnotations = "false"] - Flatten the annotations?. The default value is false. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.id] - Comma separated list family IDs up to a maximum of 100. Also admits basic regular expressions using the * operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * @param {String} [params.name] - Comma separated list family names up to a maximum of 100. Also admits basic regular expressions using @@ -177,7 +155,7 @@ export default class Family extends OpenCGAParentClass { /** Returns the acl of the families. If member is provided, it will only return the acl for the member. * @param {String} families - Comma separated list of family IDs or names up to a maximum of 100. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.member] - User or group id. * @param {Boolean} [params.silent = "false"] - Boolean to retrieve all possible entries that are queried for, false to raise an * exception whenever one of the entries looked for cannot be shown for whichever reason. The default value is false. @@ -190,7 +168,7 @@ export default class Family extends OpenCGAParentClass { /** Delete existing families * @param {String} families - Comma separated list of family ids. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ delete(families, params) { @@ -203,7 +181,7 @@ export default class Family extends OpenCGAParentClass { * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. * @param {Boolean} [params.flattenAnnotations = "false"] - Flatten the annotations?. The default value is false. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.version] - Comma separated list of family versions. 'all' to get all the family versions. Not supported if * multiple family ids are provided. * @param {Boolean} [params.deleted = "false"] - Boolean to retrieve deleted families. The default value is false. @@ -219,7 +197,7 @@ export default class Family extends OpenCGAParentClass { * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Boolean} [params.updateRoles = "false"] - Update the member roles within the family. The default value is false. * @param {Boolean} [params.updatePedigreeGraph = "false"] - Update the family pedigree graph. The default value is false. * @param {"ADD SET REMOVE"} [params.annotationSetsAction = "ADD"] - Action to be performed if the array of annotationSets is being @@ -239,7 +217,7 @@ export default class Family extends OpenCGAParentClass { * 'remove' containing the comma separated variables to be removed as a value when the action is REMOVE or a json with only the key * 'reset' containing the comma separated variables that will be set to the default value when the action is RESET. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {"ADD SET REMOVE RESET REPLACE"} [params.action = "ADD"] - Action to be performed: ADD to add new annotations; REPLACE to * replace the value of an already existing annotation; SET to set the new list of annotations removing any possible old annotations; * REMOVE to remove some annotations; RESET to set some annotations to the default value configured in the corresponding variables of the diff --git a/opencga-client/src/main/javascript/File.js b/opencga-client/src/main/javascript/File.js index c11712707b8..cfedee4d933 100644 --- a/opencga-client/src/main/javascript/File.js +++ b/opencga-client/src/main/javascript/File.js @@ -1,9 +1,12 @@ /** - * Copyright 2015-2020 OpenCB + * Copyright 2015-2024 OpenCB + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -12,7 +15,6 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-04-25 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -37,49 +39,19 @@ export default class File extends OpenCGAParentClass { * @param {Object} data - JSON containing the parameters to add ACLs. * @param {"SET ADD REMOVE RESET"} action = "ADD" - Action to be performed [ADD, SET, REMOVE or RESET]. The default value is ADD. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ updateAcl(members, action, data, params) { return this._post("files", null, "acl", members, "update", data, {action, ...params}); } - /** Fetch catalog file stats - * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. - * @param {String} [params.name] - Name. - * @param {String} [params.type] - Type. - * @param {String} [params.format] - Format. - * @param {String} [params.bioformat] - Bioformat. - * @param {String} [params.creationYear] - Creation year. - * @param {String} [params.creationMonth] - Creation month (JANUARY, FEBRUARY...). - * @param {String} [params.creationDay] - Creation day. - * @param {String} [params.creationDayOfWeek] - Creation day of week (MONDAY, TUESDAY...). - * @param {String} [params.status] - Status. - * @param {String} [params.release] - Release. - * @param {Boolean} [params.external] - External. - * @param {String} [params.size] - Size. - * @param {String} [params.software] - Software. - * @param {String} [params.experiment] - Experiment. - * @param {String} [params.numSamples] - Number of samples. - * @param {String} [params.numRelatedFiles] - Number of related files. - * @param {String} [params.annotation] - Annotation filters. Example: age>30;gender=FEMALE. For more information, please visit - * http://docs.opencb.org/display/opencga/AnnotationSets+1.4.0. - * @param {Boolean} [params.default = "false"] - Calculate default stats. The default value is false. - * @param {String} [params.field] - List of fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: - * studies>>biotype;type;numSamples[0..10]:1. - * @returns {Promise} Promise object in the form of RestResponse instance. - */ - aggregationStats(params) { - return this._get("files", null, null, null, "aggregationStats", params); - } - /** Load annotation sets from a TSV file * @param {Object} [data] - JSON containing the 'content' of the TSV file if this has not yet been registered into OpenCGA. * @param {String} variableSetId - Variable set ID or name. * @param {String} path - Path where the TSV file is located in OpenCGA or where it should be located. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Boolean} [params.parents] - Flag indicating whether to create parent directories if they don't exist (only when TSV file was * not previously associated). * @param {String} [params.annotationSetId] - Annotation set id. If not provided, variableSetId will be used. @@ -100,7 +72,7 @@ export default class File extends OpenCGAParentClass { /** Create file or folder * @param {Object} data - File parameters. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Boolean} [params.parents] - Create the parent directories if they do not exist. * @returns {Promise} Promise object in the form of RestResponse instance. */ @@ -111,7 +83,7 @@ export default class File extends OpenCGAParentClass { /** File distinct method * @param {String} field - Comma separated list of fields for which to obtain the distinct values. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.id] - Comma separated list of file IDs up to a maximum of 100. Also admits basic regular expressions using the * operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * @param {String} [params.uuid] - Comma separated list file UUIDs up to a maximum of 100. @@ -159,7 +131,11 @@ export default class File extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ fetch(data, params) { @@ -177,7 +153,7 @@ export default class File extends OpenCGAParentClass { /** Link an external file into catalog. * @param {Object} data - File parameters. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Boolean} [params.parents] - Create the parent directories if they do not exist. * @returns {Promise} Promise object in the form of RestResponse instance. */ @@ -188,12 +164,16 @@ export default class File extends OpenCGAParentClass { /** Link an external file into catalog asynchronously. * @param {Object} data - File parameters. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not * provided. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runLink(data, params) { @@ -203,12 +183,16 @@ export default class File extends OpenCGAParentClass { /** Associate non-registered samples for files with high volumes of samples. * @param {Object} data - File parameters. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not * provided. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runPostlink(data, params) { @@ -224,7 +208,7 @@ export default class File extends OpenCGAParentClass { * @param {Boolean} [params.count = "false"] - Get the total number of results matching the query. Deactivated by default. The default * value is false. * @param {Boolean} [params.flattenAnnotations = "false"] - Boolean indicating to flatten the annotations. The default value is false. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.id] - Comma separated list of file IDs up to a maximum of 100. Also admits basic regular expressions using the * operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * @param {String} [params.uuid] - Comma separated list file UUIDs up to a maximum of 100. @@ -275,21 +259,21 @@ export default class File extends OpenCGAParentClass { * ANNOTATION_GENEVSANNOTATION OTHER_NEWICK OTHER_BLAST OTHER_INTERACTION OTHER_GENOTYPE OTHER_PLINK OTHER_VCF OTHER_PED VCF4 VARIANT * ALIGNMENT COVERAGE SEQUENCE PEDIGREE REFERENCE_GENOME NONE UNKNOWN"} [params.bioformat] - File bioformat. * @param {String} [params.checksum] - Expected MD5 file checksum. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.relativeFilePath] - Path within catalog where the file will be located (default: root folder). * @param {String} [params.description] - description. * @param {Boolean} [params.parents] - Create the parent directories if they do not exist. * @returns {Promise} Promise object in the form of RestResponse instance. */ upload(params) { - return this._post("files", null, null, null, "upload", params); + return this._post("files", null, null, null, "upload", null, params); } /** Return the acl defined for the file or folder. If member is provided, it will only return the acl for the member. * @param {String} files - Comma separated list of file IDs or names up to a maximum of 100. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Comma separated list of Studies [[user@]project:]study where study and project can be either the ID - * or UUID up to a maximum of 100. + * @param {String} [params.study] - Comma separated list of Studies [[organization@]project:]study where study and project can be either + * the ID or UUID up to a maximum of 100. * @param {String} [params.member] - User or group id. * @param {Boolean} [params.silent = "false"] - Boolean to retrieve all possible entries that are queried for, false to raise an * exception whenever one of the entries looked for cannot be shown for whichever reason. The default value is false. @@ -302,7 +286,7 @@ export default class File extends OpenCGAParentClass { /** Delete existing files and folders * @param {String} files - Comma separated list of file ids, names or paths. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Boolean} [params.skipTrash = "false"] - Skip trash and delete the files/folders from disk directly (CANNOT BE RECOVERED). The * default value is false. * @returns {Promise} Promise object in the form of RestResponse instance. @@ -317,7 +301,7 @@ export default class File extends OpenCGAParentClass { * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. * @param {Boolean} [params.flattenAnnotations = "false"] - Flatten the annotations?. The default value is false. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Boolean} [params.deleted = "false"] - Boolean to retrieve deleted files. The default value is false. * @returns {Promise} Promise object in the form of RestResponse instance. */ @@ -328,7 +312,7 @@ export default class File extends OpenCGAParentClass { /** Unlink linked files and folders * @param {String} files - Comma separated list of file ids, names or paths. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ unlink(files, params) { @@ -341,7 +325,7 @@ export default class File extends OpenCGAParentClass { * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {"ADD SET REMOVE"} [params.sampleIdsAction = "ADD"] - Action to be performed if the array of samples is being updated. The * default value is ADD. * @param {"ADD SET REMOVE"} [params.annotationSetsAction = "ADD"] - Action to be performed if the array of annotationSets is being @@ -363,7 +347,7 @@ export default class File extends OpenCGAParentClass { * 'remove' containing the comma separated variables to be removed as a value when the action is REMOVE or a json with only the key * 'reset' containing the comma separated variables that will be set to the default value when the action is RESET. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {"ADD SET REMOVE RESET REPLACE"} [params.action = "ADD"] - Action to be performed: ADD to add new annotations; REPLACE to * replace the value of an already existing annotation; SET to set the new list of annotations removing any possible old annotations; * REMOVE to remove some annotations; RESET to set some annotations to the default value configured in the corresponding variables of the @@ -377,7 +361,7 @@ export default class File extends OpenCGAParentClass { /** Download file * @param {String} file - File id, name or path. Paths must be separated by : instead of /. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ download(file, params) { @@ -387,7 +371,7 @@ export default class File extends OpenCGAParentClass { /** Filter lines of the file containing the pattern * @param {String} file - File uuid, id, or name. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.pattern] - String pattern. * @param {Boolean} [params.ignoreCase] - Flag to perform a case insensitive search. * @param {Number} [params.maxCount] - Stop reading a file after 'n' matching lines. 0 means no limit. @@ -400,7 +384,7 @@ export default class File extends OpenCGAParentClass { /** Show the first lines of a file (up to a limit) * @param {String} file - File uuid, id, or name. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Number} [params.offset] - Starting byte from which the file will be read. * @param {Number} [params.lines = "20"] - Maximum number of lines to be returned up to a maximum of 1000. The default value is 20. * @returns {Promise} Promise object in the form of RestResponse instance. @@ -412,7 +396,7 @@ export default class File extends OpenCGAParentClass { /** Obtain the base64 content of an image * @param {String} file - File ID. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ image(file, params) { @@ -425,7 +409,7 @@ export default class File extends OpenCGAParentClass { * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ move(file, data, params) { @@ -435,7 +419,7 @@ export default class File extends OpenCGAParentClass { /** Refresh metadata from the selected file or folder. Return updated files. * @param {String} file - File id, name or path. Paths must be separated by : instead of /. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ refresh(file, params) { @@ -445,7 +429,7 @@ export default class File extends OpenCGAParentClass { /** Show the last lines of a file (up to a limit) * @param {String} file - File uuid, id, or name. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Number} [params.lines = "20"] - Maximum number of lines to be returned up to a maximum of 1000. The default value is 20. * @returns {Promise} Promise object in the form of RestResponse instance. */ @@ -462,7 +446,7 @@ export default class File extends OpenCGAParentClass { * @param {Number} [params.skip] - Number of results to skip. * @param {Boolean} [params.count = "false"] - Get the total number of results matching the query. Deactivated by default. The default * value is false. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ list(folder, params) { @@ -474,7 +458,7 @@ export default class File extends OpenCGAParentClass { * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Number} [params.maxDepth] - Maximum depth to get files from. * @returns {Promise} Promise object in the form of RestResponse instance. */ diff --git a/opencga-client/src/main/javascript/GA4GH.js b/opencga-client/src/main/javascript/GA4GH.js index c74108c5ca5..222eaa7c67c 100644 --- a/opencga-client/src/main/javascript/GA4GH.js +++ b/opencga-client/src/main/javascript/GA4GH.js @@ -1,9 +1,12 @@ /** - * Copyright 2015-2020 OpenCB + * Copyright 2015-2024 OpenCB + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -12,7 +15,6 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-04-25 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -37,12 +39,12 @@ export default class GA4GH extends OpenCGAParentClass { * @returns {Promise} Promise object in the form of RestResponse instance. */ searchReads() { - return this._post("ga4gh", null, "reads", null, "search"); + return this._post("ga4gh", null, "reads", null, "search", null); } /** Fetch alignment files using HTSget protocol * @param {String} file - File id, name or path. - * @param {String} study - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} study - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.referenceName] - Reference sequence name (Example: 'chr1', '1' or 'chrX'. * @param {Number} [params.start] - The start position of the range on the reference, 0-based, inclusive. @@ -76,7 +78,7 @@ export default class GA4GH extends OpenCGAParentClass { * @returns {Promise} Promise object in the form of RestResponse instance. */ searchVariants() { - return this._post("ga4gh", null, "variants", null, "search"); + return this._post("ga4gh", null, "variants", null, "search", null); } } \ No newline at end of file diff --git a/opencga-client/src/main/javascript/Individual.js b/opencga-client/src/main/javascript/Individual.js index 5ec331eeca4..fb33520e6db 100644 --- a/opencga-client/src/main/javascript/Individual.js +++ b/opencga-client/src/main/javascript/Individual.js @@ -1,9 +1,12 @@ /** - * Copyright 2015-2020 OpenCB + * Copyright 2015-2024 OpenCB + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -12,7 +15,6 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-04-25 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -38,7 +40,7 @@ export default class Individual extends OpenCGAParentClass { * the permissions defined to the samples that are associated to the matching individuals. * @param {"SET ADD REMOVE RESET"} action = "ADD" - Action to be performed [ADD, SET, REMOVE or RESET]. The default value is ADD. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Boolean} [params.propagate = "false"] - Propagate individual permissions to related samples. The default value is false. * @returns {Promise} Promise object in the form of RestResponse instance. */ @@ -46,43 +48,12 @@ export default class Individual extends OpenCGAParentClass { return this._post("individuals", null, "acl", members, "update", data, {action, ...params}); } - /** Fetch catalog individual stats - * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. - * @param {Boolean} [params.hasFather] - Has father. - * @param {Boolean} [params.hasMother] - Has mother. - * @param {String} [params.sex] - Sex. - * @param {String} [params.karyotypicSex] - Karyotypic sex. - * @param {String} [params.ethnicity] - Ethnicity. - * @param {String} [params.population] - Population. - * @param {String} [params.creationYear] - Creation year. - * @param {String} [params.creationMonth] - Creation month (JANUARY, FEBRUARY...). - * @param {String} [params.creationDay] - Creation day. - * @param {String} [params.creationDayOfWeek] - Creation day of week (MONDAY, TUESDAY...). - * @param {String} [params.status] - Status. - * @param {String} [params.lifeStatus] - Life status. - * @param {String} [params.phenotypes] - Phenotypes. - * @param {String} [params.numSamples] - Number of samples. - * @param {Boolean} [params.parentalConsanguinity] - Parental consanguinity. - * @param {String} [params.release] - Release. - * @param {String} [params.version] - Version. - * @param {String} [params.annotation] - Annotation filters. Example: age>30;gender=FEMALE. For more information, please visit - * http://docs.opencb.org/display/opencga/AnnotationSets+1.4.0. - * @param {Boolean} [params.default = "false"] - Calculate default stats. The default value is false. - * @param {String} [params.field] - List of fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: - * studies>>biotype;type;numSamples[0..10]:1. - * @returns {Promise} Promise object in the form of RestResponse instance. - */ - aggregationStats(params) { - return this._get("individuals", null, null, null, "aggregationStats", params); - } - /** Load annotation sets from a TSV file * @param {Object} [data] - JSON containing the 'content' of the TSV file if this has not yet been registered into OpenCGA. * @param {String} variableSetId - Variable set ID or name. * @param {String} path - Path where the TSV file is located in OpenCGA or where it should be located. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Boolean} [params.parents] - Flag indicating whether to create parent directories if they don't exist (only when TSV file was * not previously associated). * @param {String} [params.annotationSetId] - Annotation set id. If not provided, variableSetId will be used. @@ -97,7 +68,7 @@ export default class Individual extends OpenCGAParentClass { * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.samples] - Comma separated list of sample ids to be associated to the created individual. * @param {Boolean} [params.includeResult = "false"] - Flag indicating to include the created or updated document result in the response. * The default value is false. @@ -110,7 +81,7 @@ export default class Individual extends OpenCGAParentClass { /** Individual distinct method * @param {String} field - Comma separated list of fields for which to obtain the distinct values. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.id] - Comma separated list individual IDs up to a maximum of 100. Also admits basic regular expressions using * the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * @param {String} [params.uuid] - Comma separated list individual UUIDs up to a maximum of 100. @@ -161,7 +132,7 @@ export default class Individual extends OpenCGAParentClass { * @param {Boolean} [params.count = "false"] - Get the total number of results matching the query. Deactivated by default. The default * value is false. * @param {Boolean} [params.flattenAnnotations = "false"] - Flatten the annotations?. The default value is false. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.id] - Comma separated list individual IDs up to a maximum of 100. Also admits basic regular expressions using * the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * @param {String} [params.uuid] - Comma separated list individual UUIDs up to a maximum of 100. @@ -206,7 +177,7 @@ export default class Individual extends OpenCGAParentClass { /** Return the acl of the individual. If member is provided, it will only return the acl for the member. * @param {String} individuals - Comma separated list of individual IDs, names or UUIDs up to a maximum of 100. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.member] - User or group id. * @param {Boolean} [params.silent = "false"] - Boolean to retrieve all possible entries that are queried for, false to raise an * exception whenever one of the entries looked for cannot be shown for whichever reason. The default value is false. @@ -221,7 +192,7 @@ export default class Individual extends OpenCGAParentClass { * @param {Object} [params] - The Object containing the following optional parameters: * @param {Boolean} [params.force = "false"] - Force the deletion of individuals that already belong to families. The default value is * false. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ delete(individuals, params) { @@ -234,7 +205,7 @@ export default class Individual extends OpenCGAParentClass { * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. * @param {Boolean} [params.flattenAnnotations = "false"] - Flatten the annotations?. The default value is false. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.version] - Comma separated list of individual versions. 'all' to get all the individual versions. Not * supported if multiple individual ids are provided. * @param {Boolean} [params.deleted = "false"] - Boolean to retrieve deleted individuals. The default value is false. @@ -250,7 +221,7 @@ export default class Individual extends OpenCGAParentClass { * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {"ADD SET REMOVE"} [params.samplesAction = "ADD"] - Action to be performed if the array of samples is being updated. The * default value is ADD. * @param {"ADD SET REMOVE"} [params.phenotypesAction = "ADD"] - Action to be performed if the array of phenotypes is being updated [SET, @@ -274,7 +245,7 @@ export default class Individual extends OpenCGAParentClass { * 'remove' containing the comma separated variables to be removed as a value when the action is REMOVE or a json with only the key * 'reset' containing the comma separated variables that will be set to the default value when the action is RESET. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {"ADD SET REMOVE RESET REPLACE"} [params.action = "ADD"] - Action to be performed: ADD to add new annotations; REPLACE to * replace the value of an already existing annotation; SET to set the new list of annotations removing any possible old annotations; * REMOVE to remove some annotations; RESET to set some annotations to the default value configured in the corresponding variables of the @@ -291,7 +262,7 @@ export default class Individual extends OpenCGAParentClass { * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. * @param {Boolean} [params.flattenAnnotations = "false"] - Flatten the annotations?. The default value is false. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Number} [params.degree = "2"] - Pedigree degree. The default value is 2. * @returns {Promise} Promise object in the form of RestResponse instance. */ diff --git a/opencga-client/src/main/javascript/Job.js b/opencga-client/src/main/javascript/Job.js index 67b8db8e091..bb88f18dd7a 100644 --- a/opencga-client/src/main/javascript/Job.js +++ b/opencga-client/src/main/javascript/Job.js @@ -1,9 +1,12 @@ /** - * Copyright 2015-2020 OpenCB + * Copyright 2015-2024 OpenCB + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -12,7 +15,6 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-04-25 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -42,37 +44,10 @@ export default class Job extends OpenCGAParentClass { return this._post("jobs", null, "acl", members, "update", data, action); } - /** Fetch catalog job stats - * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. - * @param {String} [params.toolId] - Tool id. - * @param {String} [params.toolScope] - Tool scope. - * @param {String} [params.toolType] - Tool type. - * @param {String} [params.toolResource] - Tool resource. - * @param {String} [params.userId] - User id. - * @param {String} [params.priority] - Priority. - * @param {String} [params.tags] - Tags. - * @param {String} [params.executorId] - Executor id. - * @param {String} [params.executorFramework] - Executor framework. - * @param {String} [params.creationYear] - Creation year. - * @param {String} [params.creationMonth] - Creation month (JANUARY, FEBRUARY...). - * @param {String} [params.creationDay] - Creation day. - * @param {String} [params.creationDayOfWeek] - Creation day of week (MONDAY, TUESDAY...). - * @param {String} [params.status] - Status. - * @param {String} [params.release] - Release. - * @param {Boolean} [params.default = "false"] - Calculate default stats. The default value is false. - * @param {String} [params.field] - List of fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: - * studies>>biotype;type;numSamples[0..10]:1. - * @returns {Promise} Promise object in the form of RestResponse instance. - */ - aggregationStats(params) { - return this._get("jobs", null, null, null, "aggregationStats", params); - } - /** Register an executed job with POST method * @param {Object} data - job. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ create(data, params) { @@ -82,7 +57,7 @@ export default class Job extends OpenCGAParentClass { /** Job distinct method * @param {String} field - Comma separated list of fields for which to obtain the distinct values. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Boolean} [params.otherStudies = "false"] - Flag indicating the entries being queried can belong to any related study, not just * the primary one. The default value is false. * @param {String} [params.id] - Comma separated list of job IDs up to a maximum of 100. Also admits basic regular expressions using the @@ -120,7 +95,8 @@ export default class Job extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ retry(data, params) { @@ -135,7 +111,7 @@ export default class Job extends OpenCGAParentClass { * @param {Number} [params.skip] - Number of results to skip. * @param {Boolean} [params.count = "false"] - Get the total number of results matching the query. Deactivated by default. The default * value is false. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Boolean} [params.otherStudies = "false"] - Flag indicating the entries being queried can belong to any related study, not just * the primary one. The default value is false. * @param {String} [params.id] - Comma separated list of job IDs up to a maximum of 100. Also admits basic regular expressions using the @@ -168,7 +144,7 @@ export default class Job extends OpenCGAParentClass { /** Provide a summary of the running jobs * @param {Object} [params] - The Object containing the following optional parameters: * @param {Number} [params.limit = "20"] - Maximum number of jobs to be returned. The default value is 20. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.internalStatus] - Filter by internal status. * @param {String} [params.priority] - Priority of the job. * @param {String} [params.userId] - User that created the job. @@ -195,7 +171,7 @@ export default class Job extends OpenCGAParentClass { /** Delete existing jobs * @param {String} jobs - Comma separated list of job ids. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ delete(jobs, params) { @@ -207,7 +183,7 @@ export default class Job extends OpenCGAParentClass { * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Boolean} [params.deleted = "false"] - Boolean to retrieve deleted jobs. The default value is false. * @returns {Promise} Promise object in the form of RestResponse instance. */ @@ -221,7 +197,7 @@ export default class Job extends OpenCGAParentClass { * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Boolean} [params.includeResult = "false"] - Flag indicating to include the created or updated document result in the response. * The default value is false. * @returns {Promise} Promise object in the form of RestResponse instance. @@ -230,10 +206,20 @@ export default class Job extends OpenCGAParentClass { return this._post("jobs", jobs, null, null, "update", data, params); } + /** Send a signal to kill a pending or running job + * @param {String} job - Job ID or UUID. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + kill(job, params) { + return this._post("jobs", job, null, null, "kill", null, params); + } + /** Show the first lines of a log file (up to a limit) * @param {String} job - Job ID or UUID. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Number} [params.offset] - Starting byte from which the file will be read. * @param {Number} [params.lines = "20"] - Maximum number of lines to be returned up to a maximum of 1000. The default value is 20. * @param {String} [params.type] - Log file to be shown (stdout or stderr). @@ -246,7 +232,7 @@ export default class Job extends OpenCGAParentClass { /** Show the last lines of a log file (up to a limit) * @param {String} job - Job ID or UUID. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Number} [params.lines = "20"] - Maximum number of lines to be returned up to a maximum of 1000. The default value is 20. * @param {String} [params.type] - Log file to be shown (stdout or stderr). * @returns {Promise} Promise object in the form of RestResponse instance. diff --git a/opencga-client/src/main/javascript/Meta.js b/opencga-client/src/main/javascript/Meta.js index 0cea26d1a89..0a536de0c03 100644 --- a/opencga-client/src/main/javascript/Meta.js +++ b/opencga-client/src/main/javascript/Meta.js @@ -1,9 +1,12 @@ /** - * Copyright 2015-2020 OpenCB + * Copyright 2015-2024 OpenCB + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -12,7 +15,6 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-04-25 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Organization.js b/opencga-client/src/main/javascript/Organization.js new file mode 100644 index 00000000000..09b79429246 --- /dev/null +++ b/opencga-client/src/main/javascript/Organization.js @@ -0,0 +1,183 @@ +/** + * Copyright 2015-2024 OpenCB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * WARNING: AUTOGENERATED CODE + * + * This code was generated by a tool. + * + * Manual changes to this file may cause unexpected behavior in your application. + * Manual changes to this file will be overwritten if the code is regenerated. + * +**/ + +import OpenCGAParentClass from "./../opencga-parent-class.js"; + + +/** + * This class contains the methods for the "Organization" resource + */ + +export default class Organization extends OpenCGAParentClass { + + constructor(config) { + super(config); + } + + /** Create a new organization + * @param {Object} data - JSON containing the organization to be created. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. + * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. + * @param {Boolean} [params.includeResult = "false"] - Flag indicating to include the created or updated document result in the response. + * The default value is false. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + create(data, params) { + return this._post("organizations", null, null, null, "create", data, params); + } + + /** Create a new note + * @param {Object} data - JSON containing the Note to be added. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. + * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. + * @param {Boolean} [params.includeResult = "false"] - Flag indicating to include the created or updated document result in the response. + * The default value is false. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + createNotes(data, params) { + return this._post("organizations", null, "notes", null, "create", data, params); + } + + /** Search for notes of scope ORGANIZATION + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. + * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. + * @param {String} [params.creationDate] - Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + * @param {String} [params.modificationDate] - Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + * @param {String} [params.id] - Note unique identifier. + * @param {String} [params.scope] - Scope of the Note. + * @param {String} [params.visibility] - Visibility of the Note. + * @param {String} [params.uuid] - Unique 32-character identifier assigned automatically by OpenCGA. + * @param {String} [params.userId] - User that wrote that Note. + * @param {String} [params.tags] - Note tags. + * @param {String} [params.version] - Autoincremental version assigned to the registered entry. By default, updates does not create new + * versions. To enable versioning, users must set the `incVersion` flag from the /update web service when updating the document. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + searchNotes(params) { + return this._get("organizations", null, "notes", null, "search", params); + } + + /** Delete note + * @param {String} id - Note unique identifier. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {Boolean} [params.includeResult = "false"] - Flag indicating to include the created or updated document result in the response. + * The default value is false. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + deleteNotes(id, params) { + return this._delete("organizations", null, "notes", id, "delete", params); + } + + /** Update a note + * @param {String} id - Note unique identifier. + * @param {Object} data - JSON containing the Note fields to be updated. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. + * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. + * @param {"ADD SET REMOVE"} [params.tagsAction = "ADD"] - Action to be performed if the array of tags is being updated. The default + * value is ADD. + * @param {Boolean} [params.includeResult = "false"] - Flag indicating to include the created or updated document result in the response. + * The default value is false. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + updateNotes(id, data, params) { + return this._post("organizations", null, "notes", id, "update", data, params); + } + + /** Update the user status + * @param {String} user - User ID. + * @param {Object} data - JSON containing the User fields to be updated. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. + * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. + * @param {String} [params.organization] - Organization id. + * @param {Boolean} [params.includeResult = "false"] - Flag indicating to include the created or updated document result in the response. + * The default value is false. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + userUpdateStatus(user, data, params) { + return this._post("organizations/user", user, "status", null, "update", data, params); + } + + /** Update the user information + * @param {String} user - User ID. + * @param {Object} data - JSON containing the User fields to be updated. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. + * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. + * @param {String} [params.organization] - Organization id. + * @param {Boolean} [params.includeResult = "false"] - Flag indicating to include the created or updated document result in the response. + * The default value is false. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + updateUser(user, data, params) { + return this._post("organizations", null, "user", user, "update", data, params); + } + + /** Update the Organization configuration attributes + * @param {String} organization - Organization id. + * @param {Object} data - JSON containing the params to be updated. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. + * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. + * @param {Boolean} [params.includeResult = "false"] - Flag indicating to include the created or updated document result in the response. + * The default value is false. + * @param {"ADD SET REMOVE REPLACE"} [params.authenticationOriginsAction = "ADD"] - Action to be performed if the array of + * authenticationOrigins is being updated. The default value is ADD. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + updateConfiguration(organization, data, params) { + return this._post("organizations", organization, "configuration", null, "update", data, params); + } + + /** Return the organization information + * @param {String} organization - Organization id. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. + * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + info(organization, params) { + return this._get("organizations", organization, null, null, "info", params); + } + + /** Update some organization attributes + * @param {String} organization - Organization id. + * @param {Object} data - JSON containing the params to be updated. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. + * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. + * @param {Boolean} [params.includeResult = "false"] - Flag indicating to include the created or updated document result in the response. + * The default value is false. + * @param {"ADD REMOVE"} [params.adminsAction = "ADD"] - Action to be performed if the array of admins is being updated. The default + * value is ADD. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + update(organization, data, params) { + return this._post("organizations", organization, null, null, "update", data, params); + } + +} \ No newline at end of file diff --git a/opencga-client/src/main/javascript/Project.js b/opencga-client/src/main/javascript/Project.js index 1afb1f8c793..041fa7ed691 100644 --- a/opencga-client/src/main/javascript/Project.js +++ b/opencga-client/src/main/javascript/Project.js @@ -1,9 +1,12 @@ /** - * Copyright 2015-2020 OpenCB + * Copyright 2015-2024 OpenCB + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -12,7 +15,6 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-04-25 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -51,11 +53,10 @@ export default class Project extends OpenCGAParentClass { * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. * @param {Number} [params.limit] - Number of results to be returned. * @param {Number} [params.skip] - Number of results to skip. - * @param {String} [params.owner] - Owner of the project. - * @param {String} [params.id] - Project [user@]project where project can be either the ID or the alias. + * @param {String} [params.organization] - Project organization. + * @param {String} [params.id] - Project [organization@]project where project can be either the ID or the alias. * @param {String} [params.name] - Project name. * @param {String} [params.fqn] - Project fqn. - * @param {String} [params.organization] - Project organization. * @param {String} [params.description] - Project description. * @param {String} [params.study] - Study id. * @param {String} [params.creationDate] - Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. @@ -68,30 +69,8 @@ export default class Project extends OpenCGAParentClass { return this._get("projects", null, null, null, "search", params); } - /** Fetch catalog project stats - * @param {String} projects - Comma separated list of projects [user@]project up to a maximum of 100. - * @param {Object} [params] - The Object containing the following optional parameters: - * @param {Boolean} [params.default = "true"] - Calculate default stats. The default value is true. - * @param {String} [params.fileFields] - List of file fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: - * studies>>biotype;type. - * @param {String} [params.individualFields] - List of individual fields separated by semicolons, e.g.: studies;type. For nested fields - * use >>, e.g.: studies>>biotype;type. - * @param {String} [params.familyFields] - List of family fields separated by semicolons, e.g.: studies;type. For nested fields use >>, - * e.g.: studies>>biotype;type. - * @param {String} [params.sampleFields] - List of sample fields separated by semicolons, e.g.: studies;type. For nested fields use >>, - * e.g.: studies>>biotype;type. - * @param {String} [params.cohortFields] - List of cohort fields separated by semicolons, e.g.: studies;type. For nested fields use >>, - * e.g.: studies>>biotype;type. - * @param {String} [params.jobFields] - List of job fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: - * studies>>biotype;type. - * @returns {Promise} Promise object in the form of RestResponse instance. - */ - aggregationStats(projects, params) { - return this._get("projects", projects, null, null, "aggregationStats", params); - } - /** Fetch project information - * @param {String} projects - Comma separated list of projects [user@]project up to a maximum of 100. + * @param {String} projects - Comma separated list of projects [organization@]project up to a maximum of 100. * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. @@ -102,15 +81,15 @@ export default class Project extends OpenCGAParentClass { } /** Increment current release number in the project - * @param {String} project - Project [user@]project where project can be either the ID or the alias. + * @param {String} project - Project [organization@]project where project can be either the ID or the alias. * @returns {Promise} Promise object in the form of RestResponse instance. */ incRelease(project) { - return this._post("projects", project, null, null, "incRelease"); + return this._post("projects", project, null, null, "incRelease", null); } /** Fetch all the studies contained in the project - * @param {String} project - Project [user@]project where project can be either the ID or the alias. + * @param {String} project - Project [organization@]project where project can be either the ID or the alias. * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. @@ -123,7 +102,7 @@ export default class Project extends OpenCGAParentClass { } /** Update some project attributes - * @param {String} project - Project [user@]project where project can be either the ID or the alias. + * @param {String} project - Project [organization@]project where project can be either the ID or the alias. * @param {Object} data - JSON containing the params to be updated. It will be only possible to update organism fields not previously * defined. * @param {Object} [params] - The Object containing the following optional parameters: diff --git a/opencga-client/src/main/javascript/Sample.js b/opencga-client/src/main/javascript/Sample.js index 907ffe1b935..48edf930fc8 100644 --- a/opencga-client/src/main/javascript/Sample.js +++ b/opencga-client/src/main/javascript/Sample.js @@ -1,9 +1,12 @@ /** - * Copyright 2015-2020 OpenCB + * Copyright 2015-2024 OpenCB + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -12,7 +15,6 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-04-25 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -38,44 +40,19 @@ export default class Sample extends OpenCGAParentClass { * the permissions defined to the individuals that are associated to the matching samples. * @param {"SET ADD REMOVE RESET"} action = "ADD" - Action to be performed [ADD, SET, REMOVE or RESET]. The default value is ADD. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ updateAcl(members, action, data, params) { return this._post("samples", null, "acl", members, "update", data, {action, ...params}); } - /** Fetch catalog sample stats - * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. - * @param {String} [params.source] - Source. - * @param {String} [params.creationYear] - Creation year. - * @param {String} [params.creationMonth] - Creation month (JANUARY, FEBRUARY...). - * @param {String} [params.creationDay] - Creation day. - * @param {String} [params.creationDayOfWeek] - Creation day of week (MONDAY, TUESDAY...). - * @param {String} [params.status] - Status. - * @param {String} [params.type] - Type. - * @param {String} [params.phenotypes] - Phenotypes. - * @param {String} [params.release] - Release. - * @param {String} [params.version] - Version. - * @param {Boolean} [params.somatic] - Somatic. - * @param {String} [params.annotation] - Annotation filters. Example: age>30;gender=FEMALE. For more information, please visit - * http://docs.opencb.org/display/opencga/AnnotationSets+1.4.0. - * @param {Boolean} [params.default = "false"] - Calculate default stats. The default value is false. - * @param {String} [params.field] - List of fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: - * studies>>biotype;type;numSamples[0..10]:1. - * @returns {Promise} Promise object in the form of RestResponse instance. - */ - aggregationStats(params) { - return this._get("samples", null, null, null, "aggregationStats", params); - } - /** Load annotation sets from a TSV file * @param {Object} [data] - JSON containing the 'content' of the TSV file if this has not yet been registered into OpenCGA. * @param {String} variableSetId - Variable set ID or name. * @param {String} path - Path where the TSV file is located in OpenCGA or where it should be located. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Boolean} [params.parents] - Flag indicating whether to create parent directories if they don't exist (only when TSV file was * not previously associated). * @param {String} [params.annotationSetId] - Annotation set id. If not provided, variableSetId will be used. @@ -90,7 +67,7 @@ export default class Sample extends OpenCGAParentClass { * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Boolean} [params.includeResult = "false"] - Flag indicating to include the created or updated document result in the response. * The default value is false. * @returns {Promise} Promise object in the form of RestResponse instance. @@ -102,7 +79,7 @@ export default class Sample extends OpenCGAParentClass { /** Sample distinct method * @param {String} field - Comma separated list of fields for which to obtain the distinct values. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.id] - Comma separated list sample IDs up to a maximum of 100. Also admits basic regular expressions using the * operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * @param {String} [params.uuid] - Comma separated list sample UUIDs up to a maximum of 100. @@ -156,7 +133,7 @@ export default class Sample extends OpenCGAParentClass { /** Load samples from a ped file [EXPERIMENTAL] * @param {String} file - file. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.variableSet] - variableSet. * @returns {Promise} Promise object in the form of RestResponse instance. */ @@ -174,7 +151,7 @@ export default class Sample extends OpenCGAParentClass { * value is false. * @param {Boolean} [params.includeIndividual = "false"] - Include Individual object as an attribute. The default value is false. * @param {Boolean} [params.flattenAnnotations = "false"] - Flatten the annotations?. The default value is false. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.id] - Comma separated list sample IDs up to a maximum of 100. Also admits basic regular expressions using the * operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * @param {String} [params.uuid] - Comma separated list sample UUIDs up to a maximum of 100. @@ -228,7 +205,7 @@ export default class Sample extends OpenCGAParentClass { /** Returns the acl of the samples. If member is provided, it will only return the acl for the member. * @param {String} samples - Comma separated list sample IDs or UUIDs up to a maximum of 100. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.member] - User or group id. * @param {Boolean} [params.silent = "false"] - Boolean to retrieve all possible entries that are queried for, false to raise an * exception whenever one of the entries looked for cannot be shown for whichever reason. The default value is false. @@ -247,7 +224,7 @@ export default class Sample extends OpenCGAParentClass { * deleted. Possible actions are NONE, TRASH, DELETE. The default value is NONE. * @param {Boolean} [params.deleteEmptyCohorts = "false"] - Boolean indicating if the cohorts associated only to the sample to be deleted * should be also deleted. The default value is false. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ delete(samples, params) { @@ -261,7 +238,7 @@ export default class Sample extends OpenCGAParentClass { * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. * @param {Boolean} [params.includeIndividual = "false"] - Include Individual object as an attribute. The default value is false. * @param {Boolean} [params.flattenAnnotations = "false"] - Flatten the annotations?. The default value is false. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.version] - Comma separated list of sample versions. 'all' to get all the sample versions. Not supported if * multiple sample ids are provided. * @param {Boolean} [params.deleted = "false"] - Boolean to retrieve deleted entries. The default value is false. @@ -277,7 +254,7 @@ export default class Sample extends OpenCGAParentClass { * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {"ADD SET REMOVE"} [params.annotationSetsAction = "ADD"] - Action to be performed if the array of annotationSets is being * updated. The default value is ADD. * @param {"ADD SET REMOVE"} [params.phenotypesAction = "ADD"] - Action to be performed if the array of phenotypes is being updated [SET, @@ -297,7 +274,7 @@ export default class Sample extends OpenCGAParentClass { * 'remove' containing the comma separated variables to be removed as a value when the action is REMOVE or a json with only the key * 'reset' containing the comma separated variables that will be set to the default value when the action is RESET. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {"ADD SET REMOVE RESET REPLACE"} [params.action = "ADD"] - Action to be performed: ADD to add new annotations; REPLACE to * replace the value of an already existing annotation; SET to set the new list of annotations removing any possible old annotations; * REMOVE to remove some annotations; RESET to set some annotations to the default value configured in the corresponding variables of the diff --git a/opencga-client/src/main/javascript/Study.js b/opencga-client/src/main/javascript/Study.js index fed4b5a6c39..bbe53e93df9 100644 --- a/opencga-client/src/main/javascript/Study.js +++ b/opencga-client/src/main/javascript/Study.js @@ -1,9 +1,12 @@ /** - * Copyright 2015-2020 OpenCB + * Copyright 2015-2024 OpenCB + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -12,7 +15,6 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-04-25 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -47,7 +49,7 @@ export default class Study extends OpenCGAParentClass { * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. - * @param {String} [params.project] - Project [user@]project where project can be either the ID or the alias. + * @param {String} [params.project] - Project [organization@]project where project can be either the ID or the alias. * @param {Boolean} [params.includeResult = "false"] - Flag indicating to include the created or updated document result in the response. * The default value is false. * @returns {Promise} Promise object in the form of RestResponse instance. @@ -57,7 +59,7 @@ export default class Study extends OpenCGAParentClass { } /** Search studies - * @param {String} project - Project [user@]project where project can be either the ID or the alias. + * @param {String} project - Project [organization@]project where project can be either the ID or the alias. * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. @@ -81,8 +83,8 @@ export default class Study extends OpenCGAParentClass { } /** Return the acl of the study. If member is provided, it will only return the acl for the member. - * @param {String} studies - Comma separated list of Studies [[user@]project:]study where study and project can be either the ID or UUID - * up to a maximum of 100. + * @param {String} studies - Comma separated list of Studies [[organization@]project:]study where study and project can be either the ID + * or UUID up to a maximum of 100. * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.member] - User or group id. * @param {Boolean} [params.silent = "false"] - Boolean to retrieve all possible entries that are queried for, false to raise an @@ -93,31 +95,9 @@ export default class Study extends OpenCGAParentClass { return this._get("studies", studies, null, null, "acl", params); } - /** Fetch catalog study stats - * @param {String} studies - Comma separated list of studies [[user@]project:]study up to a maximum of 100. - * @param {Object} [params] - The Object containing the following optional parameters: - * @param {Boolean} [params.default = "true"] - Calculate default stats. The default value is true. - * @param {String} [params.fileFields] - List of file fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: - * studies>>biotype;type. - * @param {String} [params.individualFields] - List of individual fields separated by semicolons, e.g.: studies;type. For nested fields - * use >>, e.g.: studies>>biotype;type. - * @param {String} [params.familyFields] - List of family fields separated by semicolons, e.g.: studies;type. For nested fields use >>, - * e.g.: studies>>biotype;type. - * @param {String} [params.sampleFields] - List of sample fields separated by semicolons, e.g.: studies;type. For nested fields use >>, - * e.g.: studies>>biotype;type. - * @param {String} [params.cohortFields] - List of cohort fields separated by semicolons, e.g.: studies;type. For nested fields use >>, - * e.g.: studies>>biotype;type. - * @param {String} [params.jobFields] - List of job fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: - * studies>>biotype;type. - * @returns {Promise} Promise object in the form of RestResponse instance. - */ - aggregationStats(studies, params) { - return this._get("studies", studies, null, null, "aggregationStats", params); - } - /** Fetch study information - * @param {String} studies - Comma separated list of Studies [[user@]project:]study where study and project can be either the ID or UUID - * up to a maximum of 100. + * @param {String} studies - Comma separated list of Studies [[organization@]project:]study where study and project can be either the ID + * or UUID up to a maximum of 100. * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. @@ -138,8 +118,8 @@ export default class Study extends OpenCGAParentClass { * @param {String} [params.operationId] - Audit operation UUID. * @param {String} [params.userId] - User ID. * @param {String} [params.action] - Action performed by the user. - * @param {"AUDIT USER PROJECT STUDY FILE SAMPLE JOB INDIVIDUAL COHORT DISEASE_PANEL FAMILY CLINICAL_ANALYSIS INTERPRETATION VARIANT - * ALIGNMENT CLINICAL EXPRESSION RGA FUNCTIONAL"} [params.resource] - Resource involved. + * @param {"AUDIT NOTE ORGANIZATION USER PROJECT STUDY FILE SAMPLE JOB INDIVIDUAL COHORT DISEASE_PANEL FAMILY CLINICAL_ANALYSIS + * INTERPRETATION VARIANT ALIGNMENT CLINICAL EXPRESSION RGA FUNCTIONAL"} [params.resource] - Resource involved. * @param {String} [params.resourceId] - Resource ID. * @param {String} [params.resourceUuid] - resource UUID. * @param {"SUCCESS ERROR"} [params.status] - Filter by status. @@ -151,7 +131,7 @@ export default class Study extends OpenCGAParentClass { } /** Return the groups present in the study. For owners and administrators only. - * @param {String} study - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} study - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.id] - Group id. If provided, it will only fetch information for the provided group. * @param {Boolean} [params.silent = "false"] - Boolean to retrieve all possible entries that are queried for, false to raise an @@ -163,7 +143,7 @@ export default class Study extends OpenCGAParentClass { } /** Add or remove a group - * @param {String} study - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} study - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Object} data - JSON containing the parameters. * @param {Object} [params] - The Object containing the following optional parameters: * @param {"ADD REMOVE"} [params.action = "ADD"] - Action to be performed: ADD or REMOVE a group. The default value is ADD. @@ -174,7 +154,7 @@ export default class Study extends OpenCGAParentClass { } /** Add, set or remove users from an existing group - * @param {String} study - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} study - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} group - Group name. * @param {Object} data - JSON containing the parameters. * @param {Object} [params] - The Object containing the following optional parameters: @@ -186,8 +166,71 @@ export default class Study extends OpenCGAParentClass { return this._post("studies", study, "groups", group, "users/update", data, params); } + /** Create a new note + * @param {String} study - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @param {Object} data - JSON containing the Note to be added. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. + * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. + * @param {Boolean} [params.includeResult = "false"] - Flag indicating to include the created or updated document result in the response. + * The default value is false. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + createNotes(study, data, params) { + return this._post("studies", study, "notes", null, "create", data, params); + } + + /** Search for notes of scope STUDY + * @param {String} study - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. + * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. + * @param {String} [params.creationDate] - Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + * @param {String} [params.modificationDate] - Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + * @param {String} [params.id] - Note unique identifier. + * @param {String} [params.uuid] - Unique 32-character identifier assigned automatically by OpenCGA. + * @param {String} [params.userId] - User that wrote that Note. + * @param {String} [params.tags] - Note tags. + * @param {String} [params.visibility] - Visibility of the Note. + * @param {String} [params.version] - Autoincremental version assigned to the registered entry. By default, updates does not create new + * versions. To enable versioning, users must set the `incVersion` flag from the /update web service when updating the document. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + searchNotes(study, params) { + return this._get("studies", study, "notes", null, "search", params); + } + + /** Delete note + * @param {String} study - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @param {String} id - Note unique identifier. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {Boolean} [params.includeResult = "false"] - Flag indicating to include the created or updated document result in the response. + * The default value is false. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + deleteNotes(study, id, params) { + return this._delete("studies", study, "notes", id, "delete", params); + } + + /** Update a note + * @param {String} study - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @param {String} id - Note unique identifier. + * @param {Object} data - JSON containing the Note fields to be updated. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. + * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. + * @param {"ADD SET REMOVE"} [params.tagsAction = "ADD"] - Action to be performed if the array of tags is being updated. The default + * value is ADD. + * @param {Boolean} [params.includeResult = "false"] - Flag indicating to include the created or updated document result in the response. + * The default value is false. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + updateNotes(study, id, data, params) { + return this._post("studies", study, "notes", id, "update", data, params); + } + /** Fetch permission rules - * @param {String} study - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} study - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {"SAMPLES FILES COHORTS INDIVIDUALS FAMILIES JOBS CLINICAL_ANALYSES DISEASE_PANELS"} entity - Entity where the permission rules * should be applied to. * @returns {Promise} Promise object in the form of RestResponse instance. @@ -197,7 +240,7 @@ export default class Study extends OpenCGAParentClass { } /** Add or remove a permission rule - * @param {String} study - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} study - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Object} data - JSON containing the permission rule to be created or removed. * @param {"SAMPLES FILES COHORTS INDIVIDUALS FAMILIES JOBS CLINICAL_ANALYSES DISEASE_PANELS"} entity - Entity where the permission rules * should be applied to. @@ -213,7 +256,7 @@ export default class Study extends OpenCGAParentClass { } /** Execute template - * @param {String} study - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} study - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Object} data - Template loader parameters. * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not @@ -221,6 +264,10 @@ export default class Study extends OpenCGAParentClass { * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runTemplates(study, data, params) { @@ -228,17 +275,17 @@ export default class Study extends OpenCGAParentClass { } /** Resource to upload a zipped template - * @param {String} study - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} study - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Object} [params] - The Object containing the following optional parameters: * @param {InputStream} [params.file] - File to upload. * @returns {Promise} Promise object in the form of RestResponse instance. */ uploadTemplates(study, params) { - return this._post("studies", study, "templates", null, "upload", params); + return this._post("studies", study, "templates", null, "upload", null, params); } /** Delete template - * @param {String} [study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} study - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} templateId - Template id. * @returns {Promise} Promise object in the form of RestResponse instance. */ @@ -247,7 +294,7 @@ export default class Study extends OpenCGAParentClass { } /** Update some study attributes - * @param {String} study - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} study - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Object} data - JSON containing the params to be updated. * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. @@ -261,7 +308,7 @@ export default class Study extends OpenCGAParentClass { } /** Fetch variableSets from a study - * @param {String} study - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} study - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.id] - Id of the variableSet to be retrieved. If no id is passed, it will show all the variableSets of the * study. @@ -272,7 +319,7 @@ export default class Study extends OpenCGAParentClass { } /** Add or remove a variableSet - * @param {String} study - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} study - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Object} data - JSON containing the VariableSet to be created or removed. * @param {Object} [params] - The Object containing the following optional parameters: * @param {"ADD REMOVE FORCE_REMOVE"} [params.action = "ADD"] - Action to be performed: ADD, REMOVE or FORCE_REMOVE a variableSet. The @@ -284,7 +331,7 @@ export default class Study extends OpenCGAParentClass { } /** Add or remove variables to a VariableSet - * @param {String} study - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} study - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} variableSet - VariableSet id of the VariableSet to be updated. * @param {Object} data - JSON containing the variable to be added or removed. For removing, only the variable id will be needed. * @param {Object} [params] - The Object containing the following optional parameters: diff --git a/opencga-client/src/main/javascript/User.js b/opencga-client/src/main/javascript/User.js index 8775caa0071..32b28182c79 100644 --- a/opencga-client/src/main/javascript/User.js +++ b/opencga-client/src/main/javascript/User.js @@ -1,9 +1,12 @@ /** - * Copyright 2015-2020 OpenCB + * Copyright 2015-2024 OpenCB + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -12,7 +15,6 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-04-25 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -32,6 +34,22 @@ export default class User extends OpenCGAParentClass { super(config); } + /** Get an anonymous token to gain access to the system + * @param {String} organization - Organization id. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + anonymous(organization) { + return this._post("users", null, null, null, "anonymous", null, organization); + } + + /** Create a new user + * @param {Object} data - JSON containing the parameters. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + create(data) { + return this._post("users", null, null, null, "create", data); + } + /** Get identified and gain access to the system * @param {Object} [data] - JSON containing the authentication parameters. * @returns {Promise} Promise object in the form of RestResponse instance. @@ -48,11 +66,30 @@ export default class User extends OpenCGAParentClass { return this._post("users", null, null, null, "password", data); } + /** User search method + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. + * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. + * @param {Number} [params.limit] - Number of results to be returned. + * @param {Number} [params.skip] - Number of results to skip. + * @param {Boolean} [params.count = "false"] - Get the total number of results matching the query. Deactivated by default. The default + * value is false. + * @param {String} [params.organization] - Organization id. + * @param {String} [params.id] - Comma separated list user IDs up to a maximum of 100. Also admits basic regular expressions using the + * operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + * @param {String} [params.authenticationId] - Authentication origin ID. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + search(params) { + return this._get("users", null, null, null, "search", params); + } + /** Return the user information including its projects and studies * @param {String} users - Comma separated list of user IDs. * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. + * @param {String} [params.organization] - Organization id. * @returns {Promise} Promise object in the form of RestResponse instance. */ info(users, params) { @@ -120,19 +157,6 @@ export default class User extends OpenCGAParentClass { return this._get("users", user, "password", null, "reset"); } - /** Retrieve the projects of the user - * @param {String} user - User ID. - * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. - * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. - * @param {Number} [params.limit] - Number of results to be returned. - * @param {Number} [params.skip] - Number of results to skip. - * @returns {Promise} Promise object in the form of RestResponse instance. - */ - projects(user, params) { - return this._get("users", user, null, null, "projects", params); - } - /** Update some user attributes * @param {String} user - User ID. * @param {Object} data - JSON containing the params to be updated. diff --git a/opencga-client/src/main/javascript/Variant.js b/opencga-client/src/main/javascript/Variant.js index ecfa45cd988..7d8b01966da 100644 --- a/opencga-client/src/main/javascript/Variant.js +++ b/opencga-client/src/main/javascript/Variant.js @@ -1,9 +1,12 @@ /** - * Copyright 2015-2020 OpenCB + * Copyright 2015-2024 OpenCB + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -12,7 +15,6 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-04-25 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -39,9 +41,9 @@ export default class Variant extends OpenCGAParentClass { * e.g.: 2,3:100000-200000. * @param {String} [params.type] - List of types, accepted values are SNV, MNV, INDEL, SV, COPY_NUMBER, COPY_NUMBER_LOSS, * COPY_NUMBER_GAIN, INSERTION, DELETION, DUPLICATION, TANDEM_DUPLICATION, BREAKEND, e.g. SNV,INDEL. - * @param {String} [params.project] - Project [user@]project where project can be either the ID or the alias. + * @param {String} [params.project] - Project [organization@]project where project can be either the ID or the alias. * @param {String} [params.study] - Filter variants from the given studies, these can be either the numeric ID or the alias with the - * format user@project:study. + * format organization@project:study. * @param {String} [params.cohort] - Select variants with calculated stats for the selected cohorts. * @param {String} [params.cohortStatsRef] - Reference Allele Frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL<=0.4. * @param {String} [params.cohortStatsAlt] - Alternate Allele Frequency: [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL<=0.4. @@ -94,7 +96,7 @@ export default class Variant extends OpenCGAParentClass { /** Read variant annotations metadata from any saved versions * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.annotationId] - Annotation identifier. - * @param {String} [params.project] - Project [user@]project where project can be either the ID or the alias. + * @param {String} [params.project] - Project [organization@]project where project can be either the ID or the alias. * @returns {Promise} Promise object in the form of RestResponse instance. */ metadataAnnotation(params) { @@ -152,12 +154,16 @@ export default class Variant extends OpenCGAParentClass { /** Compute cohort variant stats for the selected list of samples. * @param {Object} data - Cohort variant stats params. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not * provided. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runCohortStats(data, params) { @@ -173,6 +179,10 @@ export default class Variant extends OpenCGAParentClass { * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runExomiser(data, params) { @@ -184,13 +194,17 @@ export default class Variant extends OpenCGAParentClass { * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. - * @param {String} [params.project] - Project [user@]project where project can be either the ID or the alias. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.project] - Project [organization@]project where project can be either the ID or the alias. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not * provided. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runExport(data, params) { @@ -201,7 +215,7 @@ export default class Variant extends OpenCGAParentClass { * @param {"AUTOSOMAL_DOMINANT AUTOSOMAL_RECESSIVE X_LINKED_DOMINANT X_LINKED_RECESSIVE Y_LINKED MITOCHONDRIAL DE_NOVO MENDELIAN_ERROR * COMPOUND_HETEROZYGOUS UNKNOWN"} modeOfInheritance = "MONOALLELIC" - Mode of inheritance. The default value is MONOALLELIC. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.family] - Family id. * @param {String} [params.clinicalAnalysis] - Clinical analysis id. * @param {"COMPLETE INCOMPLETE UNKNOWN"} [params.penetrance = "COMPLETE"] - Penetrance. The default value is COMPLETE. @@ -216,12 +230,16 @@ export default class Variant extends OpenCGAParentClass { * @param {Object} data - Family QC analysis params. Family ID. Relatedness method, by default 'PLINK/IBD'. Minor allele frequence (MAF) * is used to filter variants before computing relatedness, e.g.: 1000G:CEU>0.35 or cohort:ALL>0.05. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not * provided. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runFamilyQc(data, params) { @@ -235,7 +253,11 @@ export default class Variant extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.file] - Files to remove. * @param {Boolean} [params.resume] - Resume a previously failed indexation. * @returns {Promise} Promise object in the form of RestResponse instance. @@ -253,6 +275,10 @@ export default class Variant extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runGatk(data, params) { @@ -264,12 +290,16 @@ export default class Variant extends OpenCGAParentClass { * (i.e., the number of points to display), the general query and the list of tracks. Currently, the supported track types are: COPY- * NUMBER, INDEL, REARRANGEMENT and SNV. In addition, each track can contain a specific query. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not * provided. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runGenomePlot(data, params) { @@ -279,12 +309,16 @@ export default class Variant extends OpenCGAParentClass { /** Run a Genome Wide Association Study between two cohorts. * @param {Object} data - Gwas analysis params. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not * provided. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runGwas(data, params) { @@ -294,12 +328,16 @@ export default class Variant extends OpenCGAParentClass { /** Run HRDetect analysis for a given somatic sample. * @param {Object} data - HRDetect analysis parameters. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not * provided. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runHrDetect(data, params) { @@ -309,12 +347,16 @@ export default class Variant extends OpenCGAParentClass { /** [DEPRECATED] Use operation/variant/index * @param {Object} data - Variant index params. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not * provided. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runIndex(data, params) { @@ -324,12 +366,16 @@ export default class Variant extends OpenCGAParentClass { /** Run quality control (QC) for a given individual. It includes inferred sex and mendelian errors (UDP) * @param {Object} data - Individual QC analysis params. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not * provided. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runIndividualQc(data, params) { @@ -339,12 +385,16 @@ export default class Variant extends OpenCGAParentClass { /** Infer sex from chromosome mean coverages. * @param {Object} data - Inferred sex analysis params. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not * provided. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runInferredSex(data, params) { @@ -384,6 +434,10 @@ export default class Variant extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runKnockout(data, params) { @@ -393,12 +447,16 @@ export default class Variant extends OpenCGAParentClass { /** Run mendelian error analysis to infer uniparental disomy regions. * @param {Object} data - Mendelian error analysis params. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not * provided. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runMendelianError(data, params) { @@ -407,9 +465,9 @@ export default class Variant extends OpenCGAParentClass { /** * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.project] - Project [user@]project where project can be either the ID or the alias. + * @param {String} [params.project] - Project [organization@]project where project can be either the ID or the alias. * @param {String} [params.study] - Filter variants from the given studies, these can be either the numeric ID or the alias with the - * format user@project:study. + * format organization@project:study. * @param {String} [params.file] - Filter variants from the files specified. This will set includeFile parameter when not provided. * @param {String} [params.sample] - Filter variants by sample genotype. This will automatically set 'includeSample' parameter when not * provided. This filter accepts multiple 3 forms: 1) List of samples: Samples that contain the main variant. Accepts AND (;) and OR (,) @@ -437,7 +495,7 @@ export default class Variant extends OpenCGAParentClass { /** Run mutational signature analysis for a given sample. Use context index. * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.study] - Filter variants from the given studies, these can be either the numeric ID or the alias with the - * format user@project:study. + * format organization@project:study. * @param {String} [params.sample] - Sample name. * @param {String} [params.type] - Variant type. Valid values: SNV, SV. * @param {String} [params.ct] - List of SO consequence types, e.g. missense_variant,stop_lost or SO:0001583,SO:0001578. Accepts aliases @@ -478,12 +536,16 @@ export default class Variant extends OpenCGAParentClass { * @param {Object} data - Mutational signature analysis parameters to index the genome context for that sample, and to compute both * catalogue counts and signature fitting. In order to skip one of them, , use the following keywords: , catalogue, fitting. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not * provided. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runMutationalSignature(data, params) { @@ -499,6 +561,10 @@ export default class Variant extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runPlink(data, params) { @@ -525,9 +591,9 @@ export default class Variant extends OpenCGAParentClass { * COPY_NUMBER_GAIN, INSERTION, DELETION, DUPLICATION, TANDEM_DUPLICATION, BREAKEND, e.g. SNV,INDEL. * @param {String} [params.reference] - Reference allele. * @param {String} [params.alternate] - Main alternate allele. - * @param {String} [params.project] - Project [user@]project where project can be either the ID or the alias. + * @param {String} [params.project] - Project [organization@]project where project can be either the ID or the alias. * @param {String} [params.study] - Filter variants from the given studies, these can be either the numeric ID or the alias with the - * format user@project:study. + * format organization@project:study. * @param {String} [params.file] - Filter variants from the files specified. This will set includeFile parameter when not provided. * @param {String} [params.filter] - Specify the FILTER for any of the files. If 'file' filter is provided, will match the file and the * filter. e.g.: PASS,LowGQX. @@ -640,12 +706,16 @@ export default class Variant extends OpenCGAParentClass { /** Compute a score to quantify relatedness between samples. * @param {Object} data - Relatedness analysis params. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not * provided. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runRelatedness(data, params) { @@ -661,6 +731,10 @@ export default class Variant extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runRvtests(data, params) { @@ -674,9 +748,9 @@ export default class Variant extends OpenCGAParentClass { * e.g.: 2,3:100000-200000. * @param {String} [params.type] - List of types, accepted values are SNV, MNV, INDEL, SV, COPY_NUMBER, COPY_NUMBER_LOSS, * COPY_NUMBER_GAIN, INSERTION, DELETION, DUPLICATION, TANDEM_DUPLICATION, BREAKEND, e.g. SNV,INDEL. - * @param {String} [params.project] - Project [user@]project where project can be either the ID or the alias. + * @param {String} [params.project] - Project [organization@]project where project can be either the ID or the alias. * @param {String} [params.study] - Filter variants from the given studies, these can be either the numeric ID or the alias with the - * format user@project:study. + * format organization@project:study. * @param {String} [params.file] - Filter variants from the files specified. This will set includeFile parameter when not provided. * @param {String} [params.filter] - Specify the FILTER for any of the files. If 'file' filter is provided, will match the file and the * filter. e.g.: PASS,LowGQX. @@ -725,12 +799,16 @@ export default class Variant extends OpenCGAParentClass { /** Filter samples by a complex query involving metadata and variants data * @param {Object} data - . * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not * provided. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runSampleEligibility(data, params) { @@ -743,12 +821,16 @@ export default class Variant extends OpenCGAParentClass { * order to skip some metrics, use the following keywords (separated by commas): variant-stats, signature, signature-catalogue, * signature-fitting, genome-plot. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not * provided. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runSampleQc(data, params) { @@ -771,12 +853,16 @@ export default class Variant extends OpenCGAParentClass { /** Get samples given a set of variants * @param {Object} data - Sample variant filter params. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not * provided. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runSample(data, params) { @@ -790,7 +876,7 @@ export default class Variant extends OpenCGAParentClass { * e.g.: 2,3:100000-200000. * @param {String} [params.type] - List of types, accepted values are SNV, MNV, INDEL, SV, COPY_NUMBER, COPY_NUMBER_LOSS, * COPY_NUMBER_GAIN, INSERTION, DELETION, DUPLICATION, TANDEM_DUPLICATION, BREAKEND, e.g. SNV,INDEL. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.file] - Filter variants from the files specified. This will set includeFile parameter when not provided. * @param {String} [params.filter] - Specify the FILTER for any of the files. If 'file' filter is provided, will match the file and the * filter. e.g.: PASS,LowGQX. @@ -818,12 +904,16 @@ export default class Variant extends OpenCGAParentClass { * @param {Object} data - Sample variant stats params. Use index=true and indexId='' to store the result in catalog sample QC. * indexId=ALL requires an empty query. Use sample=all to compute sample stats of all samples in the variant storage. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not * provided. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runSampleStats(data, params) { @@ -833,13 +923,17 @@ export default class Variant extends OpenCGAParentClass { /** Export calculated variant stats and frequencies * @param {Object} data - Variant stats export params. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.project] - Project [user@]project where project can be either the ID or the alias. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.project] - Project [organization@]project where project can be either the ID or the alias. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not * provided. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runStatsExport(data, params) { @@ -849,12 +943,16 @@ export default class Variant extends OpenCGAParentClass { /** Compute variant stats for any cohort and any set of variants. * @param {Object} data - Variant stats params. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not * provided. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runStats(data, params) { diff --git a/opencga-client/src/main/javascript/VariantOperation.js b/opencga-client/src/main/javascript/VariantOperation.js index 89b31d6f794..feb3959ec70 100644 --- a/opencga-client/src/main/javascript/VariantOperation.js +++ b/opencga-client/src/main/javascript/VariantOperation.js @@ -1,9 +1,12 @@ /** - * Copyright 2015-2020 OpenCB + * Copyright 2015-2024 OpenCB + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -12,7 +15,6 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2024-04-25 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -35,7 +37,7 @@ export default class VariantOperation extends OpenCGAParentClass { /** Update Cellbase configuration * @param {Object} [data] - New cellbase configuration. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.project] - Project [user@]project where project can be either the ID or the alias. + * @param {String} [params.project] - Project [organization@]project where project can be either the ID or the alias. * @param {Boolean} [params.annotationUpdate] - Create and load variant annotations into the database. * @param {String} [params.annotationSaveId] - Save a copy of the current variant annotation at the database. * @returns {Promise} Promise object in the form of RestResponse instance. @@ -52,7 +54,11 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ aggregateVariant(data, params) { @@ -66,7 +72,11 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. - * @param {String} [params.project] - Project [user@]project where project can be either the ID or the alias. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * @param {String} [params.project] - Project [organization@]project where project can be either the ID or the alias. * @param {String} [params.annotationId] - Annotation identifier. * @returns {Promise} Promise object in the form of RestResponse instance. */ @@ -82,8 +92,12 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. - * @param {String} [params.project] - Project [user@]project where project can be either the ID or the alias. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * @param {String} [params.project] - Project [organization@]project where project can be either the ID or the alias. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ indexVariantAnnotation(data, params) { @@ -98,7 +112,11 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. - * @param {String} [params.project] - Project [user@]project where project can be either the ID or the alias. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * @param {String} [params.project] - Project [organization@]project where project can be either the ID or the alias. * @returns {Promise} Promise object in the form of RestResponse instance. */ saveVariantAnnotation(data, params) { @@ -108,8 +126,8 @@ export default class VariantOperation extends OpenCGAParentClass { /** Update Variant Storage Engine configuration. Can be updated at Project or Study level * @param {Object} [data] - Configuration params to update. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.project] - Project [user@]project where project can be either the ID or the alias. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.project] - Project [organization@]project where project can be either the ID or the alias. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ configureVariant(data, params) { @@ -124,7 +142,11 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ deleteVariant(data, params) { @@ -139,7 +161,11 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ aggregateVariantFamily(data, params) { @@ -154,7 +180,11 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ indexVariantFamily(data, params) { @@ -169,7 +199,11 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ indexVariant(data, params) { @@ -184,7 +218,11 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ launcherVariantIndex(data, params) { @@ -199,6 +237,10 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @param {String} [params.project] - project. * @returns {Promise} Promise object in the form of RestResponse instance. */ @@ -214,6 +256,10 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ repairVariantMetadata(data, params) { @@ -228,7 +274,11 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ synchronizeVariantMetadata(data, params) { @@ -243,6 +293,10 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ pruneVariant(data, params) { @@ -257,7 +311,11 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ deleteVariantSample(data, params) { @@ -272,7 +330,11 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ indexVariantSample(data, params) { @@ -282,7 +344,7 @@ export default class VariantOperation extends OpenCGAParentClass { /** DEPRECATED You should use the new sample index configure method. * @param {Object} [data] - New SampleIndexConfiguration. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Boolean} [params.skipRebuild] - Skip sample index re-build. * @returns {Promise} Promise object in the form of RestResponse instance. */ @@ -297,7 +359,11 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.name] - Unique name of the score within the study. * @param {Boolean} [params.resume] - Resume a previously failed remove. * @param {Boolean} [params.force] - Force remove of partially indexed scores. @@ -322,7 +388,11 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ indexVariantScore(data, params) { @@ -337,8 +407,12 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. - * @param {String} [params.project] - Project [user@]project where project can be either the ID or the alias. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * @param {String} [params.project] - Project [organization@]project where project can be either the ID or the alias. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ variantSecondaryAnnotationIndex(data, params) { @@ -353,7 +427,11 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ variantSecondarySampleIndex(data, params) { @@ -363,7 +441,7 @@ export default class VariantOperation extends OpenCGAParentClass { /** Update SampleIndex configuration (New!) * @param {Object} [data] - New SampleIndexConfiguration. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {Boolean} [params.skipRebuild] - Skip sample index re-build. * @returns {Promise} Promise object in the form of RestResponse instance. */ @@ -379,8 +457,12 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. - * @param {String} [params.project] - Project [user@]project where project can be either the ID or the alias. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * @param {String} [params.project] - Project [organization@]project where project can be either the ID or the alias. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ secondaryIndexVariant(data, params) { @@ -394,7 +476,11 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.samples] - Samples to remove. Needs to provide all the samples in the secondary index. * @returns {Promise} Promise object in the form of RestResponse instance. */ @@ -402,15 +488,29 @@ export default class VariantOperation extends OpenCGAParentClass { return this._delete("operation", null, "variant/secondaryIndex", null, "delete", params); } + /** Execute Variant Setup to allow using the variant engine. This setup is necessary before starting any variant operation. + * @param {Object} [data] - Variant setup params. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + setupVariant(data, params) { + return this._post("operation", null, "variant", null, "setup", data, params); + } + /** Deletes the VariantStats of a cohort/s from the database * @param {Object} data - Variant stats delete params. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not * provided. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ deleteVariantStats(data, params) { @@ -420,12 +520,16 @@ export default class VariantOperation extends OpenCGAParentClass { /** Compute variant stats for any cohort and any set of variants and index the result in the variant storage database. * @param {Object} data - Variant stats params. * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not * provided. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ indexVariantStats(data, params) { @@ -440,7 +544,11 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. - * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ deleteVariantStudy(data, params) { diff --git a/opencga-client/src/main/python/README.rst b/opencga-client/src/main/python/README.rst index d8c3f51b64c..43b39e1c983 100644 --- a/opencga-client/src/main/python/README.rst +++ b/opencga-client/src/main/python/README.rst @@ -4,34 +4,28 @@ PyOpenCGA ========== This Python client package makes use of the comprehensive RESTful web services API implemented for the `OpenCGA`_ platform. -OpenCGA is an open-source project that implements a high-performance, scalable and secure platform for Genomic data analysis and visualisation +OpenCGA is an open-source project that implements a high-performance, scalable and secure platform for Genomic data analysis and visualisation. OpenCGA implements a secure and high performance platform for Big Data analysis and visualisation in current genomics. -OpenCGA uses the most modern and advanced technologies to scale to petabytes of data. OpenCGA is designed and implemented to work with -few million genomes. It is built on top of three main components: Catalog, Variant and Alignment Storage and Analysis. +OpenCGA uses the most modern and advanced technologies to scale to petabytes of data. OpenCGA is designed and implemented to work with few million genomes. It is built on top of three main components: Catalog, Variant and Alignment Storage and Analysis. -More info about this project in the `OpenCGA Docs`_ +More info about this project in `OpenCGA Docs`_ Installation ------------ -Cloning -``````` -PyOpenCGA can be cloned in your local machine by executing in your terminal:: +PyOpenCGA can be installed from the Pypi repository. Make sure you have pip available in your machine. You can check this by running:: - $ git clone https://github.com/opencb/opencga.git + $ python3 -m pip --version -Once you have downloaded the project you can install the library. We recommend to install it inside a `virtual environment`_:: - $ cd opencga/tree/develop/opencga-client/src/main/python/pyOpenCGA - $ python setup.py install +If you don't have Python or pip, please refer to https://packaging.python.org/en/latest/tutorials/installing-packages/ -Pip install -``````````` -Run the following command in the shell:: +To install PyOpencga, run the following command in the shell:: $ pip install pyopencga + Usage ----- @@ -48,14 +42,14 @@ The first step is to import the ClientConfiguration and OpenCGAClient from pyOpe Setting up server host configuration ```````````````````````````````````` -The second step is to generate a ClientConfiguration instance by passing a configuration dictionary containing the host to point to or a client-configuration.yml file: +The second step is to generate a ClientConfiguration instance by passing a configuration dictionary containing the opencga host OR a client-configuration.yml file with that information: .. code-block:: python >>> config = ClientConfiguration('/opt/opencga/conf/client-configuration.yml') >>> config = ClientConfiguration({ "rest": { - "host": "http://bioinfo.hpc.cam.ac.uk/opencga-demo" + "host": "https://demo.app.zettagenomics.com/opencga" } }) @@ -67,33 +61,22 @@ With this configuration you can initialize the OpenCGAClient, and log in: .. code-block:: python >>> oc = OpenCGAClient(config) - >>> oc.login('user') - -For scripting or using Jupyter Notebooks is preferable to load user credentials from an external JSON file. - -Once you are logged in, it is mandatory to use the token of the session to propagate the access of the clients to the host server: - -.. code-block:: python - - >>> token = oc.token - >>> print(token) - eyJhbGciOi... - - >>> oc = OpenCGAClient(configuration=config_dict, token=token) + >>> oc.login(user='user', password='pass', organization='organization') Examples ```````` -The next step is to get an instance of the clients we may want to use: +The first step is to get an instance of the clients we may want to use: .. code-block:: python - >>> projects = oc.projects # Project client - >>> studies = oc.studies # Study client - >>> samples = oc.samples # Sample client - >>> cohorts = oc.cohorts # Cohort client + >>> projects = oc.projects # Project client + >>> studies = oc.studies # Study client + >>> samples = oc.samples # Sample client + >>> individuals = oc.individuals # Individual client + >>> cohorts = oc.cohorts # Cohort client -Now you can start asking to the OpenCGA RESTful service with pyOpenCGA: +Now you can start querying with pyOpenCGA: .. code-block:: python @@ -103,41 +86,38 @@ Now you can start asking to the OpenCGA RESTful service with pyOpenCGA: project2 [...] -There are two different ways to access to the query response data: +There are two different ways to access query response data: .. code-block:: python - >>> foo_client.method().get_results() # Iterates over all the results of all the QueryResults - >>> foo_client.method().get_responses() # Iterates over all the responses + >>> foo_client.method().get_responses() # Iterates over all the responses + >>> foo_client.method().get_results() # Iterates over all the results of the first response -Data can be accessed specifying comma-separated IDs or a list of IDs: +Data can be accessed specifying comma-separated IDs or a list of IDs. -.. code-block:: python +e.g. Retrieving individual karyotypic sex for a list of individuals: - >>> samples = 'NA12877,NA12878,NA12879' - >>> samples_list = ['NA12877','NA12878','NA12879'] - >>> sc = oc.samples +.. code-block:: python - >>> for result in sc.info(query_id=samples, study='user@project1:study1').get_results(): - ... print(result['id'], result['attributes']['OPENCGA_INDIVIDUAL']['disorders']) - NA12877 [{'id': 'OMIM6500', 'name': "Chron's Disease"}] - NA12878 [] - NA12879 [{'id': 'OMIM6500', 'name': "Chron's Disease"}] + >>> for result in oc.samples.info(samples='NA12877,NA12878,NA12889', study='platinum').get_results(): + ... print(result['id'], result['karyotypicSex']) + NA12877 XY + NA12878 XX + NA12889 XY - >>> for result in sc.info(query_id=samples_list, study='user@project1:study1').get_results(): - ... print(result['id'], result['attributes']['OPENCGA_INDIVIDUAL']['disorders']) - NA12877 [{'id': 'OMIM6500', 'name': "Chron's Disease"}] - NA12878 [] - NA12879 [{'id': 'OMIM6500', 'name': "Chron's Disease"}] + >>> for result in oc.samples.info(samples=['NA12877', 'NA12878', 'NA12889'], study='platinum').get_results(): + ... print(result['id'], result['karyotypicSex']) + NA12877 XY + NA12878 XX + NA12889 XY Optional filters and extra options can be added as key-value parameters (where the values can be a comma-separated string or a list). What can I ask for? ``````````````````` -The best way to know which data can be retrieved for each client check `OpenCGA web services`_ swagger. - +The best way to know which data can be retrieved for each client, log into `OpenCGA Demo`_ and check the **OpenCGA REST API** in the **About** section (at the top right corner of the screen). .. _OpenCGA: https://github.com/opencb/opencga .. _OpenCGA Docs: http://docs.opencb.org/display/opencga -.. _virtual environment: https://help.dreamhost.com/hc/en-us/articles/115000695551-Installing-and-using-virtualenv-with-Python-3 -.. _OpenCGA web services: http://bioinfodev.hpc.cam.ac.uk/opencga/webservices/ +.. _OpenCGA REST API: https://demo.app.zettagenomics.com/ +.. _OpenCGA Demo: https://demo.app.zettagenomics.com/ diff --git a/opencga-client/src/main/python/pyopencga/__init__.py b/opencga-client/src/main/python/pyopencga/__init__.py index d781d658402..e69de29bb2d 100644 --- a/opencga-client/src/main/python/pyopencga/__init__.py +++ b/opencga-client/src/main/python/pyopencga/__init__.py @@ -1,2 +0,0 @@ -"refactoring from from fork pyCGA 7f2e3e404 branch release-1.4.0" -__author__="dev_team@mgviz.org" \ No newline at end of file diff --git a/opencga-client/src/main/python/pyopencga/opencga_client.py b/opencga-client/src/main/python/pyopencga/opencga_client.py index 0d2937ca638..4c5baf4506e 100644 --- a/opencga-client/src/main/python/pyopencga/opencga_client.py +++ b/opencga-client/src/main/python/pyopencga/opencga_client.py @@ -2,11 +2,15 @@ import time import sys import re +if sys.version_info >= (3, 8): + from importlib.metadata import version +else: + from importlib_metadata import version from pyopencga.opencga_config import ClientConfiguration from pyopencga.rest_clients.admin_client import Admin from pyopencga.rest_clients.alignment_client import Alignment -from pyopencga.rest_clients.clinical_client import Clinical +from pyopencga.rest_clients.clinical_analysis_client import ClinicalAnalysis from pyopencga.rest_clients.cohort_client import Cohort from pyopencga.rest_clients.family_client import Family from pyopencga.rest_clients.file_client import File @@ -21,6 +25,7 @@ from pyopencga.rest_clients.variant_operation_client import VariantOperation from pyopencga.rest_clients.user_client import User from pyopencga.rest_clients.variant_client import Variant +from pyopencga.rest_clients.organization_client import Organization class OpencgaClient(object): @@ -50,8 +55,10 @@ def __exit__(self, exc_type, exc_val, exc_tb): self.logout() def _check_versions(self): + # Getting client and server versions + client_version = version("pyopencga") server_version = self.meta.about().get_result(0)['Version'].split('-')[0] - client_version = re.findall(r'Client version: (.+)\n', str(self.meta.__doc__))[0] + ansi_reset = "\033[0m" ansi_red = "\033[31m" ansi_yellow = "\033[33m" @@ -60,10 +67,12 @@ def _check_versions(self): ' Some client features may not be implemented in the server.\n'.format(client_version, server_version) sys.stdout.write('{}{}{}'.format(ansi_red, msg, ansi_reset)) elif tuple(server_version.split('.')[:2]) > tuple(client_version.split('.')[:2]): - msg = '[INFO]: Client version ({}) is lower than server version ({}).\n'.format(client_version, server_version) + msg = '[INFO]: Client version ({}) is lower than server version ({}).' \ + ' Some client features may not work as intended.\n'.format(client_version, server_version) sys.stdout.write('{}{}{}'.format(ansi_yellow, msg, ansi_reset)) def _create_clients(self): + self.organizations = Organization(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh) self.users = User(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh) self.projects = Project(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh) self.studies = Study(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh) @@ -76,7 +85,7 @@ def _create_clients(self): self.disease_panels = DiseasePanel(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh) self.alignments = Alignment(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh) self.variants = Variant(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh) - self.clinical = Clinical(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh) + self.clinical = ClinicalAnalysis(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh) self.operations = VariantOperation(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh) self.variant_operations = self.operations # DEPRECATED: use 'self.operations' self.meta = Meta(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh) @@ -84,7 +93,7 @@ def _create_clients(self): self.admin = Admin(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh) self.clients = [ - self.users, self.projects, self.studies, self.files, self.jobs, + self.organizations, self.users, self.projects, self.studies, self.files, self.jobs, self.samples, self.individuals, self.families, self.cohorts, self.disease_panels, self.alignments, self.variants, self.clinical, self.variant_operations, self.meta, self.ga4gh, self.admin @@ -93,7 +102,7 @@ def _create_clients(self): for client in self.clients: client.on_retry = self.on_retry - def _make_login_handler(self, user, password): + def _make_login_handler(self, user, password, organization): """ Returns a closure that performs the log-in. This will be called on retries if the current session ever expires. @@ -109,13 +118,15 @@ def login_handler(refresh=False): data = {'user': user, 'password': password} else: data = {'user': user, 'password': password} + if organization: + data.update({'organization': organization}) tokens = User(self.configuration).login(data=data).get_result(0) self.token = tokens['token'] self.refresh_token = tokens['refreshToken'] return self.token return login_handler - def login(self, user=None, password=None): + def login(self, user=None, password=None, organization=None): if user is not None: if password is None: password = getpass.getpass() @@ -125,7 +136,7 @@ def login(self, user=None, password=None): except AssertionError: raise ValueError("User and password required") - self._login_handler = self._make_login_handler(user, password) + self._login_handler = self._make_login_handler(user, password, organization) self._login_handler() for client in self.clients: client.token = self.token @@ -174,15 +185,14 @@ def _get_help_info(self, client_name=None, parameters=False): # Description and path class_docstring = client.__doc__ - cls_desc = re.findall('(.+)\n +Client version', class_docstring)[0] - cls_desc = cls_desc.strip().replace('This class contains methods', 'Client') + cls_desc = re.findall('(This class contains methods .+)\n', class_docstring)[0] cls_path = re.findall('PATH: (.+)\n', class_docstring)[0] # Methods methods = [] method_names = [method_name for method_name in dir(client) if callable(getattr(client, method_name)) - and not method_name.startswith('_')] + and not method_name.startswith('_') and method_name != 'login_handler'] for method_name in method_names: if client_name is None: continue @@ -251,6 +261,9 @@ def help(self, client_name=None, show_parameters=False): )] sys.stdout.write('\n'.join(help_txt) + '\n') + def get_organization_client(self): + return self.organizations + def get_user_client(self): return self.users diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/admin_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/admin_client.py index cf56a794019..7311fa7e5dd 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/admin_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/admin_client.py @@ -2,7 +2,6 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-04-25 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +13,6 @@ class Admin(_ParentRestClient): """ This class contains methods for the 'Admin' webservices - Client version: 2.12.4-SNAPSHOT PATH: /{apiVersion}/admin """ @@ -27,9 +25,9 @@ def group_by_audit(self, fields, entity, **options): PATH: /{apiVersion}/admin/audit/groupBy :param str entity: Entity to be grouped by. Allowed values: ['AUDIT - USER PROJECT STUDY FILE SAMPLE JOB INDIVIDUAL COHORT DISEASE_PANEL - FAMILY CLINICAL_ANALYSIS INTERPRETATION VARIANT ALIGNMENT CLINICAL - EXPRESSION RGA FUNCTIONAL'] (REQUIRED) + NOTE ORGANIZATION USER PROJECT STUDY FILE SAMPLE JOB INDIVIDUAL + COHORT DISEASE_PANEL FAMILY CLINICAL_ANALYSIS INTERPRETATION + VARIANT ALIGNMENT CLINICAL EXPRESSION RGA FUNCTIONAL'] (REQUIRED) :param str fields: Comma separated list of fields by which to group by. (REQUIRED) :param bool count: Count the number of elements matching the group. @@ -45,18 +43,6 @@ def group_by_audit(self, fields, entity, **options): options['entity'] = entity return self._get(category='admin', resource='groupBy', subcategory='audit', **options) - def index_stats_catalog(self, **options): - """ - Sync Catalog into the Solr. - PATH: /{apiVersion}/admin/catalog/indexStats - - :param str collection: Collection to be indexed (file, sample, - individual, family, cohort and/or job). If not provided, all of - them will be indexed. - """ - - return self._post(category='admin', resource='indexStats', subcategory='catalog', **options) - def install_catalog(self, data=None, **options): """ Install OpenCGA database. @@ -73,6 +59,7 @@ def jwt_catalog(self, data=None, **options): PATH: /{apiVersion}/admin/catalog/jwt :param dict data: JSON containing the parameters. (REQUIRED) + :param str organization: Organization id. """ return self._post(category='admin', resource='jwt', subcategory='catalog', data=data, **options) @@ -93,10 +80,26 @@ def import_users(self, data=None, **options): PATH: /{apiVersion}/admin/users/import :param dict data: JSON containing the parameters. (REQUIRED) + :param str organization: Organization id. """ return self._post(category='admin', resource='import', subcategory='users', data=data, **options) + def permissions_users(self, **options): + """ + User permissions. + PATH: /{apiVersion}/admin/users/permissions + + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. + :param str entry_ids: Comma separated list of entry ids. + :param str permissions: Comma separated list of permissions to be + retrieved. + :param str category: Category corresponding to the id's provided. + """ + + return self._get(category='admin', resource='permissions', subcategory='users', **options) + def search_users(self, **options): """ User search method. @@ -110,8 +113,8 @@ def search_users(self, **options): :param int skip: Number of results to skip. :param bool count: Get the total number of results matching the query. Deactivated by default. + :param str organization: Organization id. :param str user: User ID. - :param str account: Account type [GUEST, FULL, ADMINISTRATOR]. :param str authentication_id: Authentication origin ID. """ @@ -124,6 +127,7 @@ def sync_users(self, data=None, **options): PATH: /{apiVersion}/admin/users/sync :param dict data: JSON containing the parameters. (REQUIRED) + :param str organization: Organization id. """ return self._post(category='admin', resource='sync', subcategory='users', data=data, **options) @@ -135,6 +139,7 @@ def users_update_groups(self, user, data=None, **options): :param dict data: JSON containing the parameters. (REQUIRED) :param str user: User ID. (REQUIRED) + :param str organization: Organization id. :param str action: Action to be performed: ADD or REMOVE user to/from groups. Allowed values: ['ADD REMOVE'] """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/alignment_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/alignment_client.py index c61107c6533..0374b6d4ecc 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/alignment_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/alignment_client.py @@ -2,7 +2,6 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-04-25 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +13,6 @@ class Alignment(_ParentRestClient): """ This class contains methods for the 'Analysis - Alignment' webservices - Client version: 2.12.4-SNAPSHOT PATH: /{apiVersion}/analysis/alignment """ @@ -35,6 +33,13 @@ def run_bwa(self, data=None, **options): the job will depend on. :param str job_description: Job description. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='alignment/bwa', data=data, **options) @@ -52,6 +57,13 @@ def run_coverage_index(self, data=None, **options): the job will depend on. :param str job_description: Job description. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='alignment/coverage/index', data=data, **options) @@ -71,6 +83,13 @@ def coverage_qc_gene_coverage_stats_run(self, data=None, **options): the job will depend on. :param str job_description: Job description. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='alignment/coverage/qc/geneCoverageStats', data=data, **options) @@ -81,8 +100,8 @@ def query_coverage(self, file, **options): PATH: /{apiVersion}/analysis/alignment/coverage/query :param str file: File ID. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str region: Comma separated list of regions 'chr:start-end, e.g.: 2,3:63500-65000. :param str gene: Comma separated list of genes, e.g.: BCRA2,TP53. @@ -111,8 +130,8 @@ def ratio_coverage(self, file1, file2, **options): :param str file2: Input file #2 (e.g. germline file). (REQUIRED) :param str file1: Input file #1 (e.g. somatic file). (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param bool skip_log2: Do not apply Log2 to normalise the coverage ratio. :param str region: Comma separated list of regions 'chr:start-end, @@ -140,8 +159,8 @@ def stats_coverage(self, file, gene, **options): :param str gene: Comma separated list of genes, e.g.: BCRA2,TP53. (REQUIRED) :param str file: File ID. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param int threshold: Only regions whose coverage depth is under this threshold will be reported. """ @@ -166,6 +185,13 @@ def run_deeptools(self, data=None, **options): the job will depend on. :param str job_description: Job description. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='alignment/deeptools', data=data, **options) @@ -183,6 +209,13 @@ def run_fastqc(self, data=None, **options): the job will depend on. :param str job_description: Job description. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='alignment/fastqc', data=data, **options) @@ -200,6 +233,13 @@ def run_index(self, data=None, **options): the job will depend on. :param str job_description: Job description. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='alignment/index', data=data, **options) @@ -221,6 +261,13 @@ def run_picard(self, data=None, **options): the job will depend on. :param str job_description: Job description. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='alignment/picard', data=data, **options) @@ -243,6 +290,13 @@ def run_qc(self, data=None, **options): the job will depend on. :param str job_description: Job description. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='alignment/qc', data=data, **options) @@ -257,8 +311,8 @@ def query(self, file, **options): :param int skip: Number of results to skip. :param bool count: Get the total number of results matching the query. Deactivated by default. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str region: Comma separated list of regions 'chr:start-end, e.g.: 2,3:63500-65000. :param str gene: Comma separated list of genes, e.g.: BCRA2,TP53. @@ -304,6 +358,13 @@ def run_samtools(self, data=None, **options): the job will depend on. :param str job_description: Job description. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='alignment/samtools', data=data, **options) diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py index eacad69006d..0a531515217 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py @@ -2,7 +2,6 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-04-25 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +13,6 @@ class ClinicalAnalysis(_ParentRestClient): """ This class contains methods for the 'Analysis - Clinical' webservices - Client version: 2.12.4-SNAPSHOT PATH: /{apiVersion}/analysis/clinical """ @@ -32,8 +30,8 @@ def update_acl(self, members, action, data=None, **options): Allowed values: ['SET ADD REMOVE RESET'] (REQUIRED) :param str members: Comma separated list of user or group IDs. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param bool propagate: Propagate permissions to related families, individuals, samples and files. """ @@ -49,8 +47,8 @@ def load_annotation_sets(self, variable_set_id, path, data=None, **options): :param str path: Path where the TSV file is located in OpenCGA or where it should be located. (REQUIRED) :param str variable_set_id: Variable set ID or name. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param bool parents: Flag indicating whether to create parent directories if they don't exist (only when TSV file was not previously associated). @@ -69,8 +67,8 @@ def update_clinical_configuration(self, data=None, **options): Update Clinical Analysis configuration. PATH: /{apiVersion}/analysis/clinical/clinical/configuration/update - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param dict data: Configuration params to update. """ @@ -87,8 +85,8 @@ def create(self, data=None, **options): must be provided. :param str exclude: Fields excluded in the response, whole JSON path must be provided. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param bool skip_create_default_interpretation: Flag to skip creating and initialise an empty default primary interpretation (Id will be '{clinicalAnalysisId}.1'). This flag is only considered if no @@ -106,8 +104,8 @@ def distinct(self, field, **options): :param str field: Comma separated list of fields for which to obtain the distinct values. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str id: Comma separated list of Clinical Analysis IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case @@ -143,6 +141,8 @@ def distinct(self, field, **options): :param str quality_control_summary: Clinical Analysis quality control summary. :param str release: Release when it was created. + :param int snapshot: Snapshot value (Latest version of the entry in + the specified release). :param str status: Filter by status. :param str internal_status: Filter by internal status. :param str annotation: Annotation filters. Example: @@ -161,14 +161,16 @@ def distinct_interpretation(self, field, **options): :param str field: Comma separated list of fields for which to obtain the distinct values. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str id: Comma separated list of Interpretation IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. :param str uuid: Comma separated list of Interpretation UUIDs up to a maximum of 100. + :param str name: Comma separated list of Interpretation names up to a + maximum of 100. :param str clinical_analysis_id: Clinical Analysis id. :param str analyst_id: Analyst ID. :param str method_name: Interpretation method name. Also admits basic @@ -202,14 +204,16 @@ def search_interpretation(self, **options): :param int limit: Number of results to be returned. :param int skip: Number of results to skip. :param bool sort: Sort the results. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str id: Comma separated list of Interpretation IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. :param str uuid: Comma separated list of Interpretation UUIDs up to a maximum of 100. + :param str name: Comma separated list of Interpretation names up to a + maximum of 100. :param str clinical_analysis_id: Clinical Analysis id. :param str analyst_id: Analyst ID. :param str method_name: Interpretation method name. Also admits basic @@ -241,8 +245,8 @@ def info_interpretation(self, interpretations, **options): must be provided. :param str exclude: Fields excluded in the response, whole JSON path must be provided. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str version: Comma separated list of interpretation versions. 'all' to get all the interpretation versions. Not supported if multiple interpretation ids are provided. @@ -258,14 +262,21 @@ def run_interpreter_cancer_tiering(self, data=None, **options): :param dict data: Cancer tiering interpretation analysis params. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str job_id: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. :param str job_description: Job description. :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='clinical/interpreter/cancerTiering', data=data, **options) @@ -275,15 +286,22 @@ def run_interpreter_exomiser(self, data=None, **options): Run exomiser interpretation analysis. PATH: /{apiVersion}/analysis/clinical/interpreter/exomiser/run - :param dict data: Exomizer interpretation analysis params. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param dict data: Exomiser interpretation analysis params. (REQUIRED) + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str job_id: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. :param str job_description: Job description. :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='clinical/interpreter/exomiser', data=data, **options) @@ -294,14 +312,21 @@ def run_interpreter_team(self, data=None, **options): PATH: /{apiVersion}/analysis/clinical/interpreter/team/run :param dict data: TEAM interpretation analysis params. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str job_id: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. :param str job_description: Job description. :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='clinical/interpreter/team', data=data, **options) @@ -312,14 +337,21 @@ def run_interpreter_tiering(self, data=None, **options): PATH: /{apiVersion}/analysis/clinical/interpreter/tiering/run :param dict data: Tiering interpretation analysis params. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str job_id: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. :param str job_description: Job description. :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='clinical/interpreter/tiering', data=data, **options) @@ -330,18 +362,51 @@ def run_interpreter_zetta(self, data=None, **options): PATH: /{apiVersion}/analysis/clinical/interpreter/zetta/run :param dict data: Zetta interpretation analysis params. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str job_id: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. :param str job_description: Job description. :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='clinical/interpreter/zetta', data=data, **options) + def load(self, data=None, **options): + """ + Load clinical analyses from a file. + PATH: /{apiVersion}/analysis/clinical/load + + :param dict data: Parameters to load clinical analysis in OpenCGA + catalog from a file. (REQUIRED) + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. + :param str job_id: Job ID. It must be a unique string within the + study. An ID will be autogenerated automatically if not provided. + :param str job_description: Job description. + :param str job_depends_on: Comma separated list of existing job IDs + the job will depend on. + :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. + """ + + return self._post(category='analysis', resource='load', subcategory='clinical', data=data, **options) + def aggregation_stats_rga(self, field, **options): """ RGA aggregation stats. @@ -372,8 +437,8 @@ def aggregation_stats_rga(self, field, **options): :param str clinical_significance: Filter by clinical significance. :param str population_frequency: Filter by population frequency. :param str consequence_type: Filter by consequence type. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. """ options['field'] = field @@ -417,8 +482,8 @@ def query_rga_gene(self, **options): :param str clinical_significance: Filter by clinical significance. :param str population_frequency: Filter by population frequency. :param str consequence_type: Filter by consequence type. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. """ return self._get(category='analysis', resource='query', subcategory='clinical/rga/gene', **options) @@ -452,8 +517,8 @@ def summary_rga_gene(self, **options): :param str clinical_significance: Filter by clinical significance. :param str population_frequency: Filter by population frequency. :param str consequence_type: Filter by consequence type. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. """ return self._get(category='analysis', resource='summary', subcategory='clinical/rga/gene', **options) @@ -464,14 +529,21 @@ def run_rga_index(self, data=None, **options): PATH: /{apiVersion}/analysis/clinical/rga/index/run :param dict data: Recessive Gene Analysis index params. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str job_id: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. :param str job_description: Job description. :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. :param bool auxiliar_index: Index auxiliar collection to improve performance assuming RGA is completely indexed. """ @@ -511,8 +583,8 @@ def query_rga_individual(self, **options): :param str clinical_significance: Filter by clinical significance. :param str population_frequency: Filter by population frequency. :param str consequence_type: Filter by consequence type. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. """ return self._get(category='analysis', resource='query', subcategory='clinical/rga/individual', **options) @@ -546,8 +618,8 @@ def summary_rga_individual(self, **options): :param str clinical_significance: Filter by clinical significance. :param str population_frequency: Filter by population frequency. :param str consequence_type: Filter by consequence type. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. """ return self._get(category='analysis', resource='summary', subcategory='clinical/rga/individual', **options) @@ -590,8 +662,8 @@ def query_rga_variant(self, **options): :param str clinical_significance: Filter by clinical significance. :param str population_frequency: Filter by population frequency. :param str consequence_type: Filter by consequence type. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. """ return self._get(category='analysis', resource='query', subcategory='clinical/rga/variant', **options) @@ -625,8 +697,8 @@ def summary_rga_variant(self, **options): :param str clinical_significance: Filter by clinical significance. :param str population_frequency: Filter by population frequency. :param str consequence_type: Filter by consequence type. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. """ return self._get(category='analysis', resource='summary', subcategory='clinical/rga/variant', **options) @@ -645,8 +717,8 @@ def search(self, **options): :param bool count: Get the total number of results matching the query. Deactivated by default. :param bool flatten_annotations: Flatten the annotations?. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str id: Comma separated list of Clinical Analysis IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case @@ -682,6 +754,8 @@ def search(self, **options): :param str quality_control_summary: Clinical Analysis quality control summary. :param str release: Release when it was created. + :param int snapshot: Snapshot value (Latest version of the entry in + the specified release). :param str status: Filter by status. :param str internal_status: Filter by internal status. :param str annotation: Annotation filters. Example: @@ -724,7 +798,7 @@ def query_variant(self, **options): SNV,INDEL. :param str study: Filter variants from the given studies, these can be either the numeric ID or the alias with the format - user@project:study. + organization@project:study. :param str file: Filter variants from the files specified. This will set includeFile parameter when not provided. :param str filter: Specify the FILTER for any of the files. If 'file' @@ -866,8 +940,8 @@ def acl(self, clinical_analyses, **options): :param str clinical_analyses: Comma separated list of clinical analysis IDs or names up to a maximum of 100. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str member: User or group ID. :param bool silent: Boolean to retrieve all possible entries that are queried for, false to raise an exception whenever one of the @@ -883,8 +957,8 @@ def delete(self, clinical_analyses, **options): :param str clinical_analyses: Comma separated list of clinical analysis IDs or names up to a maximum of 100. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param bool force: Force deletion if the ClinicalAnalysis contains interpretations or is locked. """ @@ -904,8 +978,8 @@ def update(self, clinical_analyses, data=None, **options): must be provided. :param str exclude: Fields excluded in the response, whole JSON path must be provided. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str comments_action: Action to be performed if the array of comments is being updated. Allowed values: ['ADD REMOVE REPLACE'] :param str flags_action: Action to be performed if the array of flags @@ -932,8 +1006,8 @@ def update_annotation_sets_annotations(self, clinical_analysis, annotation_set, :param str annotation_set: AnnotationSet ID to be updated. (REQUIRED) :param str clinical_analysis: Clinical analysis ID. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str action: Action to be performed: ADD to add new annotations; REPLACE to replace the value of an already existing annotation; SET to set the new list of annotations removing any possible old @@ -963,8 +1037,11 @@ def info(self, clinical_analysis, **options): :param str exclude: Fields excluded in the response, whole JSON path must be provided. :param bool flatten_annotations: Flatten the annotations?. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. + :param str version: Comma separated list of clinical versions. 'all' + to get all the clinical versions. Not supported if multiple + clinical ids are provided. :param bool deleted: Boolean to retrieve deleted entries. """ @@ -982,7 +1059,7 @@ def create_interpretation(self, clinical_analysis, data=None, **options): must be provided. :param str exclude: Fields excluded in the response, whole JSON path must be provided. - :param str study: [[user@]project:]study id. + :param str study: [[organization@]project:]study id. :param str set_as: Set interpretation as. Allowed values: ['PRIMARY SECONDARY'] :param bool include_result: Flag indicating to include the created or @@ -999,7 +1076,7 @@ def clear_interpretation(self, clinical_analysis, interpretations, **options): :param str clinical_analysis: Clinical analysis ID. (REQUIRED) :param str interpretations: Interpretation IDs of the Clinical Analysis. (REQUIRED) - :param str study: [[user@]project:]study ID. + :param str study: [[organization@]project:]study ID. """ return self._post(category='analysis/clinical', resource='clear', query_id=clinical_analysis, subcategory='interpretation', second_query_id=interpretations, **options) @@ -1012,7 +1089,7 @@ def delete_interpretation(self, clinical_analysis, interpretations, **options): :param str interpretations: Interpretation IDs of the Clinical Analysis. (REQUIRED) :param str clinical_analysis: Clinical analysis ID. (REQUIRED) - :param str study: [[user@]project:]study ID. + :param str study: [[organization@]project:]study ID. :param str set_as_primary: Interpretation id to set as primary from the list of secondaries in case of deleting the actual primary one. """ @@ -1027,7 +1104,7 @@ def revert_interpretation(self, clinical_analysis, interpretation, version, **op :param int version: Version to revert to. (REQUIRED) :param str interpretation: Interpretation ID. (REQUIRED) :param str clinical_analysis: Clinical analysis ID. (REQUIRED) - :param str study: [[user@]project:]study ID. + :param str study: [[organization@]project:]study ID. """ options['version'] = version @@ -1046,7 +1123,7 @@ def update_interpretation(self, clinical_analysis, interpretation, data=None, ** must be provided. :param str exclude: Fields excluded in the response, whole JSON path must be provided. - :param str study: [[user@]project:]study ID. + :param str study: [[organization@]project:]study ID. :param str primary_findings_action: Action to be performed if the array of primary findings is being updated. Allowed values: ['ADD SET REMOVE REPLACE'] @@ -1081,8 +1158,8 @@ def update_report(self, clinical_analysis, data=None, **options): must be provided. :param str exclude: Fields excluded in the response, whole JSON path must be provided. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str comments_action: Action to be performed if the array of comments is being updated. Allowed values: ['ADD REMOVE REPLACE'] :param str supporting_evidences_action: Action to be performed if the diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/clinical_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/clinical_client.py deleted file mode 100644 index 3c42c2e0a16..00000000000 --- a/opencga-client/src/main/python/pyopencga/rest_clients/clinical_client.py +++ /dev/null @@ -1,1005 +0,0 @@ -""" -WARNING: AUTOGENERATED CODE - - This code was generated by a tool. - Autogenerated on: 2022-08-02 08:25:32 ->>>>>>> develop ->>>>>>> release-2.4.x ->>>>>>> release-2.4.x - - Manual changes to this file may cause unexpected behavior in your application. - Manual changes to this file will be overwritten if the code is regenerated. -""" - -from pyopencga.rest_clients._parent_rest_clients import _ParentRestClient - - -class Clinical(_ParentRestClient): - """ - This class contains methods for the 'Analysis - Clinical' webservices - Client version: 2.3.3-SNAPSHOT [fb5e7ed17448243b10ddd619bdf973e4b20b0a74] ->>>>>>> develop ->>>>>>> release-2.4.x ->>>>>>> release-2.4.x - PATH: /{apiVersion}/analysis/clinical - """ - - def __init__(self, configuration, token=None, login_handler=None, *args, **kwargs): - super(Clinical, self).__init__(configuration, token, login_handler, *args, **kwargs) - - def update_acl(self, members, action, data=None, **options): - """ - Update the set of permissions granted for the member. - PATH: /{apiVersion}/analysis/clinical/acl/{members}/update - - :param dict data: JSON containing the parameters to add ACLs. - (REQUIRED) - :param str action: Action to be performed [ADD, SET, REMOVE or RESET]. - (REQUIRED) - :param str members: Comma separated list of user or group IDs. - (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. - :param bool propagate: Propagate permissions to related families, - individuals, samples and files. - """ - - options['action'] = action - return self._post(category='analysis', resource='update', subcategory='clinical/acl', second_query_id=members, data=data, **options) - - def update_clinical_configuration(self, data=None, **options): - """ - Update Clinical Analysis configuration. - PATH: /{apiVersion}/analysis/clinical/clinical/configuration/update - - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. - :param dict data: Configuration params to update. - """ - - return self._post(category='analysis', resource='update', subcategory='clinical/clinical/configuration', data=data, **options) - - def create(self, data=None, **options): - """ - Create a new clinical analysis. - PATH: /{apiVersion}/analysis/clinical/create - - :param dict data: JSON containing clinical analysis information. - (REQUIRED) - :param str include: Fields included in the response, whole JSON path - must be provided. - :param str exclude: Fields excluded in the response, whole JSON path - must be provided. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. - :param bool skip_create_default_interpretation: Flag to skip creating - and initialise an empty default primary interpretation (Id will be - '{clinicalAnalysisId}.1'). This flag is only considered if no - Interpretation object is passed. - :param bool include_result: Flag indicating to include the created or - updated document result in the response. - """ - - return self._post(category='analysis', resource='create', subcategory='clinical', data=data, **options) - - def distinct(self, field, **options): - """ - Clinical Analysis distinct method. - PATH: /{apiVersion}/analysis/clinical/distinct - - :param str field: Field for which to obtain the distinct values. - (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. - :param str id: Comma separated list of Clinical Analysis IDs up to a - maximum of 100. - :param str uuid: Comma separated list of Clinical Analysis UUIDs up to - a maximum of 100. - :param str type: Clinical Analysis type. - :param str disorder: Clinical Analysis disorder. - :param str files: Clinical Analysis files. - :param str sample: Sample associated to the proband or any member of a - family. - :param str individual: Proband or any member of a family. - :param str proband: Clinical Analysis proband. - :param str proband_samples: Clinical Analysis proband samples. - :param str family: Clinical Analysis family. - :param str family_members: Clinical Analysis family members. - :param str family_member_samples: Clinical Analysis family members - samples. - :param str panels: Clinical Analysis panels. - :param bool locked: Locked Clinical Analyses. - :param str analyst_id: Clinical Analysis analyst id. - :param str priority: Clinical Analysis priority. - :param str flags: Clinical Analysis flags. - :param str creation_date: Clinical Analysis Creation date. Format: - yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. - :param str modification_date: Clinical Analysis Modification date. - Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. - :param str due_date: Clinical Analysis due date. Format: - yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. - :param str quality_control_summary: Clinical Analysis quality control - summary. - :param str release: Release when it was created. - :param str status: Filter by status. - :param str internal_status: Filter by internal status. - :param bool deleted: Boolean to retrieve deleted entries. - """ - - options['field'] = field - return self._get(category='analysis', resource='distinct', subcategory='clinical', **options) - - def distinct_interpretation(self, field, **options): - """ - Interpretation distinct method. - PATH: /{apiVersion}/analysis/clinical/interpretation/distinct - - :param str field: Field for which to obtain the distinct values. - (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. - :param str id: Comma separated list of Interpretation IDs up to a - maximum of 100. - :param str uuid: Comma separated list of Interpretation UUIDs up to a - maximum of 100. - :param str clinical_analysis_id: Clinical Analysis id. - :param str analyst_id: Analyst ID. - :param str method_name: Interpretation method name. - :param str panels: Interpretation panels. - :param str primary_findings: Interpretation primary findings. - :param str secondary_findings: Interpretation secondary findings. - :param str creation_date: Interpretation Creation date. Format: - yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. - :param str modification_date: Interpretation Modification date. - Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. - :param str status: Filter by status. - :param str internal_status: Filter by internal status. - :param str release: Release when it was created. - """ - - options['field'] = field - return self._get(category='analysis', resource='distinct', subcategory='clinical/interpretation', **options) - - def search_interpretation(self, **options): - """ - Search clinical interpretations. - PATH: /{apiVersion}/analysis/clinical/interpretation/search - - :param str include: Fields included in the response, whole JSON path - must be provided. - :param str exclude: Fields excluded in the response, whole JSON path - must be provided. - :param int limit: Number of results to be returned. - :param int skip: Number of results to skip. - :param bool sort: Sort the results. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. - :param str id: Comma separated list of Interpretation IDs up to a - maximum of 100. - :param str uuid: Comma separated list of Interpretation UUIDs up to a - maximum of 100. - :param str clinical_analysis_id: Clinical Analysis id. - :param str analyst_id: Analyst ID. - :param str method_name: Interpretation method name. - :param str panels: Interpretation panels. - :param str primary_findings: Interpretation primary findings. - :param str secondary_findings: Interpretation secondary findings. - :param str creation_date: Interpretation Creation date. Format: - yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. - :param str modification_date: Interpretation Modification date. - Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. - :param str status: Filter by status. - :param str internal_status: Filter by internal status. - :param str release: Release when it was created. - """ - - return self._get(category='analysis', resource='search', subcategory='clinical/interpretation', **options) - - def info_interpretation(self, interpretations, **options): - """ - Clinical interpretation information. - PATH: /{apiVersion}/analysis/clinical/interpretation/{interpretations}/info - - :param str interpretations: Comma separated list of clinical - interpretation IDs up to a maximum of 100. (REQUIRED) - :param str include: Fields included in the response, whole JSON path - must be provided. - :param str exclude: Fields excluded in the response, whole JSON path - must be provided. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. - :param str version: Comma separated list of interpretation versions. - 'all' to get all the interpretation versions. Not supported if - multiple interpretation ids are provided. - :param bool deleted: Boolean to retrieve deleted entries. - """ - - return self._get(category='analysis', resource='info', subcategory='clinical/interpretation', second_query_id=interpretations, **options) - - def run_interpreter_cancer_tiering(self, data=None, **options): - """ - Run cancer tiering interpretation analysis. - PATH: /{apiVersion}/analysis/clinical/interpreter/cancerTiering/run - - :param dict data: Cancer tiering interpretation analysis params. - (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. - :param str job_id: Job ID. It must be a unique string within the - study. An ID will be autogenerated automatically if not provided. - :param str job_description: Job description. - :param str job_depends_on: Comma separated list of existing job IDs - the job will depend on. - :param str job_tags: Job tags. - """ - - return self._post(category='analysis', resource='run', subcategory='clinical/interpreter/cancerTiering', data=data, **options) - - def run_interpreter_exomiser(self, data=None, **options): - """ - Run exomiser interpretation analysis. - PATH: /{apiVersion}/analysis/clinical/interpreter/exomiser/run - - :param dict data: Exomizer interpretation analysis params. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. - :param str job_id: Job ID. It must be a unique string within the - study. An ID will be autogenerated automatically if not provided. - :param str job_description: Job description. - :param str job_depends_on: Comma separated list of existing job IDs - the job will depend on. - :param str job_tags: Job tags. - """ - - return self._post(category='analysis', resource='run', subcategory='clinical/interpreter/exomiser', data=data, **options) - - def run_interpreter_team(self, data=None, **options): - """ - Run TEAM interpretation analysis. - PATH: /{apiVersion}/analysis/clinical/interpreter/team/run - - :param dict data: TEAM interpretation analysis params. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. - :param str job_id: Job ID. It must be a unique string within the - study. An ID will be autogenerated automatically if not provided. - :param str job_description: Job description. - :param str job_depends_on: Comma separated list of existing job IDs - the job will depend on. - :param str job_tags: Job tags. - """ - - return self._post(category='analysis', resource='run', subcategory='clinical/interpreter/team', data=data, **options) - - def run_interpreter_tiering(self, data=None, **options): - """ - Run tiering interpretation analysis. - PATH: /{apiVersion}/analysis/clinical/interpreter/tiering/run - - :param dict data: Tiering interpretation analysis params. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. - :param str job_id: Job ID. It must be a unique string within the - study. An ID will be autogenerated automatically if not provided. - :param str job_description: Job description. - :param str job_depends_on: Comma separated list of existing job IDs - the job will depend on. - :param str job_tags: Job tags. - """ - - return self._post(category='analysis', resource='run', subcategory='clinical/interpreter/tiering', data=data, **options) - - def run_interpreter_zetta(self, data=None, **options): - """ - Run Zetta interpretation analysis. - PATH: /{apiVersion}/analysis/clinical/interpreter/zetta/run - - :param dict data: Zetta interpretation analysis params. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. - :param str job_id: Job ID. It must be a unique string within the - study. An ID will be autogenerated automatically if not provided. - :param str job_description: Job description. - :param str job_depends_on: Comma separated list of existing job IDs - the job will depend on. - :param str job_tags: Job tags. - """ - - return self._post(category='analysis', resource='run', subcategory='clinical/interpreter/zetta', data=data, **options) - - def aggregation_stats_rga(self, field, **options): - """ - RGA aggregation stats. - PATH: /{apiVersion}/analysis/clinical/rga/aggregationStats - - :param str field: List of fields separated by semicolons, e.g.: - clinicalSignificances;type. For nested fields use >>, e.g.: - type>>clinicalSignificances;knockoutType. (REQUIRED) - :param int limit: Number of results to be returned. - :param int skip: Number of results to skip. - :param str sample_id: Filter by sample id. - :param str individual_id: Filter by individual id. - :param str sex: Filter by sex. - :param str phenotypes: Filter by phenotypes. - :param str disorders: Filter by disorders. - :param str num_parents: Filter by the number of parents registered. - :param str gene_id: Filter by gene id. - :param str gene_name: Filter by gene name. - :param str chromosome: Filter by chromosome. - :param str start: Filter by start position. - :param str end: Filter by end position. - :param str transcript_id: Filter by transcript id. - :param str variants: Filter by variant id. - :param str db_snps: Filter by DB_SNP id. - :param str knockout_type: Filter by knockout type. - :param str filter: Filter by filter (PASS, NOT_PASS). - :param str type: Filter by variant type. - :param str clinical_significance: Filter by clinical significance. - :param str population_frequency: Filter by population frequency. - :param str consequence_type: Filter by consequence type. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. - """ - - options['field'] = field - return self._get(category='analysis', resource='aggregationStats', subcategory='clinical/rga', **options) - - def query_rga_gene(self, **options): - """ - Query gene RGA. - PATH: /{apiVersion}/analysis/clinical/rga/gene/query - - :param str include: Fields included in the response, whole JSON path - must be provided. - :param str exclude: Fields excluded in the response, whole JSON path - must be provided. - :param int limit: Number of results to be returned. - :param int skip: Number of results to skip. - :param bool count: Get the total number of results matching the query. - Deactivated by default. - :param str include_individual: Include only the comma separated list - of individuals to the response. - :param int skip_individual: Number of individuals to skip. - :param int limit_individual: Limit number of individuals returned - (default: 1000). - :param str sample_id: Filter by sample id. - :param str individual_id: Filter by individual id. - :param str sex: Filter by sex. - :param str phenotypes: Filter by phenotypes. - :param str disorders: Filter by disorders. - :param str num_parents: Filter by the number of parents registered. - :param str gene_id: Filter by gene id. - :param str gene_name: Filter by gene name. - :param str chromosome: Filter by chromosome. - :param str start: Filter by start position. - :param str end: Filter by end position. - :param str transcript_id: Filter by transcript id. - :param str variants: Filter by variant id. - :param str db_snps: Filter by DB_SNP id. - :param str knockout_type: Filter by knockout type. - :param str filter: Filter by filter (PASS, NOT_PASS). - :param str type: Filter by variant type. - :param str clinical_significance: Filter by clinical significance. - :param str population_frequency: Filter by population frequency. - :param str consequence_type: Filter by consequence type. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. - """ - - return self._get(category='analysis', resource='query', subcategory='clinical/rga/gene', **options) - - def summary_rga_gene(self, **options): - """ - RGA gene summary stats. - PATH: /{apiVersion}/analysis/clinical/rga/gene/summary - - :param int limit: Number of results to be returned. - :param int skip: Number of results to skip. - :param bool count: Get the total number of results matching the query. - Deactivated by default. - :param str sample_id: Filter by sample id. - :param str individual_id: Filter by individual id. - :param str sex: Filter by sex. - :param str phenotypes: Filter by phenotypes. - :param str disorders: Filter by disorders. - :param str num_parents: Filter by the number of parents registered. - :param str gene_id: Filter by gene id. - :param str gene_name: Filter by gene name. - :param str chromosome: Filter by chromosome. - :param str start: Filter by start position. - :param str end: Filter by end position. - :param str transcript_id: Filter by transcript id. - :param str variants: Filter by variant id. - :param str db_snps: Filter by DB_SNP id. - :param str knockout_type: Filter by knockout type. - :param str filter: Filter by filter (PASS, NOT_PASS). - :param str type: Filter by variant type. - :param str clinical_significance: Filter by clinical significance. - :param str population_frequency: Filter by population frequency. - :param str consequence_type: Filter by consequence type. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. - """ - - return self._get(category='analysis', resource='summary', subcategory='clinical/rga/gene', **options) - - def run_rga_index(self, data=None, **options): - """ - Generate Recessive Gene Analysis secondary index. - PATH: /{apiVersion}/analysis/clinical/rga/index/run - - :param dict data: Recessive Gene Analysis index params. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. - :param str job_id: Job ID. It must be a unique string within the - study. An ID will be autogenerated automatically if not provided. - :param str job_description: Job description. - :param str job_depends_on: Comma separated list of existing job IDs - the job will depend on. - :param str job_tags: Job tags. - :param bool auxiliar_index: Index auxiliar collection to improve - performance assuming RGA is completely indexed. - """ - - return self._post(category='analysis', resource='run', subcategory='clinical/rga/index', data=data, **options) - - def query_rga_individual(self, **options): - """ - Query individual RGA. - PATH: /{apiVersion}/analysis/clinical/rga/individual/query - - :param str include: Fields included in the response, whole JSON path - must be provided. - :param str exclude: Fields excluded in the response, whole JSON path - must be provided. - :param int limit: Number of results to be returned. - :param int skip: Number of results to skip. - :param bool count: Get the total number of results matching the query. - Deactivated by default. - :param str sample_id: Filter by sample id. - :param str individual_id: Filter by individual id. - :param str sex: Filter by sex. - :param str phenotypes: Filter by phenotypes. - :param str disorders: Filter by disorders. - :param str num_parents: Filter by the number of parents registered. - :param str gene_id: Filter by gene id. - :param str gene_name: Filter by gene name. - :param str chromosome: Filter by chromosome. - :param str start: Filter by start position. - :param str end: Filter by end position. - :param str transcript_id: Filter by transcript id. - :param str variants: Filter by variant id. - :param str db_snps: Filter by DB_SNP id. - :param str knockout_type: Filter by knockout type. - :param str filter: Filter by filter (PASS, NOT_PASS). - :param str type: Filter by variant type. - :param str clinical_significance: Filter by clinical significance. - :param str population_frequency: Filter by population frequency. - :param str consequence_type: Filter by consequence type. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. - """ - - return self._get(category='analysis', resource='query', subcategory='clinical/rga/individual', **options) - - def summary_rga_individual(self, **options): - """ - RGA individual summary stats. - PATH: /{apiVersion}/analysis/clinical/rga/individual/summary - - :param int limit: Number of results to be returned. - :param int skip: Number of results to skip. - :param bool count: Get the total number of results matching the query. - Deactivated by default. - :param str sample_id: Filter by sample id. - :param str individual_id: Filter by individual id. - :param str sex: Filter by sex. - :param str phenotypes: Filter by phenotypes. - :param str disorders: Filter by disorders. - :param str num_parents: Filter by the number of parents registered. - :param str gene_id: Filter by gene id. - :param str gene_name: Filter by gene name. - :param str chromosome: Filter by chromosome. - :param str start: Filter by start position. - :param str end: Filter by end position. - :param str transcript_id: Filter by transcript id. - :param str variants: Filter by variant id. - :param str db_snps: Filter by DB_SNP id. - :param str knockout_type: Filter by knockout type. - :param str filter: Filter by filter (PASS, NOT_PASS). - :param str type: Filter by variant type. - :param str clinical_significance: Filter by clinical significance. - :param str population_frequency: Filter by population frequency. - :param str consequence_type: Filter by consequence type. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. - """ - - return self._get(category='analysis', resource='summary', subcategory='clinical/rga/individual', **options) - - def query_rga_variant(self, **options): - """ - Query variant RGA. - PATH: /{apiVersion}/analysis/clinical/rga/variant/query - - :param str include: Fields included in the response, whole JSON path - must be provided. - :param str exclude: Fields excluded in the response, whole JSON path - must be provided. - :param int limit: Number of results to be returned. - :param int skip: Number of results to skip. - :param bool count: Get the total number of results matching the query. - Deactivated by default. - :param str include_individual: Include only the comma separated list - of individuals to the response. - :param int skip_individual: Number of individuals to skip. - :param int limit_individual: Limit number of individuals returned - (default: 1000). - :param str sample_id: Filter by sample id. - :param str individual_id: Filter by individual id. - :param str sex: Filter by sex. - :param str phenotypes: Filter by phenotypes. - :param str disorders: Filter by disorders. - :param str num_parents: Filter by the number of parents registered. - :param str gene_id: Filter by gene id. - :param str gene_name: Filter by gene name. - :param str chromosome: Filter by chromosome. - :param str start: Filter by start position. - :param str end: Filter by end position. - :param str transcript_id: Filter by transcript id. - :param str variants: Filter by variant id. - :param str db_snps: Filter by DB_SNP id. - :param str knockout_type: Filter by knockout type. - :param str filter: Filter by filter (PASS, NOT_PASS). - :param str type: Filter by variant type. - :param str clinical_significance: Filter by clinical significance. - :param str population_frequency: Filter by population frequency. - :param str consequence_type: Filter by consequence type. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. - """ - - return self._get(category='analysis', resource='query', subcategory='clinical/rga/variant', **options) - - def summary_rga_variant(self, **options): - """ - RGA variant summary stats. - PATH: /{apiVersion}/analysis/clinical/rga/variant/summary - - :param int limit: Number of results to be returned. - :param int skip: Number of results to skip. - :param bool count: Get the total number of results matching the query. - Deactivated by default. - :param str sample_id: Filter by sample id. - :param str individual_id: Filter by individual id. - :param str sex: Filter by sex. - :param str phenotypes: Filter by phenotypes. - :param str disorders: Filter by disorders. - :param str num_parents: Filter by the number of parents registered. - :param str gene_id: Filter by gene id. - :param str gene_name: Filter by gene name. - :param str chromosome: Filter by chromosome. - :param str start: Filter by start position. - :param str end: Filter by end position. - :param str transcript_id: Filter by transcript id. - :param str variants: Filter by variant id. - :param str db_snps: Filter by DB_SNP id. - :param str knockout_type: Filter by knockout type. - :param str filter: Filter by filter (PASS, NOT_PASS). - :param str type: Filter by variant type. - :param str clinical_significance: Filter by clinical significance. - :param str population_frequency: Filter by population frequency. - :param str consequence_type: Filter by consequence type. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. - """ - - return self._get(category='analysis', resource='summary', subcategory='clinical/rga/variant', **options) - - def search(self, **options): - """ - Clinical analysis search. - PATH: /{apiVersion}/analysis/clinical/search - - :param str include: Fields included in the response, whole JSON path - must be provided. - :param str exclude: Fields excluded in the response, whole JSON path - must be provided. - :param int limit: Number of results to be returned. - :param int skip: Number of results to skip. - :param bool count: Get the total number of results matching the query. - Deactivated by default. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. - :param str id: Comma separated list of Clinical Analysis IDs up to a - maximum of 100. - :param str uuid: Comma separated list of Clinical Analysis UUIDs up to - a maximum of 100. - :param str type: Clinical Analysis type. - :param str disorder: Clinical Analysis disorder. - :param str files: Clinical Analysis files. - :param str sample: Sample associated to the proband or any member of a - family. - :param str individual: Proband or any member of a family. - :param str proband: Clinical Analysis proband. - :param str proband_samples: Clinical Analysis proband samples. - :param str family: Clinical Analysis family. - :param str family_members: Clinical Analysis family members. - :param str family_member_samples: Clinical Analysis family members - samples. - :param str panels: Clinical Analysis panels. - :param bool locked: Locked Clinical Analyses. - :param str analyst_id: Clinical Analysis analyst id. - :param str priority: Clinical Analysis priority. - :param str flags: Clinical Analysis flags. - :param str creation_date: Clinical Analysis Creation date. Format: - yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. - :param str modification_date: Clinical Analysis Modification date. - Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. - :param str due_date: Clinical Analysis due date. Format: - yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. - :param str quality_control_summary: Clinical Analysis quality control - summary. - :param str release: Release when it was created. - :param str status: Filter by status. - :param str internal_status: Filter by internal status. - :param bool deleted: Boolean to retrieve deleted entries. - """ - - return self._get(category='analysis', resource='search', subcategory='clinical', **options) - - def actionable_variant(self, **options): - """ - Fetch actionable clinical variants. - PATH: /{apiVersion}/analysis/clinical/variant/actionable - - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. - :param str sample: Sample ID. - """ - - return self._get(category='analysis', resource='actionable', subcategory='clinical/variant', **options) - - def query_variant(self, **options): - """ - Fetch clinical variants. - PATH: /{apiVersion}/analysis/clinical/variant/query - - :param str include: Fields included in the response, whole JSON path - must be provided. - :param str exclude: Fields excluded in the response, whole JSON path - must be provided. - :param int limit: Number of results to be returned. - :param int skip: Number of results to skip. - :param bool count: Get the total number of results matching the query. - Deactivated by default. - :param bool approximate_count: Get an approximate count, instead of an - exact total count. Reduces execution time. - :param int approximate_count_sampling_size: Sampling size to get the - approximate count. Larger values increase accuracy but also - increase execution time. - :param str saved_filter: Use a saved filter at User level. - :param str id: List of IDs, these can be rs IDs (dbSNP) or variants in - the format chrom:start:ref:alt, e.g. rs116600158,19:7177679:C:T. - :param str region: List of regions, these can be just a single - chromosome name or regions in the format chr:start-end, e.g.: - 2,3:100000-200000. - :param str type: List of types, accepted values are SNV, MNV, INDEL, - SV, COPY_NUMBER, COPY_NUMBER_LOSS, COPY_NUMBER_GAIN, INSERTION, - DELETION, DUPLICATION, TANDEM_DUPLICATION, BREAKEND, e.g. - SNV,INDEL. - :param str study: Filter variants from the given studies, these can be - either the numeric ID or the alias with the format - user@project:study. - :param str file: Filter variants from the files specified. This will - set includeFile parameter when not provided. - :param str filter: Specify the FILTER for any of the files. If 'file' - filter is provided, will match the file and the filter. e.g.: - PASS,LowGQX. - :param str qual: Specify the QUAL for any of the files. If 'file' - filter is provided, will match the file and the qual. e.g.: >123.4. - :param str file_data: Filter by file data (i.e. FILTER, QUAL and INFO - columns from VCF file). [{file}:]{key}{op}{value}[,;]* . If no file - is specified, will use all files from 'file' filter. e.g. AN>200 or - file_1.vcf:AN>200;file_2.vcf:AN<10 . Many fields can be combined. - e.g. file_1.vcf:AN>200;DB=true;file_2.vcf:AN<10,FILTER=PASS,LowDP. - :param str sample: Filter variants by sample genotype. This will - automatically set 'includeSample' parameter when not provided. This - filter accepts multiple 3 forms: 1) List of samples: Samples that - contain the main variant. Accepts AND (;) and OR (,) operators. - e.g. HG0097,HG0098 . 2) List of samples with genotypes: - {sample}:{gt1},{gt2}. Accepts AND (;) and OR (,) operators. e.g. - HG0097:0/0;HG0098:0/1,1/1 . Unphased genotypes (e.g. 0/1, 1/1) will - also include phased genotypes (e.g. 0|1, 1|0, 1|1), but not vice - versa. When filtering by multi-allelic genotypes, any secondary - allele will match, regardless of its position e.g. 1/2 will match - with genotypes 1/2, 1/3, 1/4, .... Genotype aliases accepted: - HOM_REF, HOM_ALT, HET, HET_REF, HET_ALT, HET_MISS and MISS e.g. - HG0097:HOM_REF;HG0098:HET_REF,HOM_ALT . 3) Sample with segregation - mode: {sample}:{segregation}. Only one sample accepted.Accepted - segregation modes: [ autosomalDominant, autosomalRecessive, - XLinkedDominant, XLinkedRecessive, YLinked, mitochondrial, deNovo, - mendelianError, compoundHeterozygous ]. Value is case insensitive. - e.g. HG0097:DeNovo Sample must have parents defined and indexed. . - :param str sample_data: Filter by any SampleData field from samples. - [{sample}:]{key}{op}{value}[,;]* . If no sample is specified, will - use all samples from 'sample' or 'genotype' filter. e.g. DP>200 or - HG0097:DP>200,HG0098:DP<10 . Many FORMAT fields can be combined. - e.g. HG0097:DP>200;GT=1/1,0/1,HG0098:DP<10. - :param str sample_annotation: Selects some samples using metadata - information from Catalog. e.g. - age>20;phenotype=hpo:123,hpo:456;name=smith. - :param str cohort: Select variants with calculated stats for the - selected cohorts. - :param str cohort_stats_ref: Reference Allele Frequency: - [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL<=0.4. - :param str cohort_stats_alt: Alternate Allele Frequency: - [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL<=0.4. - :param str cohort_stats_maf: Minor Allele Frequency: - [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL<=0.4. - :param str cohort_stats_mgf: Minor Genotype Frequency: - [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL<=0.4. - :param str cohort_stats_pass: Filter PASS frequency: - [{study:}]{cohort}[<|>|<=|>=]{number}. e.g. ALL>0.8. - :param str missing_alleles: Number of missing alleles: - [{study:}]{cohort}[<|>|<=|>=]{number}. - :param str missing_genotypes: Number of missing genotypes: - [{study:}]{cohort}[<|>|<=|>=]{number}. - :param str score: Filter by variant score: - [{study:}]{score}[<|>|<=|>=]{number}. - :param str family: Filter variants where any of the samples from the - given family contains the variant (HET or HOM_ALT). - :param str family_disorder: Specify the disorder to use for the family - segregation. - :param str family_segregation: Filter by segregation mode from a given - family. Accepted values: [ autosomalDominant, autosomalRecessive, - XLinkedDominant, XLinkedRecessive, YLinked, mitochondrial, deNovo, - mendelianError, compoundHeterozygous ]. - :param str family_members: Sub set of the members of a given family. - :param str family_proband: Specify the proband child to use for the - family segregation. - :param str gene: List of genes, most gene IDs are accepted (HGNC, - Ensembl gene, ...). This is an alias to 'xref' parameter. - :param str ct: List of SO consequence types, e.g. - missense_variant,stop_lost or SO:0001583,SO:0001578. Accepts - aliases 'loss_of_function' and 'protein_altering'. - :param str xref: List of any external reference, these can be genes, - proteins or variants. Accepted IDs include HGNC, Ensembl genes, - dbSNP, ClinVar, HPO, Cosmic, ... - :param str biotype: List of biotypes, e.g. protein_coding. - :param str protein_substitution: Protein substitution scores include - SIFT and PolyPhen. You can query using the score - {protein_score}[<|>|<=|>=]{number} or the description - {protein_score}[~=|=]{description} e.g. polyphen>0.1,sift=tolerant. - :param str conservation: Filter by conservation score: - {conservation_score}[<|>|<=|>=]{number} e.g. - phastCons>0.5,phylop<0.1,gerp>0.1. - :param str population_frequency_alt: Alternate Population Frequency: - {study}:{population}[<|>|<=|>=]{number}. e.g. 1000G:ALL<0.01. - :param str population_frequency_ref: Reference Population Frequency: - {study}:{population}[<|>|<=|>=]{number}. e.g. 1000G:ALL<0.01. - :param str population_frequency_maf: Population minor allele - frequency: {study}:{population}[<|>|<=|>=]{number}. e.g. - 1000G:ALL<0.01. - :param str transcript_flag: List of transcript flags. e.g. canonical, - CCDS, basic, LRG, MANE Select, MANE Plus Clinical, EGLH_HaemOnc, - TSO500. - :param str gene_trait_id: List of gene trait association id. e.g. - 'umls:C0007222' , 'OMIM:269600'. - :param str go: List of GO (Gene Ontology) terms. e.g. 'GO:0002020'. - :param str expression: List of tissues of interest. e.g. 'lung'. - :param str protein_keyword: List of Uniprot protein variant annotation - keywords. - :param str drug: List of drug names. - :param str functional_score: Functional score: - {functional_score}[<|>|<=|>=]{number} e.g. cadd_scaled>5.2 , - cadd_raw<=0.3. - :param str clinical: Clinical source: clinvar, cosmic. - :param str clinical_significance: Clinical significance: benign, - likely_benign, likely_pathogenic, pathogenic. - :param bool clinical_confirmed_status: Clinical confirmed status. - :param str custom_annotation: Custom annotation: - {key}[<|>|<=|>=]{number} or {key}[~=|=]{text}. - :param str panel: Filter by genes from the given disease panel. - :param str panel_mode_of_inheritance: Filter genes from specific - panels that match certain mode of inheritance. Accepted values : [ - autosomalDominant, autosomalRecessive, XLinkedDominant, - XLinkedRecessive, YLinked, mitochondrial, deNovo, mendelianError, - compoundHeterozygous ]. - :param str panel_confidence: Filter genes from specific panels that - match certain confidence. Accepted values : [ high, medium, low, - rejected ]. - :param str panel_role_in_cancer: Filter genes from specific panels - that match certain role in cancer. Accepted values : [ both, - oncogene, tumorSuppressorGene, fusion ]. - :param str panel_feature_type: Filter elements from specific panels by - type. Accepted values : [ gene, region, str, variant ]. - :param bool panel_intersection: Intersect panel genes and regions with - given genes and regions from que input query. This will prevent - returning variants from regions out of the panel. - :param str trait: List of traits, based on ClinVar, HPO, COSMIC, i.e.: - IDs, histologies, descriptions,... - """ - - return self._get(category='analysis', resource='query', subcategory='clinical/variant', **options) - - def acl(self, clinical_analyses, **options): - """ - Returns the acl of the clinical analyses. If member is provided, it - will only return the acl for the member. - PATH: /{apiVersion}/analysis/clinical/{clinicalAnalyses}/acl - - :param str clinical_analyses: Comma separated list of clinical - analysis IDs or names up to a maximum of 100. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. - :param str member: User or group ID. - :param bool silent: Boolean to retrieve all possible entries that are - queried for, false to raise an exception whenever one of the - entries looked for cannot be shown for whichever reason. - """ - - return self._get(category='analysis', resource='acl', subcategory='clinical', second_query_id=clinical_analyses, **options) - - def delete(self, clinical_analyses, **options): - """ - Delete clinical analyses. - PATH: /{apiVersion}/analysis/clinical/{clinicalAnalyses}/delete - - :param str clinical_analyses: Comma separated list of clinical - analysis IDs or names up to a maximum of 100. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. - :param bool force: Force deletion if the ClinicalAnalysis contains - interpretations or is locked. - """ - - return self._delete(category='analysis', resource='delete', subcategory='clinical', second_query_id=clinical_analyses, **options) - - def update(self, clinical_analyses, data=None, **options): - """ - Update clinical analysis attributes. - PATH: /{apiVersion}/analysis/clinical/{clinicalAnalyses}/update - - :param dict data: JSON containing clinical analysis information. - (REQUIRED) - :param str clinical_analyses: Comma separated list of clinical - analysis IDs. (REQUIRED) - :param str include: Fields included in the response, whole JSON path - must be provided. - :param str exclude: Fields excluded in the response, whole JSON path - must be provided. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. - :param str comments_action: Action to be performed if the array of - comments is being updated. Allowed values: ['ADD', 'REMOVE', - 'REPLACE'] - :param str flags_action: Action to be performed if the array of flags - is being updated. Allowed values: ['ADD', 'SET', 'REMOVE'] - :param str files_action: Action to be performed if the array of files - is being updated. Allowed values: ['ADD', 'SET', 'REMOVE'] - :param str panels_action: Action to be performed if the array of - panels is being updated. Allowed values: ['ADD', 'SET', 'REMOVE'] - :param bool include_result: Flag indicating to include the created or - updated document result in the response. - """ - - return self._post(category='analysis', resource='update', subcategory='clinical', second_query_id=clinical_analyses, data=data, **options) - - def info(self, clinical_analysis, **options): - """ - Clinical analysis info. - PATH: /{apiVersion}/analysis/clinical/{clinicalAnalysis}/info - - :param str clinical_analysis: Comma separated list of clinical - analysis IDs or names up to a maximum of 100. (REQUIRED) - :param str include: Fields included in the response, whole JSON path - must be provided. - :param str exclude: Fields excluded in the response, whole JSON path - must be provided. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. - :param bool deleted: Boolean to retrieve deleted entries. - """ - - return self._get(category='analysis', resource='info', subcategory='clinical', second_query_id=clinical_analysis, **options) - - def create_interpretation(self, clinical_analysis, data=None, **options): - """ - Create a new Interpretation. - PATH: /{apiVersion}/analysis/clinical/{clinicalAnalysis}/interpretation/create - - :param dict data: JSON containing clinical interpretation information. - (REQUIRED) - :param str clinical_analysis: Clinical analysis ID. (REQUIRED) - :param str include: Fields included in the response, whole JSON path - must be provided. - :param str exclude: Fields excluded in the response, whole JSON path - must be provided. - :param str study: [[user@]project:]study id. - :param str set_as: Set interpretation as. Allowed values: ['PRIMARY', - 'SECONDARY'] - :param bool include_result: Flag indicating to include the created or - updated document result in the response. - """ - - return self._post(category='analysis/clinical', resource='create', query_id=clinical_analysis, subcategory='interpretation', data=data, **options) - - def clear_interpretation(self, clinical_analysis, interpretations, **options): - """ - Clear the fields of the main interpretation of the Clinical Analysis. - PATH: /{apiVersion}/analysis/clinical/{clinicalAnalysis}/interpretation/{interpretations}/clear - - :param str clinical_analysis: Clinical analysis ID. (REQUIRED) - :param str interpretations: Interpretation IDs of the Clinical - Analysis. (REQUIRED) - :param str study: [[user@]project:]study ID. - """ - - return self._post(category='analysis/clinical', resource='clear', query_id=clinical_analysis, subcategory='interpretation', second_query_id=interpretations, **options) - - def delete_interpretation(self, clinical_analysis, interpretations, **options): - """ - Delete interpretation. - PATH: /{apiVersion}/analysis/clinical/{clinicalAnalysis}/interpretation/{interpretations}/delete - - :param str interpretations: Interpretation IDs of the Clinical - Analysis. (REQUIRED) - :param str clinical_analysis: Clinical analysis ID. (REQUIRED) - :param str study: [[user@]project:]study ID. - :param str set_as_primary: Interpretation id to set as primary from - the list of secondaries in case of deleting the actual primary one. - """ - - return self._delete(category='analysis/clinical', resource='delete', query_id=clinical_analysis, subcategory='interpretation', second_query_id=interpretations, **options) - - def revert_interpretation(self, clinical_analysis, interpretation, version, **options): - """ - Revert to a previous interpretation version. - PATH: /{apiVersion}/analysis/clinical/{clinicalAnalysis}/interpretation/{interpretation}/revert - - :param int version: Version to revert to. (REQUIRED) - :param str interpretation: Interpretation ID. (REQUIRED) - :param str clinical_analysis: Clinical analysis ID. (REQUIRED) - :param str study: [[user@]project:]study ID. - """ - - options['version'] = version - return self._post(category='analysis/clinical', resource='revert', query_id=clinical_analysis, subcategory='interpretation', second_query_id=interpretation, **options) - - def update_interpretation(self, clinical_analysis, interpretation, data=None, **options): - """ - Update interpretation fields. - PATH: /{apiVersion}/analysis/clinical/{clinicalAnalysis}/interpretation/{interpretation}/update - - :param dict data: JSON containing clinical interpretation information. - (REQUIRED) - :param str interpretation: Interpretation ID. (REQUIRED) - :param str clinical_analysis: Clinical analysis ID. (REQUIRED) - :param str include: Fields included in the response, whole JSON path - must be provided. - :param str exclude: Fields excluded in the response, whole JSON path - must be provided. - :param str study: [[user@]project:]study ID. - :param str primary_findings_action: Action to be performed if the - array of primary findings is being updated. Allowed values: ['ADD', - 'SET', 'REMOVE', 'REPLACE'] - :param str methods_action: Action to be performed if the array of - methods is being updated. Allowed values: ['ADD', 'SET', 'REMOVE'] - :param str secondary_findings_action: Action to be performed if the - array of secondary findings is being updated. Allowed values: - ['ADD', 'SET', 'REMOVE', 'REPLACE'] - :param str comments_action: Action to be performed if the array of - comments is being updated. To REMOVE or REPLACE, the date will need - to be provided to identify the comment. Allowed values: ['ADD', - 'REMOVE', 'REPLACE'] - :param str panels_action: Action to be performed if the array of - panels is being updated. Allowed values: ['ADD', 'SET', 'REMOVE'] - :param str set_as: Set interpretation as. Allowed values: ['PRIMARY', - 'SECONDARY'] - :param bool include_result: Flag indicating to include the created or - updated document result in the response. - """ - - return self._post(category='analysis/clinical', resource='update', query_id=clinical_analysis, subcategory='interpretation', second_query_id=interpretation, data=data, **options) - diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/cohort_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/cohort_client.py index e4ecc0c8ece..f38109debed 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/cohort_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/cohort_client.py @@ -2,7 +2,6 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-04-25 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +13,6 @@ class Cohort(_ParentRestClient): """ This class contains methods for the 'Cohorts' webservices - Client version: 2.12.4-SNAPSHOT PATH: /{apiVersion}/cohorts """ @@ -32,40 +30,13 @@ def update_acl(self, members, action, data=None, **options): Allowed values: ['SET ADD REMOVE RESET'] (REQUIRED) :param str members: Comma separated list of user or group ids. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. """ options['action'] = action return self._post(category='cohorts', resource='update', subcategory='acl', second_query_id=members, data=data, **options) - def aggregation_stats(self, **options): - """ - Fetch catalog cohort stats. - PATH: /{apiVersion}/cohorts/aggregationStats - - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. - :param str type: Type. - :param str creation_year: Creation year. - :param str creation_month: Creation month (JANUARY, FEBRUARY...). - :param str creation_day: Creation day. - :param str creation_day_of_week: Creation day of week (MONDAY, - TUESDAY...). - :param str num_samples: Number of samples. - :param str status: Status. - :param str release: Release. - :param str annotation: Annotation filters. Example: - age>30;gender=FEMALE. For more information, please visit - http://docs.opencb.org/display/opencga/AnnotationSets+1.4.0. - :param bool default: Calculate default stats. - :param str field: List of fields separated by semicolons, e.g.: - studies;type. For nested fields use >>, e.g.: - studies>>biotype;type;numSamples[0..10]:1. - """ - - return self._get(category='cohorts', resource='aggregationStats', **options) - def load_annotation_sets(self, variable_set_id, path, data=None, **options): """ Load annotation sets from a TSV file. @@ -74,8 +45,8 @@ def load_annotation_sets(self, variable_set_id, path, data=None, **options): :param str path: Path where the TSV file is located in OpenCGA or where it should be located. (REQUIRED) :param str variable_set_id: Variable set ID or name. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param bool parents: Flag indicating whether to create parent directories if they don't exist (only when TSV file was not previously associated). @@ -99,8 +70,8 @@ def create(self, data=None, **options): must be provided. :param str exclude: Fields excluded in the response, whole JSON path must be provided. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str variable_set: Deprecated: Use /generate web service and filter by annotation. :param str variable: Deprecated: Use /generate web service and filter @@ -118,8 +89,8 @@ def distinct(self, field, **options): :param str field: Comma separated list of fields for which to obtain the distinct values. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str id: Comma separated list of cohort IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' @@ -156,8 +127,8 @@ def generate(self, data=None, **options): must be provided. :param str exclude: Fields excluded in the response, whole JSON path must be provided. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str id: Comma separated list sample IDs or UUIDs up to a maximum of 100. :param bool somatic: Somatic sample. @@ -204,8 +175,8 @@ def search(self, **options): :param bool count: Get the total number of results matching the query. Deactivated by default. :param bool flatten_annotations: Flatten the annotations?. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str id: Comma separated list of cohort IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' @@ -239,8 +210,8 @@ def acl(self, cohorts, **options): :param str cohorts: Comma separated list of cohort IDs or UUIDs up to a maximum of 100. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str member: User or group id. :param bool silent: Boolean to retrieve all possible entries that are queried for, false to raise an exception whenever one of the @@ -255,8 +226,8 @@ def delete(self, cohorts, **options): PATH: /{apiVersion}/cohorts/{cohorts}/delete :param str cohorts: Comma separated list of cohort ids. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. """ return self._delete(category='cohorts', resource='delete', query_id=cohorts, **options) @@ -273,8 +244,8 @@ def info(self, cohorts, **options): :param str exclude: Fields excluded in the response, whole JSON path must be provided. :param bool flatten_annotations: Flatten the annotations?. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param bool deleted: Boolean to retrieve deleted cohorts. """ @@ -290,8 +261,8 @@ def update(self, cohorts, data=None, **options): must be provided. :param str exclude: Fields excluded in the response, whole JSON path must be provided. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str samples_action: Action to be performed if the array of samples is being updated. Allowed values: ['ADD SET REMOVE'] :param str annotation_sets_action: Action to be performed if the array @@ -311,8 +282,8 @@ def update_annotation_sets_annotations(self, cohort, annotation_set, data=None, :param str annotation_set: AnnotationSet ID to be updated. (REQUIRED) :param str cohort: Cohort ID. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str action: Action to be performed: ADD to add new annotations; REPLACE to replace the value of an already existing annotation; SET to set the new list of annotations removing any possible old diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/disease_panel_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/disease_panel_client.py index 77d939aa32e..8634c983e2e 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/disease_panel_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/disease_panel_client.py @@ -2,7 +2,6 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-04-25 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +13,6 @@ class DiseasePanel(_ParentRestClient): """ This class contains methods for the 'Disease Panels' webservices - Client version: 2.12.4-SNAPSHOT PATH: /{apiVersion}/panels """ @@ -32,8 +30,8 @@ def update_acl(self, members, action, data=None, **options): Allowed values: ['SET ADD REMOVE RESET'] (REQUIRED) :param str members: Comma separated list of user or group ids. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. """ options['action'] = action @@ -48,8 +46,8 @@ def create(self, data=None, **options): must be provided. :param str exclude: Fields excluded in the response, whole JSON path must be provided. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param bool include_result: Flag indicating to include the created or updated document result in the response. :param dict data: Panel parameters. @@ -64,8 +62,8 @@ def distinct(self, field, **options): :param str field: Comma separated list of fields for which to obtain the distinct values. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str id: Comma separated list of panel IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' @@ -127,14 +125,21 @@ def import_panels(self, data=None, **options): Import panels. PATH: /{apiVersion}/panels/import - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str job_id: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_description: Job description. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. :param dict data: Panel parameters. """ @@ -153,8 +158,8 @@ def search(self, **options): :param int skip: Number of results to skip. :param bool count: Get the total number of results matching the query. Deactivated by default. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str id: Comma separated list of panel IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' @@ -218,8 +223,8 @@ def acl(self, panels, **options): :param str panels: Comma separated list of panel IDs up to a maximum of 100. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str member: User or group id. :param bool silent: Boolean to retrieve all possible entries that are queried for, false to raise an exception whenever one of the @@ -234,8 +239,8 @@ def delete(self, panels, **options): PATH: /{apiVersion}/panels/{panels}/delete :param str panels: Comma separated list of panel ids. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. """ return self._delete(category='panels', resource='delete', query_id=panels, **options) @@ -251,8 +256,8 @@ def info(self, panels, **options): must be provided. :param str exclude: Fields excluded in the response, whole JSON path must be provided. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str version: Comma separated list of panel versions. 'all' to get all the panel versions. Not supported if multiple panel ids are provided. @@ -271,8 +276,8 @@ def update(self, panels, data=None, **options): must be provided. :param str exclude: Fields excluded in the response, whole JSON path must be provided. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param bool include_result: Flag indicating to include the created or updated document result in the response. :param dict data: Panel parameters. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/family_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/family_client.py index 82d22a7a3f0..278c306bfe6 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/family_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/family_client.py @@ -2,7 +2,6 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-04-25 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +13,6 @@ class Family(_ParentRestClient): """ This class contains methods for the 'Families' webservices - Client version: 2.12.4-SNAPSHOT PATH: /{apiVersion}/families """ @@ -32,8 +30,8 @@ def update_acl(self, members, action, data=None, **options): Allowed values: ['SET ADD REMOVE RESET'] (REQUIRED) :param str members: Comma separated list of user or group ids. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str propagate: Propagate family permissions to related individuals and samples. Allowed values: ['NO YES YES_AND_VARIANT_VIEW'] @@ -42,35 +40,6 @@ def update_acl(self, members, action, data=None, **options): options['action'] = action return self._post(category='families', resource='update', subcategory='acl', second_query_id=members, data=data, **options) - def aggregation_stats(self, **options): - """ - Fetch catalog family stats. - PATH: /{apiVersion}/families/aggregationStats - - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. - :param str creation_year: Creation year. - :param str creation_month: Creation month (JANUARY, FEBRUARY...). - :param str creation_day: Creation day. - :param str creation_day_of_week: Creation day of week (MONDAY, - TUESDAY...). - :param str status: Status. - :param str phenotypes: Phenotypes. - :param str release: Release. - :param str version: Version. - :param str num_members: Number of members. - :param str expected_size: Expected size. - :param str annotation: Annotation filters. Example: - age>30;gender=FEMALE. For more information, please visit - http://docs.opencb.org/display/opencga/AnnotationSets+1.4.0. - :param bool default: Calculate default stats. - :param str field: List of fields separated by semicolons, e.g.: - studies;type. For nested fields use >>, e.g.: - studies>>biotype;type;numSamples[0..10]:1. - """ - - return self._get(category='families', resource='aggregationStats', **options) - def load_annotation_sets(self, variable_set_id, path, data=None, **options): """ Load annotation sets from a TSV file. @@ -79,8 +48,8 @@ def load_annotation_sets(self, variable_set_id, path, data=None, **options): :param str path: Path where the TSV file is located in OpenCGA or where it should be located. (REQUIRED) :param str variable_set_id: Variable set ID or name. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param bool parents: Flag indicating whether to create parent directories if they don't exist (only when TSV file was not previously associated). @@ -104,8 +73,8 @@ def create(self, data=None, **options): must be provided. :param str exclude: Fields excluded in the response, whole JSON path must be provided. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str members: Comma separated list of member ids to be associated to the created family. :param bool include_result: Flag indicating to include the created or @@ -121,8 +90,8 @@ def distinct(self, field, **options): :param str field: Comma separated list of fields for which to obtain the distinct values. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str id: Comma separated list family IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for @@ -182,8 +151,8 @@ def search(self, **options): :param bool count: Get the total number of results matching the query. Deactivated by default. :param bool flatten_annotations: Flatten the annotations?. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str id: Comma separated list family IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for @@ -236,8 +205,8 @@ def acl(self, families, **options): :param str families: Comma separated list of family IDs or names up to a maximum of 100. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str member: User or group id. :param bool silent: Boolean to retrieve all possible entries that are queried for, false to raise an exception whenever one of the @@ -252,8 +221,8 @@ def delete(self, families, **options): PATH: /{apiVersion}/families/{families}/delete :param str families: Comma separated list of family ids. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. """ return self._delete(category='families', resource='delete', query_id=families, **options) @@ -270,8 +239,8 @@ def info(self, families, **options): :param str exclude: Fields excluded in the response, whole JSON path must be provided. :param bool flatten_annotations: Flatten the annotations?. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str version: Comma separated list of family versions. 'all' to get all the family versions. Not supported if multiple family ids are provided. @@ -290,8 +259,8 @@ def update(self, families, data=None, **options): must be provided. :param str exclude: Fields excluded in the response, whole JSON path must be provided. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param bool update_roles: Update the member roles within the family. :param bool update_pedigree_graph: Update the family pedigree graph. :param str annotation_sets_action: Action to be performed if the array @@ -311,8 +280,8 @@ def update_annotation_sets_annotations(self, family, annotation_set, data=None, :param str annotation_set: AnnotationSet ID to be updated. (REQUIRED) :param str family: Family id. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str action: Action to be performed: ADD to add new annotations; REPLACE to replace the value of an already existing annotation; SET to set the new list of annotations removing any possible old diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/file_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/file_client.py index 61a4f098c06..505d21464b2 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/file_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/file_client.py @@ -2,7 +2,6 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-04-25 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +13,6 @@ class File(_ParentRestClient): """ This class contains methods for the 'Files' webservices - Client version: 2.12.4-SNAPSHOT PATH: /{apiVersion}/files """ @@ -32,48 +30,13 @@ def update_acl(self, members, action, data=None, **options): Allowed values: ['SET ADD REMOVE RESET'] (REQUIRED) :param str members: Comma separated list of user or group ids. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. """ options['action'] = action return self._post(category='files', resource='update', subcategory='acl', second_query_id=members, data=data, **options) - def aggregation_stats(self, **options): - """ - Fetch catalog file stats. - PATH: /{apiVersion}/files/aggregationStats - - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. - :param str name: Name. - :param str type: Type. - :param str format: Format. - :param str bioformat: Bioformat. - :param str creation_year: Creation year. - :param str creation_month: Creation month (JANUARY, FEBRUARY...). - :param str creation_day: Creation day. - :param str creation_day_of_week: Creation day of week (MONDAY, - TUESDAY...). - :param str status: Status. - :param str release: Release. - :param bool external: External. - :param str size: Size. - :param str software: Software. - :param str experiment: Experiment. - :param str num_samples: Number of samples. - :param str num_related_files: Number of related files. - :param str annotation: Annotation filters. Example: - age>30;gender=FEMALE. For more information, please visit - http://docs.opencb.org/display/opencga/AnnotationSets+1.4.0. - :param bool default: Calculate default stats. - :param str field: List of fields separated by semicolons, e.g.: - studies;type. For nested fields use >>, e.g.: - studies>>biotype;type;numSamples[0..10]:1. - """ - - return self._get(category='files', resource='aggregationStats', **options) - def load_annotation_sets(self, variable_set_id, path, data=None, **options): """ Load annotation sets from a TSV file. @@ -82,8 +45,8 @@ def load_annotation_sets(self, variable_set_id, path, data=None, **options): :param str path: Path where the TSV file is located in OpenCGA or where it should be located. (REQUIRED) :param str variable_set_id: Variable set ID or name. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param bool parents: Flag indicating whether to create parent directories if they don't exist (only when TSV file was not previously associated). @@ -111,8 +74,8 @@ def create(self, data=None, **options): PATH: /{apiVersion}/files/create :param dict data: File parameters. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param bool parents: Create the parent directories if they do not exist. """ @@ -126,8 +89,8 @@ def distinct(self, field, **options): :param str field: Comma separated list of fields for which to obtain the distinct values. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str id: Comma separated list of file IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' @@ -199,8 +162,15 @@ def fetch(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. """ return self._post(category='files', resource='fetch', data=data, **options) @@ -219,8 +189,8 @@ def link(self, data=None, **options): PATH: /{apiVersion}/files/link :param dict data: File parameters. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param bool parents: Create the parent directories if they do not exist. """ @@ -233,14 +203,21 @@ def run_link(self, data=None, **options): PATH: /{apiVersion}/files/link/run :param dict data: File parameters. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str job_id: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_description: Job description. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='files', resource='run', subcategory='link', data=data, **options) @@ -252,14 +229,21 @@ def run_postlink(self, data=None, **options): PATH: /{apiVersion}/files/postlink/run :param dict data: File parameters. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str job_id: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_description: Job description. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='files', resource='run', subcategory='postlink', data=data, **options) @@ -279,8 +263,8 @@ def search(self, **options): Deactivated by default. :param bool flatten_annotations: Boolean indicating to flatten the annotations. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str id: Comma separated list of file IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' @@ -361,8 +345,8 @@ def upload(self, **options): OTHER_PED VCF4 VARIANT ALIGNMENT COVERAGE SEQUENCE PEDIGREE REFERENCE_GENOME NONE UNKNOWN'] :param str checksum: Expected MD5 file checksum. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str relative_file_path: Path within catalog where the file will be located (default: root folder). :param str description: description. @@ -381,8 +365,8 @@ def acl(self, files, **options): :param str files: Comma separated list of file IDs or names up to a maximum of 100. (REQUIRED) :param str study: Comma separated list of Studies - [[user@]project:]study where study and project can be either the ID - or UUID up to a maximum of 100. + [[organization@]project:]study where study and project can be + either the ID or UUID up to a maximum of 100. :param str member: User or group id. :param bool silent: Boolean to retrieve all possible entries that are queried for, false to raise an exception whenever one of the @@ -398,8 +382,8 @@ def delete(self, files, **options): :param str files: Comma separated list of file ids, names or paths. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param bool skip_trash: Skip trash and delete the files/folders from disk directly (CANNOT BE RECOVERED). """ @@ -418,8 +402,8 @@ def info(self, files, **options): :param str exclude: Fields excluded in the response, whole JSON path must be provided. :param bool flatten_annotations: Flatten the annotations?. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param bool deleted: Boolean to retrieve deleted files. """ @@ -432,8 +416,8 @@ def unlink(self, files, **options): :param str files: Comma separated list of file ids, names or paths. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. """ return self._delete(category='files', resource='unlink', query_id=files, **options) @@ -450,8 +434,8 @@ def update(self, files, data=None, **options): must be provided. :param str exclude: Fields excluded in the response, whole JSON path must be provided. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str sample_ids_action: Action to be performed if the array of samples is being updated. Allowed values: ['ADD SET REMOVE'] :param str annotation_sets_action: Action to be performed if the array @@ -474,8 +458,8 @@ def update_annotation_sets_annotations(self, file, annotation_set, data=None, ** :param str annotation_set: AnnotationSet ID to be updated. (REQUIRED) :param str file: File id, name or path. Paths must be separated by : instead of /. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str action: Action to be performed: ADD to add new annotations; REPLACE to replace the value of an already existing annotation; SET to set the new list of annotations removing any possible old @@ -500,8 +484,8 @@ def download(self, file, **options): :param str file: File id, name or path. Paths must be separated by : instead of /. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. """ return self._get(category='files', resource='download', query_id=file, **options) @@ -512,8 +496,8 @@ def grep(self, file, **options): PATH: /{apiVersion}/files/{file}/grep :param str file: File uuid, id, or name. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str pattern: String pattern. :param bool ignore_case: Flag to perform a case insensitive search. :param int max_count: Stop reading a file after 'n' matching lines. 0 @@ -528,8 +512,8 @@ def head(self, file, **options): PATH: /{apiVersion}/files/{file}/head :param str file: File uuid, id, or name. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param int offset: Starting byte from which the file will be read. :param int lines: Maximum number of lines to be returned up to a maximum of 1000. @@ -543,8 +527,8 @@ def image(self, file, **options): PATH: /{apiVersion}/files/{file}/image :param str file: File ID. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. """ return self._get(category='files', resource='image', query_id=file, **options) @@ -560,8 +544,8 @@ def move(self, file, data=None, **options): must be provided. :param str exclude: Fields excluded in the response, whole JSON path must be provided. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. """ return self._post(category='files', resource='move', query_id=file, data=data, **options) @@ -574,8 +558,8 @@ def refresh(self, file, **options): :param str file: File id, name or path. Paths must be separated by : instead of /. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. """ return self._get(category='files', resource='refresh', query_id=file, **options) @@ -586,8 +570,8 @@ def tail(self, file, **options): PATH: /{apiVersion}/files/{file}/tail :param str file: File uuid, id, or name. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param int lines: Maximum number of lines to be returned up to a maximum of 1000. """ @@ -608,8 +592,8 @@ def list(self, folder, **options): :param int skip: Number of results to skip. :param bool count: Get the total number of results matching the query. Deactivated by default. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. """ return self._get(category='files', resource='list', query_id=folder, **options) @@ -625,8 +609,8 @@ def tree(self, folder, **options): must be provided. :param str exclude: Fields excluded in the response, whole JSON path must be provided. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param int max_depth: Maximum depth to get files from. """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/ga4gh_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/ga4gh_client.py index 759d188fd17..2b850d3c6fb 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/ga4gh_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/ga4gh_client.py @@ -2,7 +2,6 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-04-25 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +13,6 @@ class GA4GH(_ParentRestClient): """ This class contains methods for the 'GA4GH' webservices - Client version: 2.12.4-SNAPSHOT PATH: /{apiVersion}/ga4gh """ @@ -34,8 +32,8 @@ def fetch_reads(self, study, file, **options): Fetch alignment files using HTSget protocol. PATH: /{apiVersion}/ga4gh/reads/{study}/{file} - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. (REQUIRED) + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. (REQUIRED) :param str file: File id, name or path. (REQUIRED) :param str reference_name: Reference sequence name (Example: 'chr1', '1' or 'chrX'. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/individual_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/individual_client.py index b4abe50abc8..515ca12ca0b 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/individual_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/individual_client.py @@ -2,7 +2,6 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-04-25 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +13,6 @@ class Individual(_ParentRestClient): """ This class contains methods for the 'Individuals' webservices - Client version: 2.12.4-SNAPSHOT PATH: /{apiVersion}/individuals """ @@ -34,8 +32,8 @@ def update_acl(self, members, action, data=None, **options): Allowed values: ['SET ADD REMOVE RESET'] (REQUIRED) :param str members: Comma separated list of user or group ids. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param bool propagate: Propagate individual permissions to related samples. """ @@ -43,42 +41,6 @@ def update_acl(self, members, action, data=None, **options): options['action'] = action return self._post(category='individuals', resource='update', subcategory='acl', second_query_id=members, data=data, **options) - def aggregation_stats(self, **options): - """ - Fetch catalog individual stats. - PATH: /{apiVersion}/individuals/aggregationStats - - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. - :param bool has_father: Has father. - :param bool has_mother: Has mother. - :param str sex: Sex. - :param str karyotypic_sex: Karyotypic sex. - :param str ethnicity: Ethnicity. - :param str population: Population. - :param str creation_year: Creation year. - :param str creation_month: Creation month (JANUARY, FEBRUARY...). - :param str creation_day: Creation day. - :param str creation_day_of_week: Creation day of week (MONDAY, - TUESDAY...). - :param str status: Status. - :param str life_status: Life status. - :param str phenotypes: Phenotypes. - :param str num_samples: Number of samples. - :param bool parental_consanguinity: Parental consanguinity. - :param str release: Release. - :param str version: Version. - :param str annotation: Annotation filters. Example: - age>30;gender=FEMALE. For more information, please visit - http://docs.opencb.org/display/opencga/AnnotationSets+1.4.0. - :param bool default: Calculate default stats. - :param str field: List of fields separated by semicolons, e.g.: - studies;type. For nested fields use >>, e.g.: - studies>>biotype;type;numSamples[0..10]:1. - """ - - return self._get(category='individuals', resource='aggregationStats', **options) - def load_annotation_sets(self, variable_set_id, path, data=None, **options): """ Load annotation sets from a TSV file. @@ -87,8 +49,8 @@ def load_annotation_sets(self, variable_set_id, path, data=None, **options): :param str path: Path where the TSV file is located in OpenCGA or where it should be located. (REQUIRED) :param str variable_set_id: Variable set ID or name. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param bool parents: Flag indicating whether to create parent directories if they don't exist (only when TSV file was not previously associated). @@ -112,8 +74,8 @@ def create(self, data=None, **options): must be provided. :param str exclude: Fields excluded in the response, whole JSON path must be provided. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str samples: Comma separated list of sample ids to be associated to the created individual. :param bool include_result: Flag indicating to include the created or @@ -129,8 +91,8 @@ def distinct(self, field, **options): :param str field: Comma separated list of fields for which to obtain the distinct values. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str id: Comma separated list individual IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' @@ -207,8 +169,8 @@ def search(self, **options): :param bool count: Get the total number of results matching the query. Deactivated by default. :param bool flatten_annotations: Flatten the annotations?. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str id: Comma separated list individual IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' @@ -278,8 +240,8 @@ def acl(self, individuals, **options): :param str individuals: Comma separated list of individual IDs, names or UUIDs up to a maximum of 100. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str member: User or group id. :param bool silent: Boolean to retrieve all possible entries that are queried for, false to raise an exception whenever one of the @@ -297,8 +259,8 @@ def delete(self, individuals, **options): (REQUIRED) :param bool force: Force the deletion of individuals that already belong to families. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. """ return self._delete(category='individuals', resource='delete', query_id=individuals, **options) @@ -315,8 +277,8 @@ def info(self, individuals, **options): :param str exclude: Fields excluded in the response, whole JSON path must be provided. :param bool flatten_annotations: Flatten the annotations?. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str version: Comma separated list of individual versions. 'all' to get all the individual versions. Not supported if multiple individual ids are provided. @@ -336,8 +298,8 @@ def update(self, individuals, data=None, **options): must be provided. :param str exclude: Fields excluded in the response, whole JSON path must be provided. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str samples_action: Action to be performed if the array of samples is being updated. Allowed values: ['ADD SET REMOVE'] :param str phenotypes_action: Action to be performed if the array of @@ -363,8 +325,8 @@ def update_annotation_sets_annotations(self, individual, annotation_set, data=No :param str annotation_set: AnnotationSet ID to be updated. (REQUIRED) :param str individual: Individual ID, name or UUID. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str action: Action to be performed: ADD to add new annotations; REPLACE to replace the value of an already existing annotation; SET to set the new list of annotations removing any possible old @@ -393,8 +355,8 @@ def relatives(self, individual, **options): :param str exclude: Fields excluded in the response, whole JSON path must be provided. :param bool flatten_annotations: Flatten the annotations?. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param int degree: Pedigree degree. """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py index 278d1595159..c8425b16e6a 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py @@ -2,7 +2,6 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-04-25 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +13,6 @@ class Job(_ParentRestClient): """ This class contains methods for the 'Jobs' webservices - Client version: 2.12.4-SNAPSHOT PATH: /{apiVersion}/jobs """ @@ -37,45 +35,14 @@ def update_acl(self, members, action, data=None, **options): options['action'] = action return self._post(category='jobs', resource='update', subcategory='acl', second_query_id=members, data=data, **options) - def aggregation_stats(self, **options): - """ - Fetch catalog job stats. - PATH: /{apiVersion}/jobs/aggregationStats - - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. - :param str tool_id: Tool id. - :param str tool_scope: Tool scope. - :param str tool_type: Tool type. - :param str tool_resource: Tool resource. - :param str user_id: User id. - :param str priority: Priority. - :param str tags: Tags. - :param str executor_id: Executor id. - :param str executor_framework: Executor framework. - :param str creation_year: Creation year. - :param str creation_month: Creation month (JANUARY, FEBRUARY...). - :param str creation_day: Creation day. - :param str creation_day_of_week: Creation day of week (MONDAY, - TUESDAY...). - :param str status: Status. - :param str release: Release. - :param bool default: Calculate default stats. - :param str field: List of fields separated by semicolons, e.g.: - studies;type. For nested fields use >>, e.g.: - studies>>biotype;type;numSamples[0..10]:1. - """ - - return self._get(category='jobs', resource='aggregationStats', **options) - def create(self, data=None, **options): """ Register an executed job with POST method. PATH: /{apiVersion}/jobs/create :param dict data: job. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. """ return self._post(category='jobs', resource='create', data=data, **options) @@ -87,8 +54,8 @@ def distinct(self, field, **options): :param str field: Comma separated list of fields for which to obtain the distinct values. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param bool other_studies: Flag indicating the entries being queried can belong to any related study, not just the primary one. :param str id: Comma separated list of job IDs up to a maximum of 100. @@ -139,8 +106,10 @@ def retry(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. """ return self._post(category='jobs', resource='retry', data=data, **options) @@ -158,8 +127,8 @@ def search(self, **options): :param int skip: Number of results to skip. :param bool count: Get the total number of results matching the query. Deactivated by default. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param bool other_studies: Flag indicating the entries being queried can belong to any related study, not just the primary one. :param str id: Comma separated list of job IDs up to a maximum of 100. @@ -203,8 +172,8 @@ def top(self, **options): PATH: /{apiVersion}/jobs/top :param int limit: Maximum number of jobs to be returned. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str internal_status: Filter by internal status. :param str priority: Priority of the job. :param str user_id: User that created the job. @@ -238,8 +207,8 @@ def delete(self, jobs, **options): PATH: /{apiVersion}/jobs/{jobs}/delete :param str jobs: Comma separated list of job ids. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. """ return self._delete(category='jobs', resource='delete', query_id=jobs, **options) @@ -255,8 +224,8 @@ def info(self, jobs, **options): must be provided. :param str exclude: Fields excluded in the response, whole JSON path must be provided. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param bool deleted: Boolean to retrieve deleted jobs. """ @@ -273,8 +242,8 @@ def update(self, jobs, data=None, **options): must be provided. :param str exclude: Fields excluded in the response, whole JSON path must be provided. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param bool include_result: Flag indicating to include the created or updated document result in the response. :param dict data: body. @@ -282,14 +251,26 @@ def update(self, jobs, data=None, **options): return self._post(category='jobs', resource='update', query_id=jobs, data=data, **options) + def kill(self, job, **options): + """ + Send a signal to kill a pending or running job. + PATH: /{apiVersion}/jobs/{job}/kill + + :param str job: Job ID or UUID. (REQUIRED) + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. + """ + + return self._post(category='jobs', resource='kill', query_id=job, **options) + def head_log(self, job, **options): """ Show the first lines of a log file (up to a limit). PATH: /{apiVersion}/jobs/{job}/log/head :param str job: Job ID or UUID. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param int offset: Starting byte from which the file will be read. :param int lines: Maximum number of lines to be returned up to a maximum of 1000. @@ -304,8 +285,8 @@ def tail_log(self, job, **options): PATH: /{apiVersion}/jobs/{job}/log/tail :param str job: Job ID or UUID. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param int lines: Maximum number of lines to be returned up to a maximum of 1000. :param str type: Log file to be shown (stdout or stderr). diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/meta_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/meta_client.py index c5beeb4d320..39c13a7bcb7 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/meta_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/meta_client.py @@ -2,7 +2,6 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-04-25 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +13,6 @@ class Meta(_ParentRestClient): """ This class contains methods for the 'Meta' webservices - Client version: 2.12.4-SNAPSHOT PATH: /{apiVersion}/meta """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/organization_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/organization_client.py new file mode 100644 index 00000000000..0d7dc0ff159 --- /dev/null +++ b/opencga-client/src/main/python/pyopencga/rest_clients/organization_client.py @@ -0,0 +1,205 @@ +""" +WARNING: AUTOGENERATED CODE + + This code was generated by a tool. + + Manual changes to this file may cause unexpected behavior in your application. + Manual changes to this file will be overwritten if the code is regenerated. +""" + +from pyopencga.rest_clients._parent_rest_clients import _ParentRestClient + + +class Organization(_ParentRestClient): + """ + This class contains methods for the 'Organizations' webservices + PATH: /{apiVersion}/organizations + """ + + def __init__(self, configuration, token=None, login_handler=None, *args, **kwargs): + super(Organization, self).__init__(configuration, token, login_handler, *args, **kwargs) + + def create(self, data=None, **options): + """ + Create a new organization. + PATH: /{apiVersion}/organizations/create + + :param dict data: JSON containing the organization to be created. + (REQUIRED) + :param str include: Fields included in the response, whole JSON path + must be provided. + :param str exclude: Fields excluded in the response, whole JSON path + must be provided. + :param bool include_result: Flag indicating to include the created or + updated document result in the response. + """ + + return self._post(category='organizations', resource='create', data=data, **options) + + def create_notes(self, data=None, **options): + """ + Create a new note. + PATH: /{apiVersion}/organizations/notes/create + + :param dict data: JSON containing the Note to be added. (REQUIRED) + :param str include: Fields included in the response, whole JSON path + must be provided. + :param str exclude: Fields excluded in the response, whole JSON path + must be provided. + :param bool include_result: Flag indicating to include the created or + updated document result in the response. + """ + + return self._post(category='organizations', resource='create', subcategory='notes', data=data, **options) + + def search_notes(self, **options): + """ + Search for notes of scope ORGANIZATION. + PATH: /{apiVersion}/organizations/notes/search + + :param str include: Fields included in the response, whole JSON path + must be provided. + :param str exclude: Fields excluded in the response, whole JSON path + must be provided. + :param str creation_date: Creation date. Format: yyyyMMddHHmmss. + Examples: >2018, 2017-2018, <201805. + :param str modification_date: Modification date. Format: + yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + :param str id: Note unique identifier. + :param str scope: Scope of the Note. + :param str visibility: Visibility of the Note. + :param str uuid: Unique 32-character identifier assigned automatically + by OpenCGA. + :param str user_id: User that wrote that Note. + :param str tags: Note tags. + :param str version: Autoincremental version assigned to the registered + entry. By default, updates does not create new versions. To enable + versioning, users must set the `incVersion` flag from the /update + web service when updating the document. + """ + + return self._get(category='organizations', resource='search', subcategory='notes', **options) + + def delete_notes(self, id, **options): + """ + Delete note. + PATH: /{apiVersion}/organizations/notes/{id}/delete + + :param str id: Note unique identifier. (REQUIRED) + :param bool include_result: Flag indicating to include the created or + updated document result in the response. + """ + + return self._delete(category='organizations', resource='delete', subcategory='notes', second_query_id=id, **options) + + def update_notes(self, id, data=None, **options): + """ + Update a note. + PATH: /{apiVersion}/organizations/notes/{id}/update + + :param dict data: JSON containing the Note fields to be updated. + (REQUIRED) + :param str id: Note unique identifier. (REQUIRED) + :param str include: Fields included in the response, whole JSON path + must be provided. + :param str exclude: Fields excluded in the response, whole JSON path + must be provided. + :param str tags_action: Action to be performed if the array of tags is + being updated. Allowed values: ['ADD SET REMOVE'] + :param bool include_result: Flag indicating to include the created or + updated document result in the response. + """ + + return self._post(category='organizations', resource='update', subcategory='notes', second_query_id=id, data=data, **options) + + def user_update_status(self, user, data=None, **options): + """ + Update the user status. + PATH: /{apiVersion}/organizations/user/{user}/status/update + + :param dict data: JSON containing the User fields to be updated. + (REQUIRED) + :param str user: User ID. (REQUIRED) + :param str include: Fields included in the response, whole JSON path + must be provided. + :param str exclude: Fields excluded in the response, whole JSON path + must be provided. + :param str organization: Organization id. + :param bool include_result: Flag indicating to include the created or + updated document result in the response. + """ + + return self._post(category='organizations/user', resource='update', query_id=user, subcategory='status', data=data, **options) + + def update_user(self, user, data=None, **options): + """ + Update the user information. + PATH: /{apiVersion}/organizations/user/{user}/update + + :param dict data: JSON containing the User fields to be updated. + (REQUIRED) + :param str user: User ID. (REQUIRED) + :param str include: Fields included in the response, whole JSON path + must be provided. + :param str exclude: Fields excluded in the response, whole JSON path + must be provided. + :param str organization: Organization id. + :param bool include_result: Flag indicating to include the created or + updated document result in the response. + """ + + return self._post(category='organizations', resource='update', subcategory='user', second_query_id=user, data=data, **options) + + def update_configuration(self, organization, data=None, **options): + """ + Update the Organization configuration attributes. + PATH: /{apiVersion}/organizations/{organization}/configuration/update + + :param dict data: JSON containing the params to be updated. (REQUIRED) + :param str organization: Organization id. (REQUIRED) + :param str include: Fields included in the response, whole JSON path + must be provided. + :param str exclude: Fields excluded in the response, whole JSON path + must be provided. + :param bool include_result: Flag indicating to include the created or + updated document result in the response. + :param str authentication_origins_action: Action to be performed if + the array of authenticationOrigins is being updated. Allowed + values: ['ADD SET REMOVE REPLACE'] + """ + + return self._post(category='organizations', resource='update', query_id=organization, subcategory='configuration', data=data, **options) + + def info(self, organization, **options): + """ + Return the organization information. + PATH: /{apiVersion}/organizations/{organization}/info + + :param str organization: Organization id. (REQUIRED) + :param str include: Fields included in the response, whole JSON path + must be provided. + :param str exclude: Fields excluded in the response, whole JSON path + must be provided. + """ + + return self._get(category='organizations', resource='info', query_id=organization, **options) + + def update(self, organization, data=None, **options): + """ + Update some organization attributes. + PATH: /{apiVersion}/organizations/{organization}/update + + :param dict data: JSON containing the params to be updated. (REQUIRED) + :param str organization: Organization id. (REQUIRED) + :param str include: Fields included in the response, whole JSON path + must be provided. + :param str exclude: Fields excluded in the response, whole JSON path + must be provided. + :param bool include_result: Flag indicating to include the created or + updated document result in the response. + :param str admins_action: Action to be performed if the array of + admins is being updated. Allowed values: ['ADD REMOVE'] + """ + + return self._post(category='organizations', resource='update', query_id=organization, data=data, **options) + diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/project_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/project_client.py index 1fc03169013..dd650ac0f26 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/project_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/project_client.py @@ -2,7 +2,6 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-04-25 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +13,6 @@ class Project(_ParentRestClient): """ This class contains methods for the 'Projects' webservices - Client version: 2.12.4-SNAPSHOT PATH: /{apiVersion}/projects """ @@ -48,12 +46,11 @@ def search(self, **options): must be provided. :param int limit: Number of results to be returned. :param int skip: Number of results to skip. - :param str owner: Owner of the project. - :param str id: Project [user@]project where project can be either the - ID or the alias. + :param str organization: Project organization. + :param str id: Project [organization@]project where project can be + either the ID or the alias. :param str name: Project name. :param str fqn: Project fqn. - :param str organization: Project organization. :param str description: Project description. :param str study: Study id. :param str creation_date: Creation date. Format: yyyyMMddHHmmss. @@ -66,43 +63,13 @@ def search(self, **options): return self._get(category='projects', resource='search', **options) - def aggregation_stats(self, projects, **options): - """ - Fetch catalog project stats. - PATH: /{apiVersion}/projects/{projects}/aggregationStats - - :param str projects: Comma separated list of projects [user@]project - up to a maximum of 100. (REQUIRED) - :param bool default: Calculate default stats. - :param str file_fields: List of file fields separated by semicolons, - e.g.: studies;type. For nested fields use >>, e.g.: - studies>>biotype;type. - :param str individual_fields: List of individual fields separated by - semicolons, e.g.: studies;type. For nested fields use >>, e.g.: - studies>>biotype;type. - :param str family_fields: List of family fields separated by - semicolons, e.g.: studies;type. For nested fields use >>, e.g.: - studies>>biotype;type. - :param str sample_fields: List of sample fields separated by - semicolons, e.g.: studies;type. For nested fields use >>, e.g.: - studies>>biotype;type. - :param str cohort_fields: List of cohort fields separated by - semicolons, e.g.: studies;type. For nested fields use >>, e.g.: - studies>>biotype;type. - :param str job_fields: List of job fields separated by semicolons, - e.g.: studies;type. For nested fields use >>, e.g.: - studies>>biotype;type. - """ - - return self._get(category='projects', resource='aggregationStats', query_id=projects, **options) - def info(self, projects, **options): """ Fetch project information. PATH: /{apiVersion}/projects/{projects}/info - :param str projects: Comma separated list of projects [user@]project - up to a maximum of 100. (REQUIRED) + :param str projects: Comma separated list of projects + [organization@]project up to a maximum of 100. (REQUIRED) :param str include: Fields included in the response, whole JSON path must be provided. :param str exclude: Fields excluded in the response, whole JSON path @@ -116,8 +83,8 @@ def inc_release(self, project, **options): Increment current release number in the project. PATH: /{apiVersion}/projects/{project}/incRelease - :param str project: Project [user@]project where project can be either - the ID or the alias. (REQUIRED) + :param str project: Project [organization@]project where project can + be either the ID or the alias. (REQUIRED) """ return self._post(category='projects', resource='incRelease', query_id=project, **options) @@ -127,8 +94,8 @@ def studies(self, project, **options): Fetch all the studies contained in the project. PATH: /{apiVersion}/projects/{project}/studies - :param str project: Project [user@]project where project can be either - the ID or the alias. (REQUIRED) + :param str project: Project [organization@]project where project can + be either the ID or the alias. (REQUIRED) :param str include: Fields included in the response, whole JSON path must be provided. :param str exclude: Fields excluded in the response, whole JSON path @@ -147,8 +114,8 @@ def update(self, project, data=None, **options): :param dict data: JSON containing the params to be updated. It will be only possible to update organism fields not previously defined. (REQUIRED) - :param str project: Project [user@]project where project can be either - the ID or the alias. (REQUIRED) + :param str project: Project [organization@]project where project can + be either the ID or the alias. (REQUIRED) :param str include: Fields included in the response, whole JSON path must be provided. :param str exclude: Fields excluded in the response, whole JSON path diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/sample_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/sample_client.py index ff401a57af2..6b10aac717f 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/sample_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/sample_client.py @@ -2,7 +2,6 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-04-25 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +13,6 @@ class Sample(_ParentRestClient): """ This class contains methods for the 'Samples' webservices - Client version: 2.12.4-SNAPSHOT PATH: /{apiVersion}/samples """ @@ -34,43 +32,13 @@ def update_acl(self, members, action, data=None, **options): Allowed values: ['SET ADD REMOVE RESET'] (REQUIRED) :param str members: Comma separated list of user or group ids. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. """ options['action'] = action return self._post(category='samples', resource='update', subcategory='acl', second_query_id=members, data=data, **options) - def aggregation_stats(self, **options): - """ - Fetch catalog sample stats. - PATH: /{apiVersion}/samples/aggregationStats - - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. - :param str source: Source. - :param str creation_year: Creation year. - :param str creation_month: Creation month (JANUARY, FEBRUARY...). - :param str creation_day: Creation day. - :param str creation_day_of_week: Creation day of week (MONDAY, - TUESDAY...). - :param str status: Status. - :param str type: Type. - :param str phenotypes: Phenotypes. - :param str release: Release. - :param str version: Version. - :param bool somatic: Somatic. - :param str annotation: Annotation filters. Example: - age>30;gender=FEMALE. For more information, please visit - http://docs.opencb.org/display/opencga/AnnotationSets+1.4.0. - :param bool default: Calculate default stats. - :param str field: List of fields separated by semicolons, e.g.: - studies;type. For nested fields use >>, e.g.: - studies>>biotype;type;numSamples[0..10]:1. - """ - - return self._get(category='samples', resource='aggregationStats', **options) - def load_annotation_sets(self, variable_set_id, path, data=None, **options): """ Load annotation sets from a TSV file. @@ -79,8 +47,8 @@ def load_annotation_sets(self, variable_set_id, path, data=None, **options): :param str path: Path where the TSV file is located in OpenCGA or where it should be located. (REQUIRED) :param str variable_set_id: Variable set ID or name. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param bool parents: Flag indicating whether to create parent directories if they don't exist (only when TSV file was not previously associated). @@ -104,8 +72,8 @@ def create(self, data=None, **options): must be provided. :param str exclude: Fields excluded in the response, whole JSON path must be provided. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param bool include_result: Flag indicating to include the created or updated document result in the response. """ @@ -119,8 +87,8 @@ def distinct(self, field, **options): :param str field: Comma separated list of fields for which to obtain the distinct values. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str id: Comma separated list sample IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for @@ -194,8 +162,8 @@ def load(self, file, **options): PATH: /{apiVersion}/samples/load :param str file: file. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str variable_set: variableSet. """ @@ -218,8 +186,8 @@ def search(self, **options): :param bool include_individual: Include Individual object as an attribute. :param bool flatten_annotations: Flatten the annotations?. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str id: Comma separated list sample IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for @@ -294,8 +262,8 @@ def acl(self, samples, **options): :param str samples: Comma separated list sample IDs or UUIDs up to a maximum of 100. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str member: User or group id. :param bool silent: Boolean to retrieve all possible entries that are queried for, false to raise an exception whenever one of the @@ -318,8 +286,8 @@ def delete(self, samples, **options): are NONE, TRASH, DELETE. :param bool delete_empty_cohorts: Boolean indicating if the cohorts associated only to the sample to be deleted should be also deleted. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. """ return self._delete(category='samples', resource='delete', query_id=samples, **options) @@ -338,8 +306,8 @@ def info(self, samples, **options): :param bool include_individual: Include Individual object as an attribute. :param bool flatten_annotations: Flatten the annotations?. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str version: Comma separated list of sample versions. 'all' to get all the sample versions. Not supported if multiple sample ids are provided. @@ -359,8 +327,8 @@ def update(self, samples, data=None, **options): must be provided. :param str exclude: Fields excluded in the response, whole JSON path must be provided. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str annotation_sets_action: Action to be performed if the array of annotationSets is being updated. Allowed values: ['ADD SET REMOVE'] @@ -381,8 +349,8 @@ def update_annotation_sets_annotations(self, sample, annotation_set, data=None, :param str annotation_set: AnnotationSet ID to be updated. (REQUIRED) :param str sample: Sample ID. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str action: Action to be performed: ADD to add new annotations; REPLACE to replace the value of an already existing annotation; SET to set the new list of annotations removing any possible old diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py index 9c6c7610d38..b730dc06d5d 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py @@ -2,7 +2,6 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-04-25 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +13,6 @@ class Study(_ParentRestClient): """ This class contains methods for the 'Studies' webservices - Client version: 2.12.4-SNAPSHOT PATH: /{apiVersion}/studies """ @@ -48,8 +46,8 @@ def create(self, data=None, **options): must be provided. :param str exclude: Fields excluded in the response, whole JSON path must be provided. - :param str project: Project [user@]project where project can be either - the ID or the alias. + :param str project: Project [organization@]project where project can + be either the ID or the alias. :param bool include_result: Flag indicating to include the created or updated document result in the response. """ @@ -61,8 +59,8 @@ def search(self, project, **options): Search studies. PATH: /{apiVersion}/studies/search - :param str project: Project [user@]project where project can be either - the ID or the alias. (REQUIRED) + :param str project: Project [organization@]project where project can + be either the ID or the alias. (REQUIRED) :param str include: Fields included in the response, whole JSON path must be provided. :param str exclude: Fields excluded in the response, whole JSON path @@ -95,8 +93,8 @@ def acl(self, studies, **options): PATH: /{apiVersion}/studies/{studies}/acl :param str studies: Comma separated list of Studies - [[user@]project:]study where study and project can be either the ID - or UUID up to a maximum of 100. (REQUIRED) + [[organization@]project:]study where study and project can be + either the ID or UUID up to a maximum of 100. (REQUIRED) :param str member: User or group id. :param bool silent: Boolean to retrieve all possible entries that are queried for, false to raise an exception whenever one of the @@ -105,44 +103,14 @@ def acl(self, studies, **options): return self._get(category='studies', resource='acl', query_id=studies, **options) - def aggregation_stats(self, studies, **options): - """ - Fetch catalog study stats. - PATH: /{apiVersion}/studies/{studies}/aggregationStats - - :param str studies: Comma separated list of studies - [[user@]project:]study up to a maximum of 100. (REQUIRED) - :param bool default: Calculate default stats. - :param str file_fields: List of file fields separated by semicolons, - e.g.: studies;type. For nested fields use >>, e.g.: - studies>>biotype;type. - :param str individual_fields: List of individual fields separated by - semicolons, e.g.: studies;type. For nested fields use >>, e.g.: - studies>>biotype;type. - :param str family_fields: List of family fields separated by - semicolons, e.g.: studies;type. For nested fields use >>, e.g.: - studies>>biotype;type. - :param str sample_fields: List of sample fields separated by - semicolons, e.g.: studies;type. For nested fields use >>, e.g.: - studies>>biotype;type. - :param str cohort_fields: List of cohort fields separated by - semicolons, e.g.: studies;type. For nested fields use >>, e.g.: - studies>>biotype;type. - :param str job_fields: List of job fields separated by semicolons, - e.g.: studies;type. For nested fields use >>, e.g.: - studies>>biotype;type. - """ - - return self._get(category='studies', resource='aggregationStats', query_id=studies, **options) - def info(self, studies, **options): """ Fetch study information. PATH: /{apiVersion}/studies/{studies}/info :param str studies: Comma separated list of Studies - [[user@]project:]study where study and project can be either the ID - or UUID up to a maximum of 100. (REQUIRED) + [[organization@]project:]study where study and project can be + either the ID or UUID up to a maximum of 100. (REQUIRED) :param str include: Fields included in the response, whole JSON path must be provided. :param str exclude: Fields excluded in the response, whole JSON path @@ -168,10 +136,10 @@ def search_audit(self, study, **options): :param str operation_id: Audit operation UUID. :param str user_id: User ID. :param str action: Action performed by the user. - :param str resource: Resource involved. Allowed values: ['AUDIT USER - PROJECT STUDY FILE SAMPLE JOB INDIVIDUAL COHORT DISEASE_PANEL - FAMILY CLINICAL_ANALYSIS INTERPRETATION VARIANT ALIGNMENT CLINICAL - EXPRESSION RGA FUNCTIONAL'] + :param str resource: Resource involved. Allowed values: ['AUDIT NOTE + ORGANIZATION USER PROJECT STUDY FILE SAMPLE JOB INDIVIDUAL COHORT + DISEASE_PANEL FAMILY CLINICAL_ANALYSIS INTERPRETATION VARIANT + ALIGNMENT CLINICAL EXPRESSION RGA FUNCTIONAL'] :param str resource_id: Resource ID. :param str resource_uuid: resource UUID. :param str status: Filter by status. Allowed values: ['SUCCESS ERROR'] @@ -187,8 +155,8 @@ def groups(self, study, **options): only. PATH: /{apiVersion}/studies/{study}/groups - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. (REQUIRED) + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. (REQUIRED) :param str id: Group id. If provided, it will only fetch information for the provided group. :param bool silent: Boolean to retrieve all possible entries that are @@ -204,8 +172,8 @@ def update_groups(self, study, data=None, **options): PATH: /{apiVersion}/studies/{study}/groups/update :param dict data: JSON containing the parameters. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. (REQUIRED) + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. (REQUIRED) :param str action: Action to be performed: ADD or REMOVE a group. Allowed values: ['ADD REMOVE'] """ @@ -219,14 +187,97 @@ def update_groups_users(self, study, group, data=None, **options): :param dict data: JSON containing the parameters. (REQUIRED) :param str group: Group name. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. (REQUIRED) + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. (REQUIRED) :param str action: Action to be performed: ADD, SET or REMOVE users to/from a group. Allowed values: ['ADD SET REMOVE'] """ return self._post(category='studies', resource='users/update', query_id=study, subcategory='groups', second_query_id=group, data=data, **options) + def create_notes(self, study, data=None, **options): + """ + Create a new note. + PATH: /{apiVersion}/studies/{study}/notes/create + + :param dict data: JSON containing the Note to be added. (REQUIRED) + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. (REQUIRED) + :param str include: Fields included in the response, whole JSON path + must be provided. + :param str exclude: Fields excluded in the response, whole JSON path + must be provided. + :param bool include_result: Flag indicating to include the created or + updated document result in the response. + """ + + return self._post(category='studies', resource='create', query_id=study, subcategory='notes', data=data, **options) + + def search_notes(self, study, **options): + """ + Search for notes of scope STUDY. + PATH: /{apiVersion}/studies/{study}/notes/search + + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. (REQUIRED) + :param str include: Fields included in the response, whole JSON path + must be provided. + :param str exclude: Fields excluded in the response, whole JSON path + must be provided. + :param str creation_date: Creation date. Format: yyyyMMddHHmmss. + Examples: >2018, 2017-2018, <201805. + :param str modification_date: Modification date. Format: + yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + :param str id: Note unique identifier. + :param str uuid: Unique 32-character identifier assigned automatically + by OpenCGA. + :param str user_id: User that wrote that Note. + :param str tags: Note tags. + :param str visibility: Visibility of the Note. + :param str version: Autoincremental version assigned to the registered + entry. By default, updates does not create new versions. To enable + versioning, users must set the `incVersion` flag from the /update + web service when updating the document. + """ + + return self._get(category='studies', resource='search', query_id=study, subcategory='notes', **options) + + def delete_notes(self, study, id, **options): + """ + Delete note. + PATH: /{apiVersion}/studies/{study}/notes/{id}/delete + + :param str id: Note unique identifier. (REQUIRED) + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. (REQUIRED) + :param bool include_result: Flag indicating to include the created or + updated document result in the response. + """ + + return self._delete(category='studies', resource='delete', query_id=study, subcategory='notes', second_query_id=id, **options) + + def update_notes(self, study, id, data=None, **options): + """ + Update a note. + PATH: /{apiVersion}/studies/{study}/notes/{id}/update + + :param dict data: JSON containing the Note fields to be updated. + (REQUIRED) + :param str id: Note unique identifier. (REQUIRED) + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. (REQUIRED) + :param str include: Fields included in the response, whole JSON path + must be provided. + :param str exclude: Fields excluded in the response, whole JSON path + must be provided. + :param str tags_action: Action to be performed if the array of tags is + being updated. Allowed values: ['ADD SET REMOVE'] + :param bool include_result: Flag indicating to include the created or + updated document result in the response. + """ + + return self._post(category='studies', resource='update', query_id=study, subcategory='notes', second_query_id=id, data=data, **options) + def permission_rules(self, study, entity, **options): """ Fetch permission rules. @@ -235,8 +286,8 @@ def permission_rules(self, study, entity, **options): :param str entity: Entity where the permission rules should be applied to. Allowed values: ['SAMPLES FILES COHORTS INDIVIDUALS FAMILIES JOBS CLINICAL_ANALYSES DISEASE_PANELS'] (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. (REQUIRED) + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. (REQUIRED) """ options['entity'] = entity @@ -252,8 +303,8 @@ def update_permission_rules(self, study, entity, data=None, **options): :param str entity: Entity where the permission rules should be applied to. Allowed values: ['SAMPLES FILES COHORTS INDIVIDUALS FAMILIES JOBS CLINICAL_ANALYSES DISEASE_PANELS'] (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. (REQUIRED) + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. (REQUIRED) :param str action: Action to be performed: ADD to add a new permission rule; REMOVE to remove all permissions assigned by an existing permission rule (even if it overlaps any manual permission); REVERT @@ -273,14 +324,21 @@ def run_templates(self, study, data=None, **options): PATH: /{apiVersion}/studies/{study}/templates/run :param dict data: Template loader parameters. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. (REQUIRED) + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. (REQUIRED) :param str job_id: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_description: Job description. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='studies', resource='run', query_id=study, subcategory='templates', data=data, **options) @@ -290,8 +348,8 @@ def upload_templates(self, study, **options): Resource to upload a zipped template. PATH: /{apiVersion}/studies/{study}/templates/upload - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. (REQUIRED) + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. (REQUIRED) :param inputstream file: File to upload. """ @@ -303,8 +361,8 @@ def delete_templates(self, study, template_id, **options): PATH: /{apiVersion}/studies/{study}/templates/{templateId}/delete :param str template_id: Template id. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. (REQUIRED) """ return self._delete(category='studies', resource='delete', query_id=study, subcategory='templates', second_query_id=template_id, **options) @@ -315,8 +373,8 @@ def update(self, study, data=None, **options): PATH: /{apiVersion}/studies/{study}/update :param dict data: JSON containing the params to be updated. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. (REQUIRED) + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. (REQUIRED) :param str include: Fields included in the response, whole JSON path must be provided. :param str exclude: Fields excluded in the response, whole JSON path @@ -332,8 +390,8 @@ def variable_sets(self, study, **options): Fetch variableSets from a study. PATH: /{apiVersion}/studies/{study}/variableSets - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. (REQUIRED) + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. (REQUIRED) :param str id: Id of the variableSet to be retrieved. If no id is passed, it will show all the variableSets of the study. """ @@ -347,8 +405,8 @@ def update_variable_sets(self, study, data=None, **options): :param dict data: JSON containing the VariableSet to be created or removed. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. (REQUIRED) + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. (REQUIRED) :param str action: Action to be performed: ADD, REMOVE or FORCE_REMOVE a variableSet. Allowed values: ['ADD REMOVE FORCE_REMOVE'] """ @@ -364,8 +422,8 @@ def update_variable_sets_variables(self, study, variable_set, data=None, **optio For removing, only the variable id will be needed. (REQUIRED) :param str variable_set: VariableSet id of the VariableSet to be updated. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. (REQUIRED) + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. (REQUIRED) :param str action: Action to be performed: ADD or REMOVE a variable. Allowed values: ['ADD REMOVE'] """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/user_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/user_client.py index 654bd1e6b9e..93451319d3d 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/user_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/user_client.py @@ -2,7 +2,6 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-04-25 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,13 +13,33 @@ class User(_ParentRestClient): """ This class contains methods for the 'Users' webservices - Client version: 2.12.4-SNAPSHOT PATH: /{apiVersion}/users """ def __init__(self, configuration, token=None, login_handler=None, *args, **kwargs): super(User, self).__init__(configuration, token, login_handler, *args, **kwargs) + def anonymous(self, organization, **options): + """ + Get an anonymous token to gain access to the system. + PATH: /{apiVersion}/users/anonymous + + :param str organization: Organization id. (REQUIRED) + """ + + options['organization'] = organization + return self._post(category='users', resource='anonymous', **options) + + def create(self, data=None, **options): + """ + Create a new user. + PATH: /{apiVersion}/users/create + + :param dict data: JSON containing the parameters. (REQUIRED) + """ + + return self._post(category='users', resource='create', data=data, **options) + def login(self, data=None, **options): """ Get identified and gain access to the system. @@ -42,6 +61,29 @@ def password(self, data=None, **options): return self._post(category='users', resource='password', data=data, **options) + def search(self, **options): + """ + User search method. + PATH: /{apiVersion}/users/search + + :param str include: Fields included in the response, whole JSON path + must be provided. + :param str exclude: Fields excluded in the response, whole JSON path + must be provided. + :param int limit: Number of results to be returned. + :param int skip: Number of results to skip. + :param bool count: Get the total number of results matching the query. + Deactivated by default. + :param str organization: Organization id. + :param str id: Comma separated list user IDs up to a maximum of 100. + Also admits basic regular expressions using the operator '~', i.e. + '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for + case insensitive search. + :param str authentication_id: Authentication origin ID. + """ + + return self._get(category='users', resource='search', **options) + def info(self, users, **options): """ Return the user information including its projects and studies. @@ -52,6 +94,7 @@ def info(self, users, **options): must be provided. :param str exclude: Fields excluded in the response, whole JSON path must be provided. + :param str organization: Organization id. """ return self._get(category='users', resource='info', query_id=users, **options) @@ -130,22 +173,6 @@ def reset_password(self, user, **options): return self._get(category='users', resource='reset', query_id=user, subcategory='password', **options) - def projects(self, user, **options): - """ - Retrieve the projects of the user. - PATH: /{apiVersion}/users/{user}/projects - - :param str user: User ID. (REQUIRED) - :param str include: Fields included in the response, whole JSON path - must be provided. - :param str exclude: Fields excluded in the response, whole JSON path - must be provided. - :param int limit: Number of results to be returned. - :param int skip: Number of results to skip. - """ - - return self._get(category='users', resource='projects', query_id=user, **options) - def update(self, user, data=None, **options): """ Update some user attributes. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/variant_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/variant_client.py index 86e1f537ba4..3993f48ba22 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/variant_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/variant_client.py @@ -2,7 +2,6 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-04-25 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +13,6 @@ class Variant(_ParentRestClient): """ This class contains methods for the 'Analysis - Variant' webservices - Client version: 2.12.4-SNAPSHOT PATH: /{apiVersion}/analysis/variant """ @@ -34,11 +32,11 @@ def aggregation_stats(self, **options): SV, COPY_NUMBER, COPY_NUMBER_LOSS, COPY_NUMBER_GAIN, INSERTION, DELETION, DUPLICATION, TANDEM_DUPLICATION, BREAKEND, e.g. SNV,INDEL. - :param str project: Project [user@]project where project can be either - the ID or the alias. + :param str project: Project [organization@]project where project can + be either the ID or the alias. :param str study: Filter variants from the given studies, these can be either the numeric ID or the alias with the format - user@project:study. + organization@project:study. :param str cohort: Select variants with calculated stats for the selected cohorts. :param str cohort_stats_ref: Reference Allele Frequency: @@ -115,8 +113,8 @@ def metadata_annotation(self, **options): PATH: /{apiVersion}/analysis/variant/annotation/metadata :param str annotation_id: Annotation identifier. - :param str project: Project [user@]project where project can be either - the ID or the alias. + :param str project: Project [organization@]project where project can + be either the ID or the alias. """ return self._get(category='analysis', resource='metadata', subcategory='variant/annotation', **options) @@ -188,14 +186,21 @@ def run_cohort_stats(self, data=None, **options): PATH: /{apiVersion}/analysis/variant/cohort/stats/run :param dict data: Cohort variant stats params. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str job_id: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. :param str job_description: Job description. :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/cohort/stats', data=data, **options) @@ -214,6 +219,13 @@ def run_exomiser(self, data=None, **options): the job will depend on. :param str job_description: Job description. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/exomiser', data=data, **options) @@ -228,16 +240,23 @@ def run_export(self, data=None, **options): must be provided. :param str exclude: Fields excluded in the response, whole JSON path must be provided. - :param str project: Project [user@]project where project can be either - the ID or the alias. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str project: Project [organization@]project where project can + be either the ID or the alias. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str job_id: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. :param str job_description: Job description. :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/export', data=data, **options) @@ -251,8 +270,8 @@ def genotypes_family(self, mode_of_inheritance, **options): ['AUTOSOMAL_DOMINANT AUTOSOMAL_RECESSIVE X_LINKED_DOMINANT X_LINKED_RECESSIVE Y_LINKED MITOCHONDRIAL DE_NOVO MENDELIAN_ERROR COMPOUND_HETEROZYGOUS UNKNOWN'] (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str family: Family id. :param str clinical_analysis: Clinical analysis id. :param str penetrance: Penetrance. Allowed values: ['COMPLETE @@ -273,14 +292,21 @@ def run_family_qc(self, data=None, **options): method, by default 'PLINK/IBD'. Minor allele frequence (MAF) is used to filter variants before computing relatedness, e.g.: 1000G:CEU>0.35 or cohort:ALL>0.05. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str job_id: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. :param str job_description: Job description. :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/family/qc', data=data, **options) @@ -296,8 +322,15 @@ def delete_file(self, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str file: Files to remove. :param bool resume: Resume a previously failed indexation. """ @@ -320,6 +353,13 @@ def run_gatk(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/gatk', data=data, **options) @@ -335,14 +375,21 @@ def run_genome_plot(self, data=None, **options): tracks. Currently, the supported track types are: COPY-NUMBER, INDEL, REARRANGEMENT and SNV. In addition, each track can contain a specific query. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str job_id: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. :param str job_description: Job description. :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/genomePlot', data=data, **options) @@ -353,14 +400,21 @@ def run_gwas(self, data=None, **options): PATH: /{apiVersion}/analysis/variant/gwas/run :param dict data: Gwas analysis params. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str job_id: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. :param str job_description: Job description. :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/gwas', data=data, **options) @@ -371,14 +425,21 @@ def run_hr_detect(self, data=None, **options): PATH: /{apiVersion}/analysis/variant/hrDetect/run :param dict data: HRDetect analysis parameters. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str job_id: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. :param str job_description: Job description. :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/hrDetect', data=data, **options) @@ -389,14 +450,21 @@ def run_index(self, data=None, **options): PATH: /{apiVersion}/analysis/variant/index/run :param dict data: Variant index params. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str job_id: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_description: Job description. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/index', data=data, **options) @@ -408,14 +476,21 @@ def run_individual_qc(self, data=None, **options): PATH: /{apiVersion}/analysis/variant/individual/qc/run :param dict data: Individual QC analysis params. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str job_id: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. :param str job_description: Job description. :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/individual/qc', data=data, **options) @@ -426,14 +501,21 @@ def run_inferred_sex(self, data=None, **options): PATH: /{apiVersion}/analysis/variant/inferredSex/run :param dict data: Inferred sex analysis params. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str job_id: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. :param str job_description: Job description. :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/inferredSex', data=data, **options) @@ -477,6 +559,13 @@ def run_knockout(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/knockout', data=data, **options) @@ -487,14 +576,21 @@ def run_mendelian_error(self, data=None, **options): PATH: /{apiVersion}/analysis/variant/mendelianError/run :param dict data: Mendelian error analysis params. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str job_id: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. :param str job_description: Job description. :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/mendelianError', data=data, **options) @@ -504,11 +600,11 @@ def metadata(self, **options): . PATH: /{apiVersion}/analysis/variant/metadata - :param str project: Project [user@]project where project can be either - the ID or the alias. + :param str project: Project [organization@]project where project can + be either the ID or the alias. :param str study: Filter variants from the given studies, these can be either the numeric ID or the alias with the format - user@project:study. + organization@project:study. :param str file: Filter variants from the files specified. This will set includeFile parameter when not provided. :param str sample: Filter variants by sample genotype. This will @@ -555,7 +651,7 @@ def query_mutational_signature(self, **options): :param str study: Filter variants from the given studies, these can be either the numeric ID or the alias with the format - user@project:study. + organization@project:study. :param str sample: Sample name. :param str type: Variant type. Valid values: SNV, SV. :param str ct: List of SO consequence types, e.g. @@ -609,14 +705,21 @@ def run_mutational_signature(self, data=None, **options): the genome context for that sample, and to compute both catalogue counts and signature fitting. In order to skip one of them, , use the following keywords: , catalogue, fitting. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str job_id: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. :param str job_description: Job description. :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/mutationalSignature', data=data, **options) @@ -635,6 +738,13 @@ def run_plink(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/plink', data=data, **options) @@ -672,11 +782,11 @@ def query(self, **options): SNV,INDEL. :param str reference: Reference allele. :param str alternate: Main alternate allele. - :param str project: Project [user@]project where project can be either - the ID or the alias. + :param str project: Project [organization@]project where project can + be either the ID or the alias. :param str study: Filter variants from the given studies, these can be either the numeric ID or the alias with the format - user@project:study. + organization@project:study. :param str file: Filter variants from the files specified. This will set includeFile parameter when not provided. :param str filter: Specify the FILTER for any of the files. If 'file' @@ -851,14 +961,21 @@ def run_relatedness(self, data=None, **options): PATH: /{apiVersion}/analysis/variant/relatedness/run :param dict data: Relatedness analysis params. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str job_id: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. :param str job_description: Job description. :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/relatedness', data=data, **options) @@ -878,6 +995,13 @@ def run_rvtests(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/rvtests', data=data, **options) @@ -895,11 +1019,11 @@ def aggregation_stats_sample(self, **options): SV, COPY_NUMBER, COPY_NUMBER_LOSS, COPY_NUMBER_GAIN, INSERTION, DELETION, DUPLICATION, TANDEM_DUPLICATION, BREAKEND, e.g. SNV,INDEL. - :param str project: Project [user@]project where project can be either - the ID or the alias. + :param str project: Project [organization@]project where project can + be either the ID or the alias. :param str study: Filter variants from the given studies, these can be either the numeric ID or the alias with the format - user@project:study. + organization@project:study. :param str file: Filter variants from the files specified. This will set includeFile parameter when not provided. :param str filter: Specify the FILTER for any of the files. If 'file' @@ -973,14 +1097,21 @@ def run_sample_eligibility(self, data=None, **options): PATH: /{apiVersion}/analysis/variant/sample/eligibility/run :param dict data: . (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str job_id: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. :param str job_description: Job description. :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/sample/eligibility', data=data, **options) @@ -997,14 +1128,21 @@ def run_sample_qc(self, data=None, **options): skip some metrics, use the following keywords (separated by commas): variant-stats, signature, signature-catalogue, signature-fitting, genome-plot. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str job_id: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. :param str job_description: Job description. :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/sample/qc', data=data, **options) @@ -1030,14 +1168,21 @@ def run_sample(self, data=None, **options): PATH: /{apiVersion}/analysis/variant/sample/run :param dict data: Sample variant filter params. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str job_id: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. :param str job_description: Job description. :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/sample', data=data, **options) @@ -1055,8 +1200,8 @@ def query_sample_stats(self, sample, **options): SV, COPY_NUMBER, COPY_NUMBER_LOSS, COPY_NUMBER_GAIN, INSERTION, DELETION, DUPLICATION, TANDEM_DUPLICATION, BREAKEND, e.g. SNV,INDEL. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str file: Filter variants from the files specified. This will set includeFile parameter when not provided. :param str filter: Specify the FILTER for any of the files. If 'file' @@ -1096,14 +1241,21 @@ def run_sample_stats(self, data=None, **options): indexId='' to store the result in catalog sample QC. indexId=ALL requires an empty query. Use sample=all to compute sample stats of all samples in the variant storage. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str job_id: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. :param str job_description: Job description. :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/sample/stats', data=data, **options) @@ -1114,16 +1266,23 @@ def run_stats_export(self, data=None, **options): PATH: /{apiVersion}/analysis/variant/stats/export/run :param dict data: Variant stats export params. (REQUIRED) - :param str project: Project [user@]project where project can be either - the ID or the alias. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str project: Project [organization@]project where project can + be either the ID or the alias. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str job_id: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. :param str job_description: Job description. :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/stats/export', data=data, **options) @@ -1134,14 +1293,21 @@ def run_stats(self, data=None, **options): PATH: /{apiVersion}/analysis/variant/stats/run :param dict data: Variant stats params. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str job_id: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. :param str job_description: Job description. :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/stats', data=data, **options) diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py index a98de91c73a..f764a18198d 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py @@ -2,7 +2,6 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2024-04-25 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +13,6 @@ class VariantOperation(_ParentRestClient): """ This class contains methods for the 'Operations - Variant Storage' webservices - Client version: 2.12.4-SNAPSHOT PATH: /{apiVersion}/operation """ @@ -26,8 +24,8 @@ def configure_cellbase(self, data=None, **options): Update Cellbase configuration. PATH: /{apiVersion}/operation/cellbase/configure - :param str project: Project [user@]project where project can be either - the ID or the alias. + :param str project: Project [organization@]project where project can + be either the ID or the alias. :param bool annotation_update: Create and load variant annotations into the database. :param str annotation_save_id: Save a copy of the current variant @@ -49,8 +47,15 @@ def aggregate_variant(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param dict data: Variant aggregate params. """ @@ -67,8 +72,15 @@ def delete_variant_annotation(self, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. - :param str project: Project [user@]project where project can be either - the ID or the alias. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. + :param str project: Project [organization@]project where project can + be either the ID or the alias. :param str annotation_id: Annotation identifier. """ @@ -85,10 +97,17 @@ def index_variant_annotation(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. - :param str project: Project [user@]project where project can be either - the ID or the alias. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. + :param str project: Project [organization@]project where project can + be either the ID or the alias. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param dict data: Variant annotation index params. """ @@ -105,8 +124,15 @@ def save_variant_annotation(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. - :param str project: Project [user@]project where project can be either - the ID or the alias. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. + :param str project: Project [organization@]project where project can + be either the ID or the alias. :param dict data: Variant annotation save params. """ @@ -118,10 +144,10 @@ def configure_variant(self, data=None, **options): or Study level. PATH: /{apiVersion}/operation/variant/configure - :param str project: Project [user@]project where project can be either - the ID or the alias. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str project: Project [organization@]project where project can + be either the ID or the alias. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param dict data: Configuration params to update. """ @@ -138,8 +164,15 @@ def delete_variant(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param dict data: Variant delete file params. """ @@ -157,8 +190,15 @@ def aggregate_variant_family(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param dict data: Variant aggregate family params. """ @@ -175,8 +215,15 @@ def index_variant_family(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param dict data: Variant family index params. """ @@ -193,8 +240,15 @@ def index_variant(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param dict data: Variant index params. """ @@ -212,8 +266,15 @@ def launcher_variant_index(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param dict data: . """ @@ -233,6 +294,13 @@ def run_variant_julie(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. :param str project: project. """ @@ -249,6 +317,13 @@ def repair_variant_metadata(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. :param dict data: Variant storage metadata repair params. """ @@ -265,8 +340,15 @@ def synchronize_variant_metadata(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param dict data: Variant storage metadata synchronize params. """ @@ -283,6 +365,13 @@ def prune_variant(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. :param dict data: Variant prune params. Use dry-run to just generate a report with the orphan variants. """ @@ -300,8 +389,15 @@ def delete_variant_sample(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param dict data: Variant delete sample params. """ @@ -318,8 +414,15 @@ def index_variant_sample(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param dict data: Variant sample index params. """ @@ -330,8 +433,8 @@ def variant_sample_index_configure(self, data=None, **options): DEPRECATED You should use the new sample index configure method. PATH: /{apiVersion}/operation/variant/sample/index/configure - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param bool skip_rebuild: Skip sample index re-build. :param dict data: New SampleIndexConfiguration. """ @@ -349,8 +452,15 @@ def delete_variant_score(self, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str name: Unique name of the score within the study. :param bool resume: Resume a previously failed remove. :param bool force: Force remove of partially indexed scores. @@ -369,8 +479,15 @@ def index_variant_score(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param dict data: Variant score index params. scoreName: Unique name of the score within the study. cohort1: Cohort used to compute the score. Use the cohort 'ALL' if all samples from the study where @@ -403,10 +520,17 @@ def variant_secondary_annotation_index(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. - :param str project: Project [user@]project where project can be either - the ID or the alias. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. + :param str project: Project [organization@]project where project can + be either the ID or the alias. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param dict data: Variant secondary annotation index params. """ @@ -423,8 +547,15 @@ def variant_secondary_sample_index(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param dict data: Variant sample index params. """ @@ -435,8 +566,8 @@ def configure_variant_secondary_sample_index(self, data=None, **options): Update SampleIndex configuration (New!). PATH: /{apiVersion}/operation/variant/secondary/sample/index/configure - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param bool skip_rebuild: Skip sample index re-build. :param dict data: New SampleIndexConfiguration. """ @@ -454,10 +585,17 @@ def secondary_index_variant(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. - :param str project: Project [user@]project where project can be either - the ID or the alias. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. + :param str project: Project [organization@]project where project can + be either the ID or the alias. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param dict data: Variant secondary annotation index params. """ @@ -475,28 +613,55 @@ def delete_variant_secondary_index(self, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str samples: Samples to remove. Needs to provide all the samples in the secondary index. """ return self._delete(category='operation', resource='delete', subcategory='variant/secondaryIndex', **options) + def setup_variant(self, data=None, **options): + """ + Execute Variant Setup to allow using the variant engine. This setup is + necessary before starting any variant operation. + PATH: /{apiVersion}/operation/variant/setup + + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. + :param dict data: Variant setup params. + """ + + return self._post(category='operation', resource='setup', subcategory='variant', data=data, **options) + def delete_variant_stats(self, data=None, **options): """ Deletes the VariantStats of a cohort/s from the database. PATH: /{apiVersion}/operation/variant/stats/delete :param dict data: Variant stats delete params. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str job_id: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. :param str job_description: Job description. :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='operation', resource='delete', subcategory='variant/stats', data=data, **options) @@ -508,14 +673,21 @@ def index_variant_stats(self, data=None, **options): PATH: /{apiVersion}/operation/variant/stats/index :param dict data: Variant stats params. (REQUIRED) - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param str job_id: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. :param str job_description: Job description. :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='operation', resource='index', subcategory='variant/stats', data=data, **options) @@ -531,8 +703,15 @@ def delete_variant_study(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. - :param str study: Study [[user@]project:]study where study and project - can be either the ID or UUID. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. :param dict data: Variant delete study params. """ diff --git a/opencga-client/src/main/python/setup.py b/opencga-client/src/main/python/setup.py index 0764eeda3db..6be7c84c577 100644 --- a/opencga-client/src/main/python/setup.py +++ b/opencga-client/src/main/python/setup.py @@ -21,14 +21,13 @@ url='https://github.com/opencb/opencga/tree/develop/opencga-client/src/main/python/pyopencga', packages=['pyopencga', 'pyopencga.rest_clients'], license='Apache Software License', - author='David Gomez-Peregrina, Pablo Marin-Garcia, Daniel Perez-Gil', - author_email='david.gomez@mgviz.org, pmarin@kanteron.com, dp529@cam.ac.uk', + author='Pablo Marin-Garcia, Daniel Perez-Gil', + author_email='pablo.marin@zettagenomics.com, daniel.perez@zettagenomics.com', classifiers=[ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'Topic :: Scientific/Engineering :: Bio-Informatics', 'License :: OSI Approved :: Apache Software License', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.5', ], diff --git a/opencga-client/src/test/java/org/opencb/opencga/client/rest/UserClientTest.java b/opencga-client/src/test/java/org/opencb/opencga/client/rest/UserClientTest.java index 5dc26ee0c1a..4a701880700 100644 --- a/opencga-client/src/test/java/org/opencb/opencga/client/rest/UserClientTest.java +++ b/opencga-client/src/test/java/org/opencb/opencga/client/rest/UserClientTest.java @@ -22,11 +22,9 @@ import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.rules.ExpectedException; -import org.opencb.commons.datastore.core.Event; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.client.exceptions.ClientException; import org.opencb.opencga.client.rest.clients.UserClient; -import org.opencb.opencga.core.models.project.Project; import org.opencb.opencga.core.models.user.AuthenticationResponse; import org.opencb.opencga.core.models.user.LoginParams; import org.opencb.opencga.core.models.user.PasswordChangeParams; @@ -34,9 +32,6 @@ import org.opencb.opencga.core.response.RestResponse; import org.opencb.opencga.core.testclassification.duration.ShortTests; -import java.io.IOException; -import java.util.stream.Collectors; - import static org.junit.Assert.*; /** @@ -58,17 +53,17 @@ public void before() throws Throwable { @Test public void login() throws ClientException { - AuthenticationResponse response = openCGAClient.login("user1", "user1_pass"); + AuthenticationResponse response = openCGAClient.login(organizationId, "user1", "user1_pass"); assertEquals(response.getToken(), openCGAClient.getToken()); thrown.expect(CatalogException.class); thrown.expectMessage("Bad user or password"); - openCGAClient.login("user1", "wrong_password"); + openCGAClient.login(organizationId, "user1", "wrong_password"); } @Test public void logout() throws ClientException { - System.out.println("token = " + userClient.login(new LoginParams("user1", "user1_pass"), null).firstResult().getToken()); + System.out.println("token = " + userClient.login(new LoginParams(organizationId, "user1", "user1_pass"), null).firstResult().getToken()); assertNotNull(openCGAClient.getToken()); openCGAClient.logout(); assertEquals(null, openCGAClient.getToken()); @@ -81,33 +76,11 @@ public void get() throws Exception { assertEquals(1, login.allResultsSize()); } - @Test - public void getProjects() throws Exception { - RestResponse projects = userClient.projects(null, null); - assertEquals(1, projects.allResultsSize()); - } - - - @Test - public void getProjectsLogout() throws IOException, ClientException { - openCGAClient.logout(); - - RestResponse projects = userClient.projects("user1", null); - assertTrue(projects.getEvents().stream() - .filter(event -> event.getType() == Event.Type.ERROR) - .map(Event::getMessage) - .collect(Collectors.joining()).contains("session id")); - - thrown.expect(CatalogException.class); - thrown.expectMessage("Missing user id"); - userClient.projects(null, null); - } - @Test public void changePassword() throws Exception { userClient.password(new PasswordChangeParams("user1", "user1_pass", "user1_newPass")); String lastSessionId = openCGAClient.getToken(); - AuthenticationResponse response = openCGAClient.login(openCGAClient.getUserId(), "user1_newPass"); + AuthenticationResponse response = openCGAClient.login(organizationId, openCGAClient.getUserId(), "user1_newPass"); assertNotEquals(lastSessionId, response.getToken()); thrown.expect(CatalogException.class); diff --git a/opencga-client/src/test/java/org/opencb/opencga/client/rest/WorkEnvironmentTest.java b/opencga-client/src/test/java/org/opencb/opencga/client/rest/WorkEnvironmentTest.java index ea2124d51cb..9e1f4c14bd2 100644 --- a/opencga-client/src/test/java/org/opencb/opencga/client/rest/WorkEnvironmentTest.java +++ b/opencga-client/src/test/java/org/opencb/opencga/client/rest/WorkEnvironmentTest.java @@ -42,6 +42,7 @@ @Category(ShortTests.class) public class WorkEnvironmentTest extends ExternalResource { + protected String organizationId = "test"; protected OpenCGAClient openCGAClient; protected Path opencgaHome; protected CatalogManager catalogManager; @@ -90,14 +91,14 @@ private void isolateOpenCGA() throws Exception { catalogManager = new CatalogManager(configuration); - CatalogDemo.createDemoDatabase(catalogManager, TestParamConstants.ADMIN_PASSWORD, true); + CatalogDemo.createDemoDatabase(catalogManager, organizationId, TestParamConstants.ADMIN_PASSWORD, true); restServer = new RestServer(opencgaHome); restServer.start(); // catalogManager = new CatalogManager(configuration); clientConfiguration = ClientConfiguration.load(getClass().getResourceAsStream("/client-configuration-test.yml")); - openCGAClient = new OpenCGAClient("user1", "user1_pass", clientConfiguration); + openCGAClient = new OpenCGAClient(organizationId, "user1", "user1_pass", clientConfiguration); } @Override diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java index b104b073f68..dbde894f78f 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java @@ -37,6 +37,36 @@ public class FieldConstants { public static final String QUALITY_CONTROL_FILES_DESCRIPTION = "File IDs related to the quality control."; public static final String QUALITY_CONTROL_COMMENTS_DESCRIPTION = "Comments related to the quality control."; + // Organization + public static final String ORGANIZATION_ID_DESCRIPTION = "Organization unique identifier."; + public static final String ORGANIZATION_NAME_DESCRIPTION = "Organization name."; + public static final String ORGANIZATION_DOMAIN_DESCRIPTION = "Organization domain. Example: 'zettagenomics.com'."; + public static final String ORGANIZATION_OWNER_DESCRIPTION = "Owner of the organization."; + public static final String ORGANIZATION_ADMINS_DESCRIPTION = "Administrative users of the organization."; + public static final String ORGANIZATION_PROJECTS_DESCRIPTION = "Projects the organization holds."; + public static final String ORGANIZATION_NOTES_DESCRIPTION = "Notes of organization scope."; + // public static final String ORGANIZATION_AUTHENTICATION_ORIGINS_DESCRIPTION = "Authentication origins used by the organization. This " +// + "contains all the configuration necessary to be able to communicate with the external authentication origins."; + public static final String ORGANIZATION_CONFIGURATION_DESCRIPTION = "Organization configuration information."; + public static final String ORGANIZATION_INTERNAL_DESCRIPTION = "Organization internal information."; + + // Notes + public static final String NOTES_ID_DESCRIPTION = "Note unique identifier."; + public static final String NOTES_SCOPE_DESCRIPTION = "Scope of the Note."; + public static final String NOTES_STUDY_DESCRIPTION = "Study FQN if the Note scope is STUDY."; + public static final String NOTES_TAGS_DESCRIPTION = "Note tags."; + public static final String NOTES_USER_ID_DESCRIPTION = "User that wrote that Note."; + public static final String NOTES_VISIBILITY_DESCRIPTION = "Visibility of the Note."; + public static final String NOTES_VALUE_TYPE_DESCRIPTION = "Type of the value written in the Note."; + public static final String NOTES_VALUE_DESCRIPTION = "Value stored in the Note."; + + public static final String NOTES_ID_PARAM = "id"; + public static final String NOTES_SCOPE_PARAM = "scope"; + public static final String NOTES_STUDY_PARAM = "study"; + public static final String NOTES_TAGS_PARAM = "tags"; + public static final String NOTES_USER_ID_PARAM = "userId"; + public static final String NOTES_VISIBILITY_PARAM = "visibility"; + //Sample public static final String SAMPLE_ID_DESCRIPTION = "Sample ID."; public static final String SAMPLE_PROCESSING_DESCRIPTION = "Describes how the sample was processed in the lab."; @@ -309,16 +339,25 @@ public class FieldConstants { + "ERROR, UNKNOWN, REGISTERING, UNREGISTERED, ABORTED, DELETED."; public static final String JOB_INTERNAL_EVENTS_DESCRIPTION = "Events of the internal job."; public static final String JOB_INTERNAL_WEBHOOK_DESCRIPTION = "Job internal Webhook."; + public static final String JOB_INTERNAL_KILL_JOB_REQUESTED_DESCRIPTION = "Flag used to track whether a user has requested to" + + " terminate the execution of a job before its natural completion. When set to true, it indicates that a request has been" + + " made to prematurely terminate the ongoing job execution. Its value is typically checked periodically during the" + + " execution of the job to determine if termination is necessary."; //JobInternalWebhook public static final String JOB_INTERNAL_WEBHOOK_URL_DESCRIPTION = "Webhook URL."; public static final String JOB_INTERNAL_WEBHOOK_STATUS_DESCRIPTION = "Webhook status map can have the values SUCCESS or ERROR."; + public static final String JOB_PARENT_ID = "jobParentId"; + public static final String JOB_SCHEDULED_START_TIME = "jobScheduledStartTime"; + public static final String JOB_OUT_DIR_DESCRIPTION = "Output dir for the job."; public static final String JOB_INPUT_DESCRIPTION = "List of input files."; public static final String JOB_OUTPUT_DESCRIPTION = "List of output files."; public static final String JOB_TAGS_DESCRIPTION = "List of tags for the job."; public static final String JOB_DEPENDS_ON_DESCRIPTION = "List of jobs the current job depends on."; + public static final String JOB_PARENT_ID_DESCRIPTION = "Id of the job that generated this job (if any)"; + public static final String JOB_SCHEDULED_START_TIME_DESCRIPTION = "Time when the job is scheduled to start."; public static final String JOB_EXECUTION_DESCRIPTION = "Result of the execution."; //ExecutorInfo @@ -350,7 +389,7 @@ public class FieldConstants { public static final String JOB_STUDY_PARAM_OTHERS = "List of strings."; //Project - public static final String PROJECT_FQN = "Full Qualified Name (user@projectId)."; + public static final String PROJECT_FQN = "Full Qualified Name (organization@projectId)."; public static final String PROJECT_ORGANISM = "Organism to which the project belongs."; public static final String PROJECT_STUDIES = "Project study list."; @@ -380,6 +419,7 @@ public class FieldConstants { + "JOBS, CLINICAL_ANALYSES and DISEASE_PANELS. The value is a List of permission rules "; public static final String STUDY_URI = "Study uri"; public static final String STUDY_EXTERNAL_SOURCES = "A List with related external sources."; + public static final String STUDY_NOTES_DESCRIPTION = "Notes of scope=STUDY."; public static final String STUDY_TYPE = "Study type description"; //PermissionRule @@ -401,9 +441,16 @@ public class FieldConstants { public static final String USER_ACCOUNT = "User account."; //Account - public static final String ACCOUNT_TYPE = "User account type can have the values GUEST, FULL and ADMINISTRATOR."; - public static final String ACCOUNT_EXPIRATION_DATE_DESCRIPTION = "Date the account expires."; - public static final String ACCOUNT_AUTHENTICATION = "How the account is authenticated"; + public static final String INTERNAL_ACCOUNT_EXPIRATION_DATE_DESCRIPTION = "Date the account expires."; + public static final String INTERNAL_ACCOUNT_FAILED_ATTEMPTS_DESCRIPTION = "Number of consecutive failed attempts. When the user logs" + + " in successfully, this field is automatically reset to 0."; + public static final String INTERNAL_ACCOUNT_PASSWORD_DESCRIPTION = "Object containing the last time the password was changed and the" + + " expiration date."; + public static final String INTERNAL_ACCOUNT_PASSWORD_LAST_MODIFIED_DESCRIPTION = "Date the user's password was last changed. This " + + "field will be null if the user account origin is different from OpenCGA."; + public static final String INTERNAL_ACCOUNT_PASSWORD_EXPIRATION_DATE_DESCRIPTION = "Date the user's password expires. This field will" + + " be null if the user account origin is different from OpenCGA and the passwordExpiration feature is disabled."; + public static final String INTERNAL_ACCOUNT_AUTHENTICATION = "How the account is authenticated"; public static final String USER_QUOTA = "User quota"; public static final String USER_PROJECTS = "A List with related projects."; public static final String USER_SHARED_PROJECTS = "A List with shared projects."; @@ -464,7 +511,7 @@ public class FieldConstants { public static final String HRDETECT_CNV_QUERY_DESCRIPTION = "CNV query"; public static final String HRDETECT_INDEL_QUERY_DESCRIPTION = "INDEL query"; public static final String HRDETECT_SNV3_CUSTOM_NAME_DESCRIPTION = "Custom signature name that will be considered as SNV3 input for" - + " HRDetect."; + + " HRDetect."; public static final String HRDETECT_SNV8_CUSTOM_NAME_DESCRIPTION = "Custom signature name that will be considered as SNV8 input for" + " HRDetect."; public static final String HRDETECT_SV3_CUSTOM_NAME_DESCRIPTION = "Custom signature name that will be considered as SV3 input for" @@ -501,5 +548,10 @@ public class FieldConstants { public static final String ALIGNMENT_QC_OVERWRITE_DESCRIPTION = "To overwrite the QC metrics already computed."; // Exomiser + public static final String EXOMISER_CLINICAL_ANALYSIS_DESCRIPTION = "Clinical analysis ID."; + public static final String EXOMISER_SAMPLE_DESCRIPTION = "Sample ID."; public static final String EXOMISER_CLINICAL_ANALYSIS_TYPE_DESCRIPTION = "Clinical analysis type: SINGLE or FAMILY."; + public static final String EXOMISER_VERSION_DESCRIPTION = "Exomiser version in the format X.Y where X is the major version and Y the" + + " minor version, e.g.: 14.0. If the version is not specified, the default version will be used. Refer to the configuration" + + " file to view all installed Exomiser versions and identify the default version."; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java b/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java index 13f02ad0bae..d3739f43ce5 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java @@ -38,6 +38,10 @@ public class ParamConstants { public static final String RELEASE_DESCRIPTION = "Release when it was created"; public static final String INTERNAL_STATUS_PARAM = "internalStatus"; public static final String INTERNAL_STATUS_DESCRIPTION = "Filter by internal status"; + public static final String ENTRY_ID_LIST_DESCRIPTION = "Comma separated list of entry ids."; + public static final String ENTRY_ID_LIST = "entryIds"; + public static final String PERMISSION_LIST_DESCRIPTION = "Comma separated list of permissions to be retrieved."; + public static final String PERMISSION_LIST = "permissions"; private static final String REGEX_SUPPORT = ". Also admits basic regular expressions using the operator '~', " + "i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search."; @Deprecated // Use INTERNAL_VARIANT_INDEX_STATUS_PARAM @@ -73,7 +77,7 @@ public class ParamConstants { public static final String DISORDERS_DESCRIPTION = "Comma separated list of disorder ids or names" + REGEX_SUPPORT; public static final String BODY_PARAM = "body"; public static final String OVERWRITE = "overwrite"; - + private static final String UP_TO_100 = " up to a maximum of 100"; public static final String CELLBASE_URL = "https://ws.zettagenomics.com/cellbase"; public static final String CELLBASE_VERSION = "v5.8"; @@ -89,6 +93,9 @@ public class ParamConstants { public static final String RESUME_DESCRIPTION = "Resume a previously failed index operation"; + public static final String CATEGORY = "category"; + public static final String CATEGORY_DESCRIPTION = "Category corresponding to the id's provided."; + // --------------------------------------------- public static final String FORCE = "force"; public static final String ANNOTATION_DOC_URL = "http://docs.opencb.org/display/opencga/AnnotationSets+1.4.0"; @@ -116,14 +123,16 @@ public class ParamConstants { */ public static final String FLATTEN_ANNOTATIONS = "flattenAnnotations"; public static final String FLATTEN_ANNOTATION_DESCRIPTION = "Boolean indicating to flatten the annotations."; - public static final String USER_PROJECT_SEPARATOR = "@"; + public static final String ORGANIZATION_PROJECT_SEPARATOR = "@"; // --------------------------------------------- public static final String PROJECT_STUDY_SEPARATOR = ":"; public static final String OPENCGA_USER_ID = "opencga"; + public static final String ADMIN_ORGANIZATION = "opencga"; + public static final String OPENCGA_USER_FQN = ADMIN_ORGANIZATION + ":" + OPENCGA_USER_ID; public static final String ADMIN_PROJECT = "admin"; public static final String ADMIN_STUDY = "admin"; public static final String ADMIN_STUDY_FQN = - OPENCGA_USER_ID + USER_PROJECT_SEPARATOR + ADMIN_PROJECT + PROJECT_STUDY_SEPARATOR + ADMIN_STUDY; + OPENCGA_USER_ID + ORGANIZATION_PROJECT_SEPARATOR + ADMIN_PROJECT + PROJECT_STUDY_SEPARATOR + ADMIN_STUDY; public static final String ANONYMOUS_USER_ID = "*"; // Any user, authenticated or not public static final String REGISTERED_USERS = "REGISTERED"; // Any authenticated user public static final String MEMBERS_GROUP = "@members"; @@ -145,19 +154,22 @@ public class ParamConstants { public static final String DATE_DESCRIPTION = "Date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805"; public static final String USER = "user"; // --------------------------------------------- + public static final String ORGANIZATION_DESCRIPTION = "Organization id"; + public static final String ORGANIZATION = "organization"; + // --------------------------------------------- + public static final String USER_ID_DESCRIPTION = "Comma separated list user IDs" + UP_TO_100 + REGEX_SUPPORT; + public static final String USER_ID_PARAM = "id"; public static final String USER_DESCRIPTION = "User ID"; public static final String USERS_DESCRIPTION = "Comma separated list of user IDs"; - public static final String USER_ACCOUNT_TYPE = "account"; - public static final String USER_ACCOUNT_TYPE_DESCRIPTION = "Account type [GUEST, FULL, ADMINISTRATOR]"; public static final String USER_AUTHENTICATION_ORIGIN = "authenticationId"; public static final String USER_AUTHENTICATION_ORIGIN_DESCRIPTION = "Authentication origin ID"; public static final String USER_CREATION_DATE = "creationDate"; public static final String USER_CREATION_DATE_DESCRIPTION = CREATION_DATE_DESCRIPTION; public static final String PROJECT_PARAM = "project"; // --------------------------------------------- - public static final String PROJECT_DESCRIPTION = "Project [user@]project where project can be either the ID or the alias"; + public static final String PROJECT_DESCRIPTION = "Project [organization@]project where project can be either the ID or the alias"; public static final String STUDY_PARAM = "study"; - public static final String STUDY_DESCRIPTION = "Study [[user@]project:]study where study and project can be either the ID or UUID"; + public static final String STUDY_DESCRIPTION = "Study [[organization@]project:]study where study and project can be either the ID or UUID"; // --------------------------------------------- public static final String OTHER_STUDIES_FLAG = "otherStudies"; public static final String OTHER_STUDIES_FLAG_DESCRIPTION = "Flag indicating the entries being queried can belong to any related " + @@ -425,6 +437,9 @@ public class ParamConstants { public static final String CLINICAL_RELEASE_PARAM = RELEASE_PARAM; public static final String CLINICAL_STATUS_PARAM = STATUS_PARAM; public static final String CLINICAL_INTERNAL_STATUS_PARAM = INTERNAL_STATUS_PARAM; + public static final String CLINICAL_VERSION_PARAM = "version"; + public static final String CLINICAL_VERSION_DESCRIPTION = "Comma separated list of clinical versions. 'all' to get all the clinical" + + " versions. Not supported if multiple clinical ids are provided"; public static final String CLINICAL_TYPE_DESCRIPTION = "Clinical Analysis type"; public static final String CLINICAL_DISORDER_DESCRIPTION = "Clinical Analysis disorder" + REGEX_SUPPORT; public static final String CLINICAL_FILES_DESCRIPTION = "Clinical Analysis files"; @@ -455,6 +470,7 @@ public class ParamConstants { + "Interpretation object is passed."; public static final String INTERPRETATION_ID_PARAM = "id"; public static final String INTERPRETATION_UUID_PARAM = "uuid"; + public static final String INTERPRETATION_NAME_PARAM = "name"; public static final String INTERPRETATION_CLINICAL_ANALYSIS_ID_PARAM = "clinicalAnalysisId"; public static final String INTERPRETATION_ANALYST_ID_PARAM = "analystId"; public static final String INTERPRETATION_METHOD_NAME_PARAM = "methodName"; @@ -548,6 +564,7 @@ public class ParamConstants { public static final String JOB_USER_PARAM = "userId"; public static final String JOB_USER_DESCRIPTION = "User that created the job"; public static final String JOB_PRIORITY_PARAM = "priority"; + public static final String SUBMIT_JOB_PRIORITY_PARAM = "jobPriority"; public static final String JOB_PRIORITY_DESCRIPTION = "Priority of the job"; public static final String JOB_INTERNAL_STATUS_PARAM = INTERNAL_STATUS_PARAM; public static final String JOB_INTERNAL_STATUS_DESCRIPTION = INTERNAL_STATUS_DESCRIPTION; @@ -558,6 +575,12 @@ public class ParamConstants { public static final String JOB_TAGS = "jobTags"; public static final String JOB_TAGS_PARAM = "tags"; public static final String JOB_TAGS_DESCRIPTION = "Job tags"; + public static final String JOB_SCHEDULED_START_TIME_DESCRIPTION = FieldConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION; + public static final String JOB_SCHEDULED_START_TIME = FieldConstants.JOB_SCHEDULED_START_TIME; + public static final String JOB_DRY_RUN = "jobDryRun"; + public static final String JOB_DRY_RUN_DESCRIPTION = "Flag indicating that the job will be executed in dry-run mode. In this mode," + + " OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will" + + " not actually run."; // --------------------------------------------- public static final String JOB_INPUT_FILES_PARAM = "input"; @@ -1477,7 +1500,6 @@ public class ParamConstants { public static final String ADMIN_CATALOG_INSTALL_EMAIL = "The body web service email parameter"; public static final String ADMIN_CATALOG_INSTALL_ORGANIZATION = "The body web service organization parameter"; public static final String ADMIN_CATALOG_JWT_SECRETKEY = "The body web service secretKey parameter"; - private static final String UP_TO_100 = " up to a maximum of 100"; public static final String FILES_DESCRIPTION = "Comma separated list of file IDs or names" + UP_TO_100; public static final String FILES_ID_DESCRIPTION = "Comma separated list of file IDs" + UP_TO_100 + REGEX_SUPPORT; public static final String FILES_UUID_DESCRIPTION = "Comma separated list file UUIDs" + UP_TO_100; @@ -1501,6 +1523,7 @@ public class ParamConstants { public static final String CLINICAL_ANALYSES_DESCRIPTION = "Comma separated list of clinical analysis IDs or names" + UP_TO_100; public static final String INTERPRETATION_ID_DESCRIPTION = "Comma separated list of Interpretation IDs" + UP_TO_100 + REGEX_SUPPORT; public static final String INTERPRETATION_UUID_DESCRIPTION = "Comma separated list of Interpretation UUIDs" + UP_TO_100; + public static final String INTERPRETATION_NAME_DESCRIPTION = "Comma separated list of Interpretation names" + UP_TO_100; public static final String INTERPRETATION_DESCRIPTION = "Comma separated list of clinical interpretation IDs " + UP_TO_100; public static final String PANEL_ID_DESCRIPTION = "Comma separated list of panel IDs " + UP_TO_100 + REGEX_SUPPORT; public static final String PANEL_UUID_DESCRIPTION = "Comma separated list of panel UUIDs " + UP_TO_100; @@ -1509,9 +1532,9 @@ public class ParamConstants { public static final String JOB_IDS_DESCRIPTION = "Comma separated list of job IDs" + UP_TO_100 + REGEX_SUPPORT; public static final String JOB_UUIDS_DESCRIPTION = "Comma separated list of job UUIDs" + UP_TO_100; // --------------------------------------------- - public static final String PROJECTS_DESCRIPTION = "Comma separated list of projects [user@]project" + UP_TO_100; + public static final String PROJECTS_DESCRIPTION = "Comma separated list of projects [organization@]project" + UP_TO_100; // --------------------------------------------- - public static final String STUDIES_DESCRIPTION = "Comma separated list of Studies [[user@]project:]study where study and project can " + - "be either the ID or UUID" + UP_TO_100; + public static final String STUDIES_DESCRIPTION = "Comma separated list of Studies [[organization@]project:]study where study " + + "and project can be either the ID or UUID" + UP_TO_100; public static final String PANELS_DESCRIPTION = "Comma separated list of panel IDs" + UP_TO_100; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/cellbase/CellBaseValidator.java b/opencga-core/src/main/java/org/opencb/opencga/core/cellbase/CellBaseValidator.java index 4dd5543e9b8..e6b7f7353c8 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/cellbase/CellBaseValidator.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/cellbase/CellBaseValidator.java @@ -23,6 +23,7 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.concurrent.Callable; import java.util.stream.Collectors; @@ -149,7 +150,7 @@ private CellBaseConfiguration validate(boolean autoComplete) throws IOException String inputVersion = getVersion(); CellBaseDataResponse species; try { - species = cellBaseClient.getMetaClient().species(); + species = retryMetaSpecies(); } catch (RuntimeException e) { throw new IllegalArgumentException("Unable to access cellbase url '" + getURL() + "', version '" + inputVersion + "'", e); } @@ -158,7 +159,7 @@ private CellBaseConfiguration validate(boolean autoComplete) throws IOException // Version might be missing the starting "v" cellBaseConfiguration.setVersion("v" + cellBaseConfiguration.getVersion()); cellBaseClient = newCellBaseClient(cellBaseConfiguration, getSpecies(), getAssembly()); - species = cellBaseClient.getMetaClient().species(); + species = retryMetaSpecies(); } } if (species == null || species.firstResult() == null) { @@ -308,7 +309,7 @@ private static String majorMinor(String version) { public String getVersionFromServer() throws IOException { if (serverVersion == null) { synchronized (this) { - ObjectMap result = retryMetaAbout(3); + ObjectMap result = retryMetaAbout(); if (result == null) { throw new IOException("Unable to get version from server for cellbase " + toString()); } @@ -322,12 +323,43 @@ public String getVersionFromServer() throws IOException { return serverVersion; } - private ObjectMap retryMetaAbout(int retries) throws IOException { - ObjectMap result = cellBaseClient.getMetaClient().about().firstResult(); - if (result == null && retries > 0) { - // Retry - logger.warn("Unable to get version from server for cellbase " + toString() + ". Retrying..."); - result = retryMetaAbout(retries - 1); + private ObjectMap retryMetaAbout() throws IOException { + return retry(3, () -> cellBaseClient.getMetaClient().about().firstResult()); + } + + private CellBaseDataResponse retryMetaSpecies() throws IOException { + return retry(3, () -> cellBaseClient.getMetaClient().species()); + } + + private T retry(int retries, Callable function) throws IOException { + if (retries <= 0) { + return null; + } + T result = null; + Exception e = null; + try { + result = function.call(); + } catch (Exception e1) { + e = e1; + } + if (result == null) { + try { + // Retry + logger.warn("Unable to get reach cellbase " + toString() + ". Retrying..."); + result = retry(retries - 1, function); + } catch (Exception e1) { + if (e == null) { + e = e1; + } else { + e.addSuppressed(e1); + } + if (e instanceof IOException) { + throw (IOException) e; + } else { + throw new IOException("Error reading from cellbase " + toString(), e); + } + } + } return result; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/common/IOUtils.java b/opencga-core/src/main/java/org/opencb/opencga/core/common/IOUtils.java index a14652c67cd..e37374e76ea 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/common/IOUtils.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/common/IOUtils.java @@ -375,6 +375,17 @@ public static String humanReadableByteCount(long bytes, boolean si) { * @return number of bytes */ public static long fromHumanReadableToByte(String value) { + return fromHumanReadableToByte(value, false); + } + + /** + * Get Bytes numbers from a human-readable string + * + * @param value Human-readable value + * @param assumeBinary Use Binary Units (power of 2) + * @return number of bytes + */ + public static long fromHumanReadableToByte(String value, boolean assumeBinary) { if (value.endsWith("B")) { value = value.substring(0, value.length() - 1); } @@ -385,8 +396,11 @@ public static long fromHumanReadableToByte(String value) { } else { si = true; } + if (assumeBinary) { + si = false; + } int unit = si ? 1000 : 1024; - int exp = (si ? "kMGTPE" : "KMGTPE").indexOf(value.charAt(value.length() - 1)) + 1; + int exp = "KMGTPE".indexOf(value.toUpperCase().charAt(value.length() - 1)) + 1; if (exp > 0) { value = value.substring(0, value.length() - 1); } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/common/MailUtils.java b/opencga-core/src/main/java/org/opencb/opencga/core/common/MailUtils.java index bf62bb7c09f..0a5a1ad4f9f 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/common/MailUtils.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/common/MailUtils.java @@ -16,8 +16,6 @@ package org.opencb.opencga.core.common; -import org.opencb.opencga.core.models.user.User; -import org.opencb.opencga.core.response.OpenCGAResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,8 +32,6 @@ public class MailUtils { private static final Logger logger = LoggerFactory.getLogger(MailUtils.class); - - public static void sendResetPasswordMail(String to, String newPassword, final String mailUser, final String mailPassword, String mailHost, String mailPort, String userId) throws Exception { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/common/PasswordUtils.java b/opencga-core/src/main/java/org/opencb/opencga/core/common/PasswordUtils.java index 17255ae3255..91fb20b8455 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/common/PasswordUtils.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/common/PasswordUtils.java @@ -1,5 +1,6 @@ package org.opencb.opencga.core.common; +import org.passay.CharacterData; import org.passay.*; import java.util.ArrayList; @@ -7,6 +8,12 @@ public class PasswordUtils { + public static final int MIN_STRONG_PASSWORD_LENGTH = 8; + public static final int DEFAULT_PASSWORD_LENGTH = 10; + public static final int DEFAULT_SALT_LENGTH = 32; + public static final String PASSWORD_REQUIREMENT = "Password must contain at least " + MIN_STRONG_PASSWORD_LENGTH + + " characters, including at least one uppercase letter, one lowercase letter, one digit and one special character."; + private final static CharacterRule SPECIAL_CHARACTER_RULE = new CharacterRule(new CharacterData() { @Override public String getErrorCode() { @@ -20,7 +27,11 @@ public String getCharacters() { }); public static String getStrongRandomPassword() { - return getStrongRandomPassword(10); + return getStrongRandomPassword(DEFAULT_PASSWORD_LENGTH); + } + + public static String getStrongRandomSalt() { + return getStrongRandomPassword(DEFAULT_SALT_LENGTH); } public static String getStrongRandomPassword(int length) { @@ -33,14 +44,10 @@ public static String getStrongRandomPassword(int length) { } public static boolean isStrongPassword(String password) { - return isStrongPassword(password, 8); - } - - public static boolean isStrongPassword(String password, int minLength) { List rules = new ArrayList<>(); //Rule 1: Password length should be in between - //minLength and 100 characters - rules.add(new LengthRule(minLength, 100)); + //MIN_STRONG_PASSWORD_LENGTH and 100 characters + rules.add(new LengthRule(MIN_STRONG_PASSWORD_LENGTH, 100)); //Rule 2: No whitespace allowed rules.add(new WhitespaceRule()); //Rule 3.a: At least one Upper-case character diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/common/TimeUtils.java b/opencga-core/src/main/java/org/opencb/opencga/core/common/TimeUtils.java index 65c2cfda80b..c0574f0afdf 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/common/TimeUtils.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/common/TimeUtils.java @@ -127,6 +127,21 @@ public static Date add1MonthtoDate(Date date) { return new Date(cal.getTimeInMillis()); } + public static Date add1YeartoDate(Date date) { + Calendar cal = new GregorianCalendar(); + cal.setTime(date); + cal.setTimeInMillis(date.getTime()); + cal.add(Calendar.YEAR, 1); + return new Date(cal.getTimeInMillis()); + } + + public static Date addDaysToCurrentDate(int days) { + Calendar cal = new GregorianCalendar(); + cal.setTime(new Date()); + cal.add(Calendar.DAY_OF_YEAR, days); + return new Date(cal.getTimeInMillis()); + } + public static Date toDate(String dateStr) { Date date = null; try { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/AccountConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/AccountConfiguration.java new file mode 100644 index 00000000000..f467a93d37d --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/AccountConfiguration.java @@ -0,0 +1,42 @@ +package org.opencb.opencga.core.config; + +public class AccountConfiguration { + + private int maxLoginAttempts; + private int passwordExpirationDays; + + public AccountConfiguration() { + } + + public AccountConfiguration(int maxLoginAttempts, int passwordExpirationDays) { + this.maxLoginAttempts = maxLoginAttempts; + this.passwordExpirationDays = passwordExpirationDays; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("AccountConfiguration{"); + sb.append("maxLoginAttempts=").append(maxLoginAttempts); + sb.append(", passwordExpirationDays=").append(passwordExpirationDays); + sb.append('}'); + return sb.toString(); + } + + public int getMaxLoginAttempts() { + return maxLoginAttempts; + } + + public AccountConfiguration setMaxLoginAttempts(int maxLoginAttempts) { + this.maxLoginAttempts = maxLoginAttempts; + return this; + } + + public int getPasswordExpirationDays() { + return passwordExpirationDays; + } + + public AccountConfiguration setPasswordExpirationDays(int passwordExpirationDays) { + this.passwordExpirationDays = passwordExpirationDays; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/Analysis.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/Analysis.java index 7d19bd65184..9f1c7b32683 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/config/Analysis.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/Analysis.java @@ -17,23 +17,40 @@ package org.opencb.opencga.core.config; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; - public class Analysis { + private List packages; + private String scratchDir; + private String resourceUrl; + + private String opencgaExtTools; + private List tools; private Execution execution; private List frameworks; public Analysis() { + packages = new ArrayList<>(); + tools = new ArrayList<>(); execution = new Execution(); frameworks = new ArrayList<>(); } + public List getPackages() { + return packages; + } + + public Analysis setPackages(List packages) { + this.packages = packages; + return this; + } + public String getScratchDir() { return scratchDir; } @@ -43,6 +60,33 @@ public Analysis setScratchDir(String scratchDir) { return this; } + public String getResourceUrl() { + return resourceUrl; + } + + public Analysis setResourceUrl(String resourceUrl) { + this.resourceUrl = resourceUrl; + return this; + } + + public String getOpencgaExtTools() { + return opencgaExtTools; + } + + public Analysis setOpencgaExtTools(String opencgaExtTools) { + this.opencgaExtTools = opencgaExtTools; + return this; + } + + public List getTools() { + return tools; + } + + public Analysis setTools(List tools) { + this.tools = tools; + return this; + } + public Execution getExecution() { return execution; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/AnalysisTool.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/AnalysisTool.java new file mode 100644 index 00000000000..3744915b6b9 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/AnalysisTool.java @@ -0,0 +1,110 @@ +/* + * Copyright 2015-2020 OpenCB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opencb.opencga.core.config; + +import java.util.HashMap; +import java.util.Map; + +public class AnalysisTool { + + private String id; + private String version; + private boolean defaultVersion; + private String dockerId; + private String params; + private Map resources; + + public AnalysisTool() { + resources = new HashMap<>(); + } + + public AnalysisTool(String id, String version, boolean defaultVersion, String dockerId, String params, Map resources) { + this.id = id; + this.version = version; + this.defaultVersion = defaultVersion; + this.dockerId = dockerId; + this.params = params; + this.resources = resources; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("AnalysisTool{"); + sb.append("id='").append(id).append('\''); + sb.append(", version='").append(version).append('\''); + sb.append(", defaultVersion=").append(defaultVersion); + sb.append(", dockerId='").append(dockerId).append('\''); + sb.append(", params='").append(params).append('\''); + sb.append(", resources=").append(resources); + sb.append('}'); + return sb.toString(); + } + + public String getId() { + return id; + } + + public AnalysisTool setId(String id) { + this.id = id; + return this; + } + + public String getVersion() { + return version; + } + + public AnalysisTool setVersion(String version) { + this.version = version; + return this; + } + + public boolean isDefaultVersion() { + return defaultVersion; + } + + public AnalysisTool setDefaultVersion(boolean defaultVersion) { + this.defaultVersion = defaultVersion; + return this; + } + + public String getDockerId() { + return dockerId; + } + + public AnalysisTool setDockerId(String dockerId) { + this.dockerId = dockerId; + return this; + } + + public String getParams() { + return params; + } + + public AnalysisTool setParams(String params) { + this.params = params; + return this; + } + + public Map getResources() { + return resources; + } + + public AnalysisTool setResources(Map resources) { + this.resources = resources; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/Authentication.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/Authentication.java index d6ee047e1e3..9b2c7b74585 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/config/Authentication.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/Authentication.java @@ -16,6 +16,7 @@ package org.opencb.opencga.core.config; +import java.util.LinkedList; import java.util.List; /** @@ -23,10 +24,11 @@ */ public class Authentication { - private Long expiration; + private Long expiration; // Expiration time in seconds private List authenticationOrigins; public Authentication() { + this(3600L, new LinkedList<>()); } public Authentication(Long expiration, List authenticationOrigins) { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/AuthenticationOrigin.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/AuthenticationOrigin.java index 4b3a8b29532..2b4962b024a 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/config/AuthenticationOrigin.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/AuthenticationOrigin.java @@ -16,7 +16,6 @@ package org.opencb.opencga.core.config; -import java.util.Collections; import java.util.Map; /** @@ -32,7 +31,8 @@ public class AuthenticationOrigin { public enum AuthenticationType { OPENCGA, LDAP, - AzureAD + AzureAD, + SSO } // Possible keys of the options map @@ -51,21 +51,20 @@ public enum AuthenticationType { public static final String CONNECTION_TIMEOUT = "connectionTimeout"; public AuthenticationOrigin() { - this("internal", AuthenticationType.OPENCGA.name(), "localhost", Collections.emptyMap()); } - public AuthenticationOrigin(String id, String type, String host, Map options) { + public AuthenticationOrigin(String id, AuthenticationType type, String host, Map options) { this.id = id; - this.type = AuthenticationType.valueOf(type); + this.type = type; this.host = host; this.options = options; } @Override public String toString() { - final StringBuilder sb = new StringBuilder("Auth{"); + final StringBuilder sb = new StringBuilder("AuthenticationOrigin{"); sb.append("id='").append(id).append('\''); - sb.append(", type='").append(type).append('\''); + sb.append(", type=").append(type); sb.append(", host='").append(host).append('\''); sb.append(", options=").append(options); sb.append('}'); @@ -99,6 +98,21 @@ public AuthenticationOrigin setHost(String host) { return this; } + @Deprecated + public AuthenticationOrigin setAlgorithm(String algorithm) { + return this; + } + + @Deprecated + public AuthenticationOrigin setSecretKey(String secretKey) { + return this; + } + + @Deprecated + public AuthenticationOrigin setExpiration(long expiration) { + return this; + } + public Map getOptions() { return options; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/Catalog.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/Catalog.java index 65fe5b7a4b8..0df3425dfba 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/config/Catalog.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/Catalog.java @@ -16,27 +16,33 @@ package org.opencb.opencga.core.config; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * Created by pfurio on 01/02/17. */ public class Catalog { private DatabaseCredentials database; - private DatabaseCredentials searchEngine; + + private static final Logger logger; + + static { + logger = LoggerFactory.getLogger(Catalog.class); + } public Catalog() { } - public Catalog(DatabaseCredentials database, DatabaseCredentials searchEngine) { + public Catalog(DatabaseCredentials database) { this.database = database; - this.searchEngine = searchEngine; } @Override public String toString() { final StringBuilder sb = new StringBuilder("Catalog{"); sb.append("database=").append(database); - sb.append(", searchEngine=").append(searchEngine); sb.append('}'); return sb.toString(); } @@ -50,12 +56,15 @@ public Catalog setDatabase(DatabaseCredentials database) { return this; } + @Deprecated public DatabaseCredentials getSearchEngine() { - return searchEngine; + return null; } + @Deprecated public Catalog setSearchEngine(DatabaseCredentials searchEngine) { - this.searchEngine = searchEngine; + logger.warn("Ignored configuration option 'configuration.yml#catalog.searchEngine' with value '{}'." + + " The option was deprecated and removed.", searchEngine); return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/Configuration.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/Configuration.java index be957b160a7..b7b869c728c 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/config/Configuration.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/Configuration.java @@ -40,14 +40,12 @@ public class Configuration { */ private String logDir; - @Deprecated - private boolean openRegister; - private String databasePrefix; private String workspace; private String jobDir; - private Admin admin; + private AccountConfiguration account; + private Monitor monitor; private HealthCheck healthCheck; private Audit audit; @@ -59,15 +57,12 @@ public class Configuration { private Analysis analysis; private Panel panel; - private Optimizations optimizations; - private ServerConfiguration server; - private Authentication authentication; - - private static Logger logger; private static final Set reportedFields = new HashSet<>(); + private static final Logger logger; + private static final String DEFAULT_CONFIGURATION_FORMAT = "yaml"; static { @@ -75,7 +70,6 @@ public class Configuration { } public Configuration() { - admin = new Admin(); monitor = new Monitor(); healthCheck = new HealthCheck(); audit = new Audit(); @@ -84,9 +78,8 @@ public Configuration() { catalog = new Catalog(); analysis = new Analysis(); panel = new Panel(); - optimizations = new Optimizations(); server = new ServerConfiguration(); - authentication = new Authentication(); + account = new AccountConfiguration(); } public void serialize(OutputStream configurationOututStream) throws IOException { @@ -121,12 +114,26 @@ public static Configuration load(InputStream configurationInputStream, String fo } catch (IOException e) { throw new IOException("Configuration file could not be parsed: " + e.getMessage(), e); } + addDefaultValueIfMissing(configuration); // We must always overwrite configuration with environment parameters overwriteWithEnvironmentVariables(configuration); return configuration; } + private static void addDefaultValueIfMissing(Configuration configuration) { + if (configuration.getAccount() == null) { + configuration.setAccount(new AccountConfiguration()); + } + if (configuration.getAccount().getMaxLoginAttempts() <= 0) { + configuration.getAccount().setMaxLoginAttempts(5); + } + if (configuration.getAccount().getPasswordExpirationDays() < 0) { + // Disable password expiration by default + configuration.getAccount().setPasswordExpirationDays(0); + } + } + private static void overwriteWithEnvironmentVariables(Configuration configuration) { Map envVariables = System.getenv(); for (String variable : envVariables.keySet()) { @@ -146,6 +153,10 @@ private static void overwriteWithEnvironmentVariables(Configuration configuratio case "OPENCGA_MONITOR_PORT": configuration.getMonitor().setPort(Integer.parseInt(value)); break; + case "OPENCGA.MAX_LOGIN_ATTEMPTS": + case "OPENCGA.ACCOUNT.MAX_LOGIN_ATTEMPTS": + configuration.getAccount().setMaxLoginAttempts(Integer.parseInt(value)); + break; case "OPENCGA_EXECUTION_MODE": case "OPENCGA_EXECUTION_ID": configuration.getAnalysis().getExecution().setId(value); @@ -177,18 +188,6 @@ private static void overwriteWithEnvironmentVariables(Configuration configuratio case "OPENCGA_CATALOG_DB_CONNECTIONS_PER_HOST": configuration.getCatalog().getDatabase().getOptions().put("connectionsPerHost", value); break; - case "OPENCGA_CATALOG_SEARCH_HOST": - configuration.getCatalog().getSearchEngine().setHosts(Collections.singletonList(value)); - break; - case "OPENCGA_CATALOG_SEARCH_TIMEOUT": - configuration.getCatalog().getSearchEngine().getOptions().put("timeout", value); - break; - case "OPENCGA_CATALOG_SEARCH_BATCH": - configuration.getCatalog().getSearchEngine().getOptions().put("insertBatchSize", value); - break; - case "OPENCGA_OPTIMIZATIONS_SIMPLIFY_PERMISSIONS": - configuration.getOptimizations().setSimplifyPermissions(Boolean.parseBoolean(value)); - break; case "OPENCGA_SERVER_REST_PORT": configuration.getServer().getRest().setPort(Integer.parseInt(value)); break; @@ -212,6 +211,16 @@ public static void reportUnusedField(String field, Object value) { } } + public static void reportMovedField(String previousField, String newField, Object value) { + // Report only if the value is not null and not an empty string + if (value != null && !(value instanceof String && ((String) value).isEmpty())) { + if (reportedFields.add(previousField)) { + // Only log the first time a field is found + logger.warn("Option '{}' with value '{}' was moved to '{}'.", previousField, value, newField); + } + } + } + @Override public String toString() { final StringBuilder sb = new StringBuilder("Configuration{"); @@ -219,16 +228,17 @@ public String toString() { sb.append(", logDir='").append(logDir).append('\''); sb.append(", databasePrefix='").append(databasePrefix).append('\''); sb.append(", workspace='").append(workspace).append('\''); - sb.append(", admin=").append(admin); + sb.append(", jobDir='").append(jobDir).append('\''); + sb.append(", account=").append(account); sb.append(", monitor=").append(monitor); + sb.append(", healthCheck=").append(healthCheck); sb.append(", audit=").append(audit); sb.append(", hooks=").append(hooks); sb.append(", email=").append(email); sb.append(", catalog=").append(catalog); + sb.append(", analysis=").append(analysis); sb.append(", panel=").append(panel); - sb.append(", optimizations=").append(optimizations); sb.append(", server=").append(server); - sb.append(", authentication=").append(authentication); sb.append('}'); return sb.toString(); } @@ -263,13 +273,13 @@ public Configuration setLogFile(String logFile) { } @Deprecated - public boolean isOpenRegister() { - return openRegister; + public Boolean isOpenRegister() { + return null; } @Deprecated public Configuration setOpenRegister(boolean openRegister) { - this.openRegister = openRegister; + reportUnusedField("configuration.yml#openRegister", openRegister); return this; } @@ -300,12 +310,35 @@ public Configuration setJobDir(String jobDir) { return this; } + public AccountConfiguration getAccount() { + return account; + } + + public Configuration setAccount(AccountConfiguration account) { + this.account = account; + return this; + } + + @Deprecated + public int getMaxLoginAttempts() { + return account.getMaxLoginAttempts(); + } + + @Deprecated + public Configuration setMaxLoginAttempts(int maxLoginAttempts) { + reportMovedField("configuration.yml#maxLoginAttempts", "configuration.yml#account.maxLoginAttempts", maxLoginAttempts); + account.setMaxLoginAttempts(maxLoginAttempts); + return this; + } + + @Deprecated public Admin getAdmin() { - return admin; + return null; } + @Deprecated public Configuration setAdmin(Admin admin) { - this.admin = admin; + reportUnusedField("configuration.yml#admin", admin); return this; } @@ -383,12 +416,14 @@ public Configuration setPanel(Panel panel) { return this; } + @Deprecated public Optimizations getOptimizations() { - return optimizations; + return null; } + @Deprecated public Configuration setOptimizations(Optimizations optimizations) { - this.optimizations = optimizations; + reportUnusedField("configuration.yml#optimizations", optimizations); return this; } @@ -401,12 +436,15 @@ public Configuration setServer(ServerConfiguration server) { return this; } + @Deprecated public Authentication getAuthentication() { - return authentication; + return null; } - public void setAuthentication(Authentication authentication) { - this.authentication = authentication; + @Deprecated + public Configuration setAuthentication(Authentication authentication) { + reportUnusedField("configuration.yml#authentication", authentication); + return this; } public HealthCheck getHealthCheck() { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/DatabaseCredentials.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/DatabaseCredentials.java index 2c4e04ea6b9..76a560f6ce8 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/config/DatabaseCredentials.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/DatabaseCredentials.java @@ -43,6 +43,13 @@ public DatabaseCredentials(List hosts, String user, String password) { this(hosts, user, password, new HashMap<>()); } + public DatabaseCredentials(DatabaseCredentials credentials) { + this.hosts = credentials.hosts; + this.user = credentials.user; + this.password = credentials.password; + this.options = credentials.options; + } + public DatabaseCredentials(List hosts, String user, String password, Map options) { this.hosts = hosts; this.user = user; @@ -64,34 +71,38 @@ public List getHosts() { return hosts; } - public void setHosts(List hosts) { + public DatabaseCredentials setHosts(List hosts) { this.hosts = hosts == null ? java.util.Collections.emptyList() : hosts.stream().flatMap(s -> java.util.Arrays.stream(s.split(","))).collect(java.util.stream.Collectors.toList()); + return this; } public String getUser() { return user; } - public void setUser(String user) { + public DatabaseCredentials setUser(String user) { this.user = user; + return this; } public String getPassword() { return password; } - public void setPassword(String password) { + public DatabaseCredentials setPassword(String password) { this.password = password; + return this; } public Map getOptions() { return options; } - public void setOptions(Map options) { + public DatabaseCredentials setOptions(Map options) { this.options = options; + return this; } } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/Optimizations.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/Optimizations.java index 35207a4c484..777af21c1a3 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/config/Optimizations.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/Optimizations.java @@ -9,6 +9,7 @@ public class Optimizations { private boolean simplifyPermissions; public Optimizations() { + this(false); } public Optimizations(boolean simplifyPermissions) { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/Acl.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/Acl.java new file mode 100644 index 00000000000..0552db21f37 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/Acl.java @@ -0,0 +1,108 @@ +package org.opencb.opencga.core.models; + +import java.util.List; + +public class Acl { + + private String id; + private String type; + private List permissions; + private long timestamp; + + public Acl() { + } + + public Acl(String id, String type, List permissions, long timestamp) { + this.id = id; + this.type = type; + this.permissions = permissions; + this.timestamp = timestamp; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("Acl{"); + sb.append("id='").append(id).append('\''); + sb.append(", type='").append(type).append('\''); + sb.append(", permissions=").append(permissions); + sb.append(", timestamp=").append(timestamp); + sb.append('}'); + return sb.toString(); + } + + public String getId() { + return id; + } + + public Acl setId(String id) { + this.id = id; + return this; + } + + public String getType() { + return type; + } + + public Acl setType(String type) { + this.type = type; + return this; + } + + public List getPermissions() { + return permissions; + } + + public Acl setPermissions(List permissions) { + this.permissions = permissions; + return this; + } + + public long getTimestamp() { + return timestamp; + } + + public Acl setTimestamp(long timestamp) { + this.timestamp = timestamp; + return this; + } + + public static class Permission { + private String id; + private List userIds; + + public Permission() { + } + + public Permission(String id, List userIds) { + this.id = id; + this.userIds = userIds; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("Permission{"); + sb.append("id='").append(id).append('\''); + sb.append(", userIds=").append(userIds); + sb.append('}'); + return sb.toString(); + } + + public String getId() { + return id; + } + + public Permission setId(String id) { + this.id = id; + return this; + } + + public List getUserIds() { + return userIds; + } + + public Permission setUserIds(List userIds) { + this.userIds = userIds; + return this; + } + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/AclEntryList.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/AclEntryList.java index 4d221ade795..393cf86549f 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/AclEntryList.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/AclEntryList.java @@ -21,7 +21,7 @@ public AclEntryList setId(String id) { @Override public String toString() { - final StringBuilder sb = new StringBuilder("SampleAclEntryList{"); + final StringBuilder sb = new StringBuilder("AclEntryList{"); sb.append("id='").append(id).append('\''); sb.append(", acl=").append(acl); sb.append('}'); diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/JwtPayload.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/JwtPayload.java new file mode 100644 index 00000000000..4d8df0f6f0a --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/JwtPayload.java @@ -0,0 +1,136 @@ +package org.opencb.opencga.core.models; + +import com.fasterxml.jackson.core.JsonProcessingException; +import org.apache.commons.lang3.StringUtils; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.opencga.core.common.JacksonUtils; +import org.opencb.opencga.core.config.AuthenticationOrigin; + +import java.util.Base64; +import java.util.Date; + +public class JwtPayload { + + private final String userId; + private final String organization; + private final AuthenticationOrigin.AuthenticationType authOrigin; + private final String issuer; // Issuer of the JWT token. + private final Date issuedAt; // Time when the JWT was issued. + private final Date expirationTime; // Expiration time of the JWT. + private final String token; + + public static final String AUTH_ORIGIN = "authOrigin"; + + public JwtPayload(String userId, String organization, AuthenticationOrigin.AuthenticationType authOrigin, String issuer, Date issuedAt, + Date expirationTime, String token) { + this.token = token; + this.userId = userId; + this.organization = organization; + this.authOrigin = authOrigin; + this.issuer = issuer; + this.issuedAt = issuedAt; + this.expirationTime = expirationTime; + } + + /** + * Parse payload from token to fill the JwtPayload fields. + * IMPORTANT: This method doesn't validate that the token hasn't been modified !! + * + * @param token JWT token. + */ + public JwtPayload(String token) { + if (StringUtils.isEmpty(token) || "null".equalsIgnoreCase(token)) { + throw new IllegalArgumentException("Missing token"); + } else { + // Analyse token + String[] split = StringUtils.split(token, "."); + if (split.length != 3) { + throw new IllegalArgumentException("Invalid JWT token"); + } + String claims = new String(Base64.getDecoder().decode(split[1])); + + ObjectMap claimsMap; + try { + claimsMap = JacksonUtils.getDefaultObjectMapper().readerFor(ObjectMap.class).readValue(claims); + } catch (JsonProcessingException e) { + throw new IllegalArgumentException("Invalid JWT token. Token could not be parsed.", e); + } + + this.token = token; + + this.userId = claimsMap.getString("sub"); + this.organization = claimsMap.getString("aud"); + this.issuer = claimsMap.getString("iss"); + + if (claimsMap.containsKey(AUTH_ORIGIN)) { + this.authOrigin = AuthenticationOrigin.AuthenticationType.valueOf(claimsMap.getString(AUTH_ORIGIN)); + } else { + this.authOrigin = null; + } + + if (claimsMap.containsKey("iat")) { + long iat = 1000L * claimsMap.getLong("iat"); + this.issuedAt = new Date(iat); + } else { + this.issuedAt = null; + } + + if (claimsMap.containsKey("exp")) { + long exp = 1000L * claimsMap.getLong("exp"); + this.expirationTime = new Date(exp); + } else { + this.expirationTime = null; + } + } + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("JwtPayload{"); + sb.append("userId='").append(userId).append('\''); + sb.append(", organization='").append(organization).append('\''); + sb.append(", authOrigin=").append(authOrigin); + sb.append(", issuer='").append(issuer).append('\''); + sb.append(", issuedAt=").append(issuedAt); + sb.append(", expirationTime=").append(expirationTime); + sb.append(", token='").append(token).append('\''); + sb.append('}'); + return sb.toString(); + } + + public String getUserId() { + return userId; + } + + public String getUserId(String organization) { + if (StringUtils.isNotEmpty(organization) && StringUtils.isNotEmpty(this.organization) && !organization.equals(this.organization)) { + // User wants to access data from a different organization. Therefore, we need to return an fqn + return this.organization + ":" + this.userId; + } + return this.userId; + } + + public String getOrganization() { + return organization; + } + + public AuthenticationOrigin.AuthenticationType getAuthOrigin() { + return authOrigin; + } + + public String getIssuer() { + return issuer; + } + + public Date getIssuedAt() { + return issuedAt; + } + + public Date getExpirationTime() { + return expirationTime; + } + + public String getToken() { + return token; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/admin/GroupSyncParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/admin/GroupSyncParams.java index 2d9e74b8c6e..ebc8fc0010e 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/admin/GroupSyncParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/admin/GroupSyncParams.java @@ -16,8 +16,6 @@ package org.opencb.opencga.core.models.admin; -import org.opencb.opencga.core.models.user.Account; - public class GroupSyncParams { private String authenticationOriginId; @@ -25,20 +23,17 @@ public class GroupSyncParams { private String to; private String study; private boolean syncAll; - private Account.AccountType type; private boolean force; public GroupSyncParams() { } - public GroupSyncParams(String authenticationOriginId, String from, String to, String study, boolean syncAll, Account.AccountType type, - boolean force) { + public GroupSyncParams(String authenticationOriginId, String from, String to, String study, boolean syncAll, boolean force) { this.authenticationOriginId = authenticationOriginId; this.from = from; this.to = to; this.study = study; this.syncAll = syncAll; - this.type = type; this.force = force; } @@ -50,7 +45,6 @@ public String toString() { sb.append(", to='").append(to).append('\''); sb.append(", study='").append(study).append('\''); sb.append(", syncAll=").append(syncAll); - sb.append(", type=").append(type); sb.append(", force=").append(force); sb.append('}'); return sb.toString(); @@ -101,15 +95,6 @@ public GroupSyncParams setSyncAll(boolean syncAll) { return this; } - public Account.AccountType getType() { - return type; - } - - public GroupSyncParams setType(Account.AccountType type) { - this.type = type; - return this; - } - public boolean isForce() { return force; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/admin/InstallationParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/admin/InstallationParams.java index 39afad5dc53..69e1f08736b 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/admin/InstallationParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/admin/InstallationParams.java @@ -21,16 +21,14 @@ public class InstallationParams { private String secretKey; private String password; private String email; - private String organization; public InstallationParams() { } - public InstallationParams(String secretKey, String password, String email, String organization) { + public InstallationParams(String secretKey, String password, String email) { this.secretKey = secretKey; this.password = password; this.email = email; - this.organization = organization; } @Override @@ -39,7 +37,6 @@ public String toString() { sb.append("secretKey='").append(secretKey).append('\''); sb.append(", password='").append(password).append('\''); sb.append(", email='").append(email).append('\''); - sb.append(", organization='").append(organization).append('\''); sb.append('}'); return sb.toString(); } @@ -71,12 +68,4 @@ public InstallationParams setEmail(String email) { return this; } - public String getOrganization() { - return organization; - } - - public InstallationParams setOrganization(String organization) { - this.organization = organization; - return this; - } } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/admin/UserCreateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/admin/UserCreateParams.java deleted file mode 100644 index 821ade7d980..00000000000 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/admin/UserCreateParams.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2015-2020 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.core.models.admin; - -import org.opencb.opencga.core.models.user.Account; -import org.opencb.opencga.core.models.user.User; - -public class UserCreateParams extends org.opencb.opencga.core.models.user.UserCreateParams { - - private Account.AccountType type; - - public UserCreateParams() { - } - - public UserCreateParams(String id, String name, String email, String password, String organization, Account.AccountType type) { - super(id, name, email, password, organization); - this.type = type; - } - - public static UserCreateParams of(User user) { - return new UserCreateParams(user.getId(), user.getName(), user.getEmail(), "", user.getOrganization(), - user.getAccount().getType()); - } - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder("UserCreateParams{"); - sb.append("type=").append(type); - sb.append('}'); - return sb.toString(); - } - - public Account.AccountType getType() { - return type; - } - - public UserCreateParams setType(Account.AccountType type) { - this.type = type; - return this; - } -} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/audit/AuditRecord.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/audit/AuditRecord.java index 312e1f32e87..f8ba4b924bc 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/audit/AuditRecord.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/audit/AuditRecord.java @@ -275,6 +275,11 @@ public Status(Result name, Error error) { this.error = error; } + public Status(Result name, Throwable exception) { + this.name = name; + this.error = new Error(0, exception.getClass().getSimpleName(), exception.getMessage()); + } + public enum Result { SUCCESS, ERROR diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysis.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysis.java index 61bc01c644f..d2d48863d31 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysis.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysis.java @@ -22,7 +22,6 @@ import org.opencb.biodata.models.clinical.ClinicalAudit; import org.opencb.biodata.models.clinical.ClinicalComment; import org.opencb.biodata.models.clinical.Disorder; -import org.opencb.biodata.models.common.Status; import org.opencb.commons.annotations.DataClass; import org.opencb.commons.annotations.DataField; import org.opencb.opencga.core.api.FieldConstants; @@ -100,9 +99,9 @@ public class ClinicalAnalysis extends Annotable { description = FieldConstants.CLINICAL_ANALYSIS_PANELS) private List panels; - @DataField(id = "panelLock", indexed = true, + @DataField(id = "panelLocked", indexed = true, description = FieldConstants.CLINICAL_ANALYSIS_PANEL_LOCK) - private boolean panelLock; + private boolean panelLocked; @DataField(id = "locked", indexed = true, description = FieldConstants.CLINICAL_ANALYSIS_LOCKED) @@ -179,6 +178,14 @@ public class ClinicalAnalysis extends Annotable { description = FieldConstants.GENERIC_RELEASE_DESCRIPTION) private int release; + /** + * Generic: Autoincremental version assigned to the registered entry. + * + * @apiNote Immutable + */ + @DataField(id = "version", managed = true, indexed = true, description = FieldConstants.GENERIC_VERSION_DESCRIPTION) + private int version; + @DataField(id = "qualityControl", indexed = true, description = FieldConstants.GENERIC_QUALITY_CONTROL) private ClinicalAnalysisQualityControl qualityControl; @@ -217,19 +224,19 @@ public class ClinicalAnalysis extends Annotable { */ @DataField(id = "status", indexed = true, uncommentedClasses = {"Status"}, description = FieldConstants.GENERIC_STATUS_DESCRIPTION) - private Status status; + private ClinicalStatus status; public ClinicalAnalysis() { } public ClinicalAnalysis(String id, String description, Type type, Disorder disorder, List files, Individual proband, - Family family, List panels, boolean panelLock, boolean locked, Interpretation interpretation, + Family family, List panels, boolean panelLocked, boolean locked, Interpretation interpretation, List secondaryInterpretations, ClinicalConsentAnnotation consent, List analysts, ClinicalReport report, ClinicalRequest request, ClinicalResponsible responsible, ClinicalPriorityAnnotation priority, List flags, String creationDate, String modificationDate, - String dueDate, int release, List comments, ClinicalAnalysisQualityControl qualityControl, - List audit, ClinicalAnalysisInternal internal, List annotationSets, - Map attributes, Status status) { + String dueDate, int release, int version, List comments, + ClinicalAnalysisQualityControl qualityControl, List audit, ClinicalAnalysisInternal internal, + List annotationSets, Map attributes, ClinicalStatus status) { this.id = id; this.description = description; this.type = type; @@ -238,7 +245,7 @@ public ClinicalAnalysis(String id, String description, Type type, Disorder disor this.proband = proband; this.family = family; this.panels = panels; - this.panelLock = panelLock; + this.panelLocked = panelLocked; this.locked = locked; this.interpretation = interpretation; this.secondaryInterpretations = secondaryInterpretations; @@ -254,6 +261,7 @@ public ClinicalAnalysis(String id, String description, Type type, Disorder disor this.dueDate = dueDate; this.qualityControl = qualityControl; this.release = release; + this.version = version; this.comments = comments; this.audit = audit; this.internal = internal; @@ -274,7 +282,7 @@ public String toString() { sb.append(", proband=").append(proband); sb.append(", family=").append(family); sb.append(", panels=").append(panels); - sb.append(", panelLock=").append(panelLock); + sb.append(", panelLocked=").append(panelLocked); sb.append(", locked=").append(locked); sb.append(", interpretation=").append(interpretation); sb.append(", secondaryInterpretations=").append(secondaryInterpretations); @@ -289,6 +297,7 @@ public String toString() { sb.append(", modificationDate='").append(modificationDate).append('\''); sb.append(", dueDate='").append(dueDate).append('\''); sb.append(", release=").append(release); + sb.append(", version=").append(version); sb.append(", qualityControl=").append(qualityControl); sb.append(", comments=").append(comments); sb.append(", audit=").append(audit); @@ -383,12 +392,23 @@ public ClinicalAnalysis setPanels(List panels) { return this; } + @Deprecated + @JsonIgnore public boolean isPanelLock() { - return panelLock; + return isPanelLocked(); } + @Deprecated public ClinicalAnalysis setPanelLock(boolean panelLock) { - this.panelLock = panelLock; + return setPanelLocked(panelLock); + } + + public boolean isPanelLocked() { + return panelLocked; + } + + public ClinicalAnalysis setPanelLocked(boolean panelLocked) { + this.panelLocked = panelLocked; return this; } @@ -532,6 +552,15 @@ public ClinicalAnalysis setRelease(int release) { return this; } + public int getVersion() { + return version; + } + + public ClinicalAnalysis setVersion(int version) { + this.version = version; + return this; + } + public List getComments() { return comments; } @@ -583,11 +612,11 @@ public ClinicalAnalysis setAttributes(Map attributes) { return this; } - public Status getStatus() { + public ClinicalStatus getStatus() { return status; } - public ClinicalAnalysis setStatus(Status status) { + public ClinicalAnalysis setStatus(ClinicalStatus status) { this.status = status; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisCreateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisCreateParams.java index 7ffbf4b4a9a..157b9f0b12e 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisCreateParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisCreateParams.java @@ -46,7 +46,7 @@ public class ClinicalAnalysisCreateParams { private FamilyParam family; private List panels; - private Boolean panelLock; + private Boolean panelLocked; @Deprecated private ClinicalAnalystParam analyst; @@ -75,7 +75,7 @@ public ClinicalAnalysisCreateParams() { public ClinicalAnalysisCreateParams(String id, String description, ClinicalAnalysis.Type type, DisorderReferenceParam disorder, List files, ProbandParam proband, FamilyParam family, - List panels, Boolean panelLock, List analysts, + List panels, Boolean panelLocked, List analysts, ClinicalReport report, ClinicalRequest request, ClinicalResponsible responsible, InterpretationCreateParams interpretation, ClinicalConsentAnnotationParam consent, String creationDate, String modificationDate, String dueDate, List comments, @@ -90,7 +90,7 @@ public ClinicalAnalysisCreateParams(String id, String description, ClinicalAnaly this.proband = proband; this.family = family; this.panels = panels; - this.panelLock = panelLock; + this.panelLocked = panelLocked; this.report = report; this.request = request; this.responsible = responsible; @@ -120,7 +120,7 @@ public static ClinicalAnalysisCreateParams of(ClinicalAnalysis clinicalAnalysis) clinicalAnalysis.getPanels() != null ? clinicalAnalysis.getPanels().stream().map(p -> new PanelReferenceParam(p.getId())).collect(Collectors.toList()) : null, - clinicalAnalysis.isPanelLock(), + clinicalAnalysis.isPanelLocked(), clinicalAnalysis.getAnalysts() != null ? clinicalAnalysis.getAnalysts().stream().map(ClinicalAnalystParam::of).collect(Collectors.toList()) : null, @@ -155,7 +155,7 @@ public String toString() { sb.append(", proband=").append(proband); sb.append(", family=").append(family); sb.append(", panels=").append(panels); - sb.append(", panelLock=").append(panelLock); + sb.append(", panelLocked=").append(panelLocked); sb.append(", analysts=").append(analysts); sb.append(", report=").append(report); sb.append(", request=").append(request); @@ -231,14 +231,14 @@ public ClinicalAnalysis toClinicalAnalysis() { } return new ClinicalAnalysis(id, description, type, disorder != null ? disorder.toDisorder() : null, caFiles, individual, f, - diseasePanelList, panelLock != null ? panelLock : false, false, primaryInterpretation, new LinkedList<>(), + diseasePanelList, panelLocked != null ? panelLocked : false, false, primaryInterpretation, new LinkedList<>(), consent != null ? consent.toClinicalConsentAnnotation() : null, clinicalAnalystList, report, request, responsible, priority != null ? priority.toClinicalPriorityAnnotation() : null, flags != null ? flags.stream().map(FlagValueParam::toFlagAnnotation).collect(Collectors.toList()) : null, creationDate, modificationDate, dueDate, - 1, + 1, 1, comments != null ? comments.stream().map(ClinicalCommentParam::toClinicalComment).collect(Collectors.toList()) : null, qualityControl != null ? qualityControl.toClinicalQualityControl() : null, new LinkedList<>(), null, - annotationSets, attributes, status != null ? status.toStatus() : null); + annotationSets, attributes, status != null ? status.toClinicalStatus() : null); } public String getId() { @@ -313,12 +313,12 @@ public ClinicalAnalysisCreateParams setPanels(List panels) return this; } - public Boolean getPanelLock() { - return panelLock; + public Boolean getPanelLocked() { + return panelLocked; } - public ClinicalAnalysisCreateParams setPanelLock(Boolean panelLock) { - this.panelLock = panelLock; + public ClinicalAnalysisCreateParams setPanelLocked(Boolean panelLocked) { + this.panelLocked = panelLocked; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisInternal.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisInternal.java index a529d6d7d27..425654bcc55 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisInternal.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisInternal.java @@ -18,21 +18,19 @@ import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.models.common.Internal; +import org.opencb.opencga.core.models.common.InternalStatus; public class ClinicalAnalysisInternal extends Internal { - private ClinicalAnalysisStatus status; - public ClinicalAnalysisInternal() { } - public ClinicalAnalysisInternal(String registrationDate, String modificationDate, ClinicalAnalysisStatus status) { - super(null, registrationDate, modificationDate); - this.status = status; + public ClinicalAnalysisInternal(String registrationDate, String modificationDate, InternalStatus status) { + super(status, registrationDate, modificationDate); } public static ClinicalAnalysisInternal init() { - return new ClinicalAnalysisInternal(TimeUtils.getTime(), TimeUtils.getTime(), new ClinicalAnalysisStatus()); + return new ClinicalAnalysisInternal(TimeUtils.getTime(), TimeUtils.getTime(), new InternalStatus()); } @Override @@ -45,11 +43,11 @@ public String toString() { return sb.toString(); } - public ClinicalAnalysisStatus getStatus() { + public InternalStatus getStatus() { return status; } - public ClinicalAnalysisInternal setStatus(ClinicalAnalysisStatus status) { + public ClinicalAnalysisInternal setStatus(InternalStatus status) { this.status = status; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisLoadParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisLoadParams.java new file mode 100644 index 00000000000..9d3f75e7359 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisLoadParams.java @@ -0,0 +1,38 @@ +package org.opencb.opencga.core.models.clinical; + +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.tools.ToolParams; + +import java.nio.file.Path; + +public class ClinicalAnalysisLoadParams extends ToolParams { + public static final String DESCRIPTION = "Parameters to load clinical analysis in OpenCGA catalog from a file"; + public static final String FILE = "file"; + + private String file; + + public ClinicalAnalysisLoadParams() { + } + + public ClinicalAnalysisLoadParams(String file) { + this.file = file; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("ClinicalAnalysisLoadParams{"); + sb.append("file=").append(file); + sb.append('}'); + return sb.toString(); + } + + public String getFile() { + return file; + } + + public ClinicalAnalysisLoadParams setFile(String file) { + this.file = file; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisPermissions.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisPermissions.java index e0cdfadd0c0..724f4ae04a5 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisPermissions.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisPermissions.java @@ -9,7 +9,8 @@ public enum ClinicalAnalysisPermissions { DELETE(Arrays.asList(VIEW, WRITE)), VIEW_ANNOTATIONS(Collections.singletonList(VIEW)), WRITE_ANNOTATIONS(Arrays.asList(VIEW_ANNOTATIONS, VIEW)), - DELETE_ANNOTATIONS(Arrays.asList(VIEW_ANNOTATIONS, WRITE_ANNOTATIONS, VIEW)); + DELETE_ANNOTATIONS(Arrays.asList(VIEW_ANNOTATIONS, WRITE_ANNOTATIONS, VIEW)), + ADMIN(Arrays.asList(VIEW, WRITE, DELETE, VIEW_ANNOTATIONS, WRITE_ANNOTATIONS, DELETE_ANNOTATIONS)); private final List implicitPermissions; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisStatus.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisStatus.java deleted file mode 100644 index b898a7c21d5..00000000000 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisStatus.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright 2015-2020 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.core.models.clinical; - -import org.opencb.opencga.core.models.common.InternalStatus; - -import java.util.Arrays; -import java.util.List; - -public class ClinicalAnalysisStatus extends InternalStatus { - - public static final String INCOMPLETE = "INCOMPLETE"; - public static final String READY_FOR_VALIDATION = "READY_FOR_VALIDATION"; - public static final String READY_FOR_INTERPRETATION = "READY_FOR_INTERPRETATION"; - public static final String INTERPRETATION_IN_PROGRESS = "INTERPRETATION_IN_PROGRESS"; - // public static final String INTERPRETED = "INTERPRETED"; - public static final String READY_FOR_INTEPRETATION_REVIEW = "READY_FOR_INTEPRETATION_REVIEW"; - public static final String INTERPRETATION_REVIEW_IN_PROGRESS = "INTERPRETATION_REVIEW_IN_PROGRESS"; - public static final String READY_FOR_REPORT = "READY_FOR_REPORT"; - public static final String REPORT_IN_PROGRESS = "REPORT_IN_PROGRESS"; - public static final String DONE = "DONE"; - public static final String REVIEW_IN_PROGRESS = "REVIEW_IN_PROGRESS"; - public static final String CLOSED = "CLOSED"; - public static final String REJECTED = "REJECTED"; - - public static final List STATUS_LIST = Arrays.asList(INCOMPLETE, READY, DELETED, READY_FOR_VALIDATION, - READY_FOR_INTERPRETATION, INTERPRETATION_IN_PROGRESS, READY_FOR_INTEPRETATION_REVIEW, INTERPRETATION_REVIEW_IN_PROGRESS, - READY_FOR_REPORT, REPORT_IN_PROGRESS, DONE, REVIEW_IN_PROGRESS, CLOSED, REJECTED); - - public ClinicalAnalysisStatus(String status, String message) { - if (isValid(status)) { - init(status, message); - } else { - throw new IllegalArgumentException("Unknown status " + status); - } - } - - public ClinicalAnalysisStatus(String status) { - this(status, ""); - } - - public ClinicalAnalysisStatus() { - this(READY_FOR_INTERPRETATION, ""); - } - - public static boolean isValid(String status) { - if (InternalStatus.isValid(status)) { - return true; - } - - if (STATUS_LIST.contains(status)) { - return true; - } - return false; - } -} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisUpdateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisUpdateParams.java index 020a44ead6a..e22feef43ae 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisUpdateParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisUpdateParams.java @@ -44,7 +44,7 @@ public class ClinicalAnalysisUpdateParams { private List files; private List panels; - private Boolean panelLock; + private Boolean panelLocked; private ProbandParam proband; private FamilyParam family; @@ -77,7 +77,7 @@ public ClinicalAnalysisUpdateParams() { public ClinicalAnalysisUpdateParams(String id, String description, ClinicalAnalysis.Type type, DisorderReferenceParam disorder, List files, ProbandParam proband, FamilyParam family, - List panels, Boolean panelLock, Boolean locked, + List panels, Boolean panelLocked, Boolean locked, List analysts, ClinicalReport report, ClinicalRequest request, ClinicalResponsible responsible, ClinicalAnalysisQualityControlUpdateParam qualityControl, ClinicalConsentAnnotationParam consent, String creationDate, String modificationDate, @@ -92,7 +92,7 @@ public ClinicalAnalysisUpdateParams(String id, String description, ClinicalAnaly this.proband = proband; this.family = family; this.panels = panels; - this.panelLock = panelLock; + this.panelLocked = panelLocked; this.locked = locked; this.analysts = analysts; this.report = report; @@ -123,7 +123,7 @@ public ClinicalAnalysis toClinicalAnalysis() { proband != null ? proband.toIndividual() : null, family != null ? family.toFamily() : null, panels != null ? panels.stream().map(p -> new Panel().setId(p.getId())).collect(Collectors.toList()) : null, - panelLock != null ? panelLock : false, + panelLocked != null ? panelLocked : false, locked != null && locked, null, null, consent != null ? consent.toClinicalConsentAnnotation() : null, @@ -133,10 +133,10 @@ public ClinicalAnalysis toClinicalAnalysis() { report, request, responsible, priority != null ? priority.toClinicalPriorityAnnotation() : null, flags != null ? flags.stream().map(FlagValueParam::toFlagAnnotation).collect(Collectors.toList()) : null, creationDate, modificationDate, dueDate, - 1, + 1, 1, comments != null ? comments.stream().map(ClinicalCommentParam::toClinicalComment).collect(Collectors.toList()) : null, qualityControl != null ? qualityControl.toClinicalQualityControl() : null, null, null, annotationSets, attributes, - status != null ? status.toStatus() : null); + status != null ? status.toClinicalStatus() : null); } @Override @@ -148,7 +148,7 @@ public String toString() { sb.append(", disorder=").append(disorder); sb.append(", files=").append(files); sb.append(", panels=").append(panels); - sb.append(", panelLock=").append(panelLock); + sb.append(", panelLocked=").append(panelLocked); sb.append(", proband=").append(proband); sb.append(", family=").append(family); sb.append(", locked=").append(locked); @@ -243,12 +243,22 @@ public ClinicalAnalysisUpdateParams setFamily(FamilyParam family) { return this; } + @Deprecated public Boolean getPanelLock() { - return panelLock; + return getPanelLocked(); } + @Deprecated public ClinicalAnalysisUpdateParams setPanelLock(Boolean panelLock) { - this.panelLock = panelLock; + return setPanelLocked(panelLock); + } + + public Boolean getPanelLocked() { + return panelLocked; + } + + public ClinicalAnalysisUpdateParams setPanelLocked(Boolean panelLocked) { + this.panelLocked = panelLocked; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalStatus.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalStatus.java new file mode 100644 index 00000000000..baa92b6f22a --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalStatus.java @@ -0,0 +1,77 @@ +package org.opencb.opencga.core.models.clinical; + +public class ClinicalStatus extends ClinicalStatusValue { + + private String author; + private String version; + private String commit; + private String date; + + public ClinicalStatus() { + } + + public ClinicalStatus(String id, String description, ClinicalStatusType type, String author, String version, String commit, + String date) { + super(id, description, type); + this.author = author; + this.version = version; + this.commit = commit; + this.date = date; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("ClinicalStatus{"); + sb.append("id='").append(id).append('\''); + sb.append(", description='").append(description).append('\''); + sb.append(", type=").append(type); + sb.append(", author='").append(author).append('\''); + sb.append(", version='").append(version).append('\''); + sb.append(", commit='").append(commit).append('\''); + sb.append(", date='").append(date).append('\''); + sb.append('}'); + return sb.toString(); + } + + @Override + public ClinicalStatus setId(String id) { + super.setId(id); + return this; + } + + public String getVersion() { + return version; + } + + public ClinicalStatus setVersion(String version) { + this.version = version; + return this; + } + + public String getCommit() { + return commit; + } + + public ClinicalStatus setCommit(String commit) { + this.commit = commit; + return this; + } + + public String getAuthor() { + return author; + } + + public ClinicalStatus setAuthor(String author) { + this.author = author; + return this; + } + + public String getDate() { + return date; + } + + public ClinicalStatus setDate(String date) { + this.date = date; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalStatusValue.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalStatusValue.java index 9efa33d4a33..68816055dde 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalStatusValue.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalStatusValue.java @@ -2,15 +2,15 @@ public class ClinicalStatusValue { - private String id; - private String description; - private ClinicalStatusType type; + protected String id; + protected String description; + protected ClinicalStatusType type; public enum ClinicalStatusType { NOT_STARTED, - IN_PROGRESS, - CLOSED, - UNKNOWN + ACTIVE, + DONE, + CLOSED } public ClinicalStatusValue() { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserInterpretationAnalysisParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserInterpretationAnalysisParams.java index 0ebc7f92d22..deb5f28ba00 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserInterpretationAnalysisParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserInterpretationAnalysisParams.java @@ -1,26 +1,32 @@ package org.opencb.opencga.core.models.clinical; +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.FieldConstants; import org.opencb.opencga.core.tools.ToolParams; - public class ExomiserInterpretationAnalysisParams extends ToolParams { - public static final String DESCRIPTION = "Exomizer interpretation analysis params"; + public static final String DESCRIPTION = "Exomiser interpretation analysis params"; + @DataField(id = "clinicalAnalysis", description = FieldConstants.EXOMISER_CLINICAL_ANALYSIS_DESCRIPTION, required = true) private String clinicalAnalysis; + @DataField(id = "exomiserVersion", description = FieldConstants.EXOMISER_VERSION_DESCRIPTION) + private String exomiserVersion; + public ExomiserInterpretationAnalysisParams() { } - public ExomiserInterpretationAnalysisParams(String clinicalAnalysis) { + public ExomiserInterpretationAnalysisParams(String clinicalAnalysis, String exomiserVersion) { this.clinicalAnalysis = clinicalAnalysis; - + this.exomiserVersion = exomiserVersion; } @Override public String toString() { - final StringBuilder sb = new StringBuilder("TieringInterpretationAnalysisParams{"); + final StringBuilder sb = new StringBuilder("ExomiserInterpretationAnalysisParams{"); sb.append("clinicalAnalysis='").append(clinicalAnalysis).append('\''); + sb.append(", exomiserVersion='").append(exomiserVersion).append('\''); sb.append('}'); return sb.toString(); } @@ -34,5 +40,12 @@ public ExomiserInterpretationAnalysisParams setClinicalAnalysis(String clinicalA return this; } + public String getExomiserVersion() { + return exomiserVersion; + } + public ExomiserInterpretationAnalysisParams setExomiserVersion(String exomiserVersion) { + this.exomiserVersion = exomiserVersion; + return this; + } } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserWrapperParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserWrapperParams.java index 3b836063439..33a5415422a 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserWrapperParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserWrapperParams.java @@ -4,14 +4,15 @@ import org.opencb.opencga.core.api.FieldConstants; import org.opencb.opencga.core.tools.ToolParams; -import java.util.Map; - public class ExomiserWrapperParams extends ToolParams { public static final String DESCRIPTION = "Exomiser parameters"; - @DataField(id = "sample", description = FieldConstants.SAMPLE_ID_DESCRIPTION) + @DataField(id = "sample", description = FieldConstants.EXOMISER_SAMPLE_DESCRIPTION, required = true) private String sample; + @DataField(id = "exomiserVersion", description = FieldConstants.EXOMISER_VERSION_DESCRIPTION) + private String exomiserVersion; + @DataField(id = "clinicalAnalysisType", description = FieldConstants.EXOMISER_CLINICAL_ANALYSIS_TYPE_DESCRIPTION, defaultValue = "SINGLE") private String clinicalAnalysisType; @@ -22,9 +23,10 @@ public class ExomiserWrapperParams extends ToolParams { public ExomiserWrapperParams() { } - public ExomiserWrapperParams(String sample, String clinicalAnalysisType, String outdir) { + public ExomiserWrapperParams(String sample, String clinicalAnalysisType, String exomiserVersion, String outdir) { this.sample = sample; this.clinicalAnalysisType = clinicalAnalysisType; + this.exomiserVersion = exomiserVersion; this.outdir = outdir; } @@ -33,6 +35,7 @@ public String toString() { final StringBuilder sb = new StringBuilder("ExomiserWrapperParams{"); sb.append("sample='").append(sample).append('\''); sb.append(", clinicalAnalysisType=").append(clinicalAnalysisType); + sb.append("exomiserVersion='").append(exomiserVersion).append('\''); sb.append(", outdir='").append(outdir).append('\''); sb.append('}'); return sb.toString(); @@ -56,6 +59,15 @@ public ExomiserWrapperParams setClinicalAnalysisType(String clinicalAnalysisType return this; } + public String getExomiserVersion() { + return exomiserVersion; + } + + public ExomiserWrapperParams setExomiserVersion(String exomiserVersion) { + this.exomiserVersion = exomiserVersion; + return this; + } + public String getOutdir() { return outdir; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/Interpretation.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/Interpretation.java index 273ce53a4c9..a6774d4f751 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/Interpretation.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/Interpretation.java @@ -20,7 +20,6 @@ import org.opencb.biodata.models.clinical.ClinicalComment; import org.opencb.biodata.models.clinical.interpretation.ClinicalVariant; import org.opencb.biodata.models.clinical.interpretation.InterpretationMethod; -import org.opencb.biodata.models.common.Status; import org.opencb.commons.annotations.DataField; import org.opencb.opencga.core.api.FieldConstants; import org.opencb.opencga.core.models.IPrivateStudyUid; @@ -54,35 +53,56 @@ public class Interpretation extends org.opencb.biodata.models.clinical.interpret description = FieldConstants.GENERIC_RELEASE_DESCRIPTION) private int release; + @DataField(id = "status", indexed = true, + description = FieldConstants.GENERIC_STATUS_DESCRIPTION) + private ClinicalStatus status; + public Interpretation() { super(); } - public Interpretation(String id, String description, String clinicalAnalysisId, ClinicalAnalyst analyst, + public Interpretation(String id, String name, String description, String clinicalAnalysisId, ClinicalAnalyst analyst, InterpretationMethod method, String creationDate, String modificationDate, boolean locked, List primaryFindings, List secondaryFindings, List panels, - List comments, Status status, Map attributes) { - super(id, "", description, clinicalAnalysisId, analyst, method, primaryFindings, secondaryFindings, comments, null, status, - creationDate, modificationDate, locked, 0, attributes); - + List comments, ClinicalStatus status, Map attributes) { + super(id, "", name, description, clinicalAnalysisId, analyst, method, primaryFindings, secondaryFindings, comments, null, locked, + creationDate, modificationDate, 0, attributes); + this.status = status; this.panels = panels; } public Interpretation(org.opencb.biodata.models.clinical.interpretation.Interpretation interpretation) { - this(interpretation.getId(), interpretation.getDescription(), interpretation.getClinicalAnalysisId(), interpretation.getAnalyst(), - interpretation.getMethod(), interpretation.getCreationDate(), interpretation.getModificationDate(), - interpretation.isLocked(), interpretation.getPrimaryFindings(), interpretation.getSecondaryFindings(), - Collections.emptyList(), interpretation.getComments(), interpretation.getStatus(), interpretation.getAttributes()); + this(interpretation.getId(), interpretation.getName(), interpretation.getDescription(), interpretation.getClinicalAnalysisId(), + interpretation.getAnalyst(), interpretation.getMethod(), interpretation.getCreationDate(), + interpretation.getModificationDate(), interpretation.isLocked(), interpretation.getPrimaryFindings(), + interpretation.getSecondaryFindings(), Collections.emptyList(), interpretation.getComments(), null, + interpretation.getAttributes()); } @Override public String toString() { final StringBuilder sb = new StringBuilder("Interpretation{"); - sb.append("studyUid=").append(studyUid); + sb.append("id='").append(id).append('\''); + sb.append(", uuid='").append(uuid).append('\''); + sb.append(", name='").append(name).append('\''); + sb.append(", description='").append(description).append('\''); + sb.append(", studyUid=").append(studyUid); sb.append(", uid=").append(uid); sb.append(", panels=").append(panels); sb.append(", internal=").append(internal); sb.append(", release=").append(release); + sb.append(", status=").append(status); + sb.append(", clinicalAnalysisId='").append(clinicalAnalysisId).append('\''); + sb.append(", analyst=").append(analyst); + sb.append(", method=").append(method); + sb.append(", primaryFindings=").append(primaryFindings); + sb.append(", secondaryFindings=").append(secondaryFindings); + sb.append(", comments=").append(comments); + sb.append(", stats=").append(stats); + sb.append(", locked=").append(locked); + sb.append(", creationDate='").append(creationDate).append('\''); + sb.append(", modificationDate='").append(modificationDate).append('\''); + sb.append(", version=").append(version); sb.append('}'); return sb.toString(); } @@ -185,14 +205,17 @@ public Interpretation setComments(List comments) { } @Override - public Interpretation setStatus(Status status) { - super.setStatus(status); + public Interpretation setLocked(boolean locked) { + super.setLocked(locked); return this; } - @Override - public Interpretation setLocked(boolean locked) { - super.setLocked(locked); + public ClinicalStatus getStatus() { + return status; + } + + public Interpretation setStatus(ClinicalStatus status) { + this.status = status; return this; } } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/InterpretationCreateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/InterpretationCreateParams.java index e35ccec46f8..9c710fa0493 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/InterpretationCreateParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/InterpretationCreateParams.java @@ -26,13 +26,13 @@ import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.stream.Collectors; import static org.opencb.opencga.core.common.JacksonUtils.getUpdateObjectMapper; public class InterpretationCreateParams { + private String name; private String description; private String clinicalAnalysisId; private String creationDate; @@ -50,11 +50,12 @@ public class InterpretationCreateParams { public InterpretationCreateParams() { } - public InterpretationCreateParams(String description, String clinicalAnalysisId, String creationDate, String modificationDate, + public InterpretationCreateParams(String name, String description, String clinicalAnalysisId, String creationDate, String modificationDate, ClinicalAnalystParam analyst, InterpretationMethod method, List primaryFindings, List secondaryFindings, List panels, List comments, StatusParam status, Boolean locked, Map attributes) { + this.name = name; this.description = description; this.clinicalAnalysisId = clinicalAnalysisId; this.creationDate = creationDate; @@ -71,7 +72,7 @@ public InterpretationCreateParams(String description, String clinicalAnalysisId, } public static InterpretationCreateParams of(Interpretation interpretation) { - return new InterpretationCreateParams(interpretation.getDescription(), + return new InterpretationCreateParams(interpretation.getName(), interpretation.getDescription(), interpretation.getClinicalAnalysisId(), interpretation.getCreationDate(), interpretation.getModificationDate(), ClinicalAnalystParam.of(interpretation.getAnalyst()), interpretation.getMethod(), interpretation.getPrimaryFindings(), interpretation.getSecondaryFindings(), @@ -89,7 +90,8 @@ public static InterpretationCreateParams of(Interpretation interpretation) { @Override public String toString() { final StringBuilder sb = new StringBuilder("InterpretationCreateParams{"); - sb.append("description='").append(description).append('\''); + sb.append("name='").append(name).append('\''); + sb.append(", description='").append(description).append('\''); sb.append(", clinicalAnalysisId='").append(clinicalAnalysisId).append('\''); sb.append(", creationDate='").append(creationDate).append('\''); sb.append(", modificationDate='").append(modificationDate).append('\''); @@ -99,25 +101,34 @@ public String toString() { sb.append(", secondaryFindings=").append(secondaryFindings); sb.append(", panels=").append(panels); sb.append(", comments=").append(comments); - sb.append(", status=").append(status); sb.append(", locked=").append(locked); + sb.append(", status=").append(status); sb.append(", attributes=").append(attributes); sb.append('}'); return sb.toString(); } public Interpretation toClinicalInterpretation() { - return new Interpretation(null, description, clinicalAnalysisId, analyst != null ? analyst.toClinicalAnalyst() : null, method, + return new Interpretation("", name, description, clinicalAnalysisId, analyst != null ? analyst.toClinicalAnalyst() : null, method, creationDate, modificationDate, locked != null ? locked : false, primaryFindings, secondaryFindings, panels != null ? panels.stream().map(p -> new Panel().setId(p.getId())).collect(Collectors.toList()) : null, comments != null ? comments.stream().map(ClinicalCommentParam::toClinicalComment).collect(Collectors.toList()) : null, - status != null ? status.toStatus() : null, attributes); + status != null ? status.toClinicalStatus() : null, attributes); } public ObjectMap toInterpretationObjectMap() throws JsonProcessingException { return new ObjectMap(getUpdateObjectMapper().writeValueAsString(this.toClinicalInterpretation())); } + public String getName() { + return name; + } + + public InterpretationCreateParams setName(String name) { + this.name = name; + return this; + } + public String getDescription() { return description; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/InterpretationUpdateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/InterpretationUpdateParams.java index c0bee8b89ce..3da416d1b28 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/InterpretationUpdateParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/InterpretationUpdateParams.java @@ -31,6 +31,7 @@ public class InterpretationUpdateParams { + private String name; private String description; private ClinicalAnalystParam analyst; private InterpretationMethod method; @@ -47,11 +48,12 @@ public class InterpretationUpdateParams { public InterpretationUpdateParams() { } - public InterpretationUpdateParams(String description, ClinicalAnalystParam analyst, InterpretationMethod method, + public InterpretationUpdateParams(String name, String description, ClinicalAnalystParam analyst, InterpretationMethod method, String creationDate, String modificationDate, List primaryFindings, List secondaryFindings, List panels, List comments, StatusParam status, Boolean locked, Map attributes) { + this.name = name; this.description = description; this.analyst = analyst; this.method = method; @@ -74,7 +76,8 @@ public ObjectMap getUpdateMap() throws JsonProcessingException { @Override public String toString() { final StringBuilder sb = new StringBuilder("InterpretationUpdateParams{"); - sb.append("description='").append(description).append('\''); + sb.append("name='").append(name).append('\''); + sb.append(", description='").append(description).append('\''); sb.append(", analyst=").append(analyst); sb.append(", method=").append(method); sb.append(", creationDate='").append(creationDate).append('\''); @@ -90,6 +93,15 @@ public String toString() { return sb.toString(); } + public String getName() { + return name; + } + + public InterpretationUpdateParams setName(String name) { + this.name = name; + return this; + } + public String getDescription() { return description; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/AnnotationSet.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/AnnotationSet.java index 490527ef6f5..79a2128be8d 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/AnnotationSet.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/AnnotationSet.java @@ -16,13 +16,10 @@ package org.opencb.opencga.core.models.common; -import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; import org.opencb.opencga.core.common.JacksonUtils; -import org.opencb.opencga.core.common.TimeUtils; -import java.util.Collections; import java.util.Map; /** @@ -31,38 +28,16 @@ public class AnnotationSet { private String id; - @Deprecated - private String name; private String variableSetId; private Map annotations; - @Deprecated - private String creationDate; - @Deprecated - private int release; - @Deprecated - private Map attributes; - public AnnotationSet() { } public AnnotationSet(String id, String variableSetId, Map annotations) { - this(id, variableSetId, annotations, TimeUtils.getTime(), 1, Collections.emptyMap()); - } - - public AnnotationSet(String id, String variableSetId, Map annotations, Map attributes) { - this(id, variableSetId, annotations, TimeUtils.getTime(), 1, attributes); - } - - public AnnotationSet(String id, String variableSetId, Map annotations, String creationDate, int release, - Map attributes) { this.id = id; - this.name = id; this.variableSetId = variableSetId; this.annotations = annotations; - this.creationDate = creationDate; - this.release = release; - this.attributes = attributes; } @Override @@ -71,9 +46,6 @@ public String toString() { sb.append("id='").append(id).append('\''); sb.append(", variableSetId=").append(variableSetId); sb.append(", annotations=").append(annotations); - sb.append(", creationDate='").append(creationDate).append('\''); - sb.append(", release=").append(release); - sb.append(", attributes=").append(attributes); sb.append('}'); return sb.toString(); } @@ -89,15 +61,13 @@ public boolean equals(Object o) { } AnnotationSet that = (AnnotationSet) o; - return new EqualsBuilder().append(variableSetId, that.variableSetId).append(release, that.release).append(id, that.id) - .append(annotations, that.annotations).append(creationDate, that.creationDate).append(attributes, that.attributes) + return new EqualsBuilder().append(variableSetId, that.variableSetId).append(id, that.id).append(annotations, that.annotations) .isEquals(); } @Override public int hashCode() { - return new HashCodeBuilder(17, 37).append(id).append(variableSetId).append(annotations).append(creationDate).append(release) - .append(attributes).toHashCode(); + return new HashCodeBuilder(17, 37).append(id).append(variableSetId).append(annotations).toHashCode(); } public String getId() { @@ -106,17 +76,6 @@ public String getId() { public AnnotationSet setId(String id) { this.id = id; - this.name = StringUtils.isEmpty(this.name) ? this.id : this.name; - return this; - } - - public String getName() { - return name; - } - - public AnnotationSet setName(String name) { - this.name = name; - this.id = StringUtils.isEmpty(this.id) ? this.name : this.id; return this; } @@ -138,33 +97,6 @@ public AnnotationSet setAnnotations(Map annotations) { return this; } - public String getCreationDate() { - return creationDate; - } - - public AnnotationSet setCreationDate(String creationDate) { - this.creationDate = creationDate; - return this; - } - - public int getRelease() { - return release; - } - - public AnnotationSet setRelease(int release) { - this.release = release; - return this; - } - - public Map getAttributes() { - return attributes; - } - - public AnnotationSet setAttributes(Map attributes) { - this.attributes = attributes; - return this; - } - public T to(Class tClass) { return JacksonUtils.getDefaultObjectMapper().convertValue(this.getAnnotations(), tClass); } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/Enums.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/Enums.java index c26bfe647a0..2777222715b 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/Enums.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/Enums.java @@ -60,6 +60,8 @@ public Resource getResource() { public enum Resource { AUDIT, + NOTE, + ORGANIZATION, USER, PROJECT, STUDY, @@ -105,6 +107,82 @@ public List getFullPermissionList() { return Collections.emptyList(); } } + + public String toStudyPermission(String permission) { + switch (this) { + case SAMPLE: + return StudyPermissions.Permissions.getStudyPermission(permission, StudyPermissions.SAMPLE).name(); + case FILE: + return StudyPermissions.Permissions.getStudyPermission(permission, StudyPermissions.FILE).name(); + case COHORT: + return StudyPermissions.Permissions.getStudyPermission(permission, StudyPermissions.COHORT).name(); + case INDIVIDUAL: + return StudyPermissions.Permissions.getStudyPermission(permission, StudyPermissions.INDIVIDUAL).name(); + case FAMILY: + return StudyPermissions.Permissions.getStudyPermission(permission, StudyPermissions.FAMILY).name(); + case JOB: + return StudyPermissions.Permissions.getStudyPermission(permission, StudyPermissions.JOB).name(); + case DISEASE_PANEL: + return StudyPermissions.Permissions.getStudyPermission(permission, StudyPermissions.DISEASE_PANEL).name(); + case CLINICAL_ANALYSIS: + return StudyPermissions.Permissions.getStudyPermission(permission, StudyPermissions.CLINICAL_ANALYSIS).name(); + case STUDY: + default: + return permission; + } + } + + public String fromStudyPermission(String permission) { + StudyPermissions.Permissions studyPermission = StudyPermissions.Permissions.valueOf(permission); + switch (this) { + case SAMPLE: + if (studyPermission.getType() == StudyPermissions.SAMPLE) { + return studyPermission.getPermission(); + } + break; + case FILE: + if (studyPermission.getType() == StudyPermissions.FILE) { + return studyPermission.getPermission(); + } + break; + case COHORT: + if (studyPermission.getType() == StudyPermissions.COHORT) { + return studyPermission.getPermission(); + } + break; + case INDIVIDUAL: + if (studyPermission.getType() == StudyPermissions.INDIVIDUAL) { + return studyPermission.getPermission(); + } + break; + case FAMILY: + if (studyPermission.getType() == StudyPermissions.FAMILY) { + return studyPermission.getPermission(); + } + break; + case JOB: + if (studyPermission.getType() == StudyPermissions.JOB) { + return studyPermission.getPermission(); + } + break; + case DISEASE_PANEL: + if (studyPermission.getType() == StudyPermissions.DISEASE_PANEL) { + return studyPermission.getPermission(); + } + break; + case CLINICAL_ANALYSIS: + if (studyPermission.getType() == StudyPermissions.CLINICAL_ANALYSIS) { + return studyPermission.getPermission(); + } + break; + case STUDY: + default: + return permission; + } + throw new IllegalArgumentException("Study permission '" + permission + "' does not seem to have an equivalent valid permission" + + " for the entity '" + this + "'"); + } + } public enum Action { @@ -172,9 +250,11 @@ public enum Action { MOVE_AND_REGISTER, VISIT, + KILL_JOB, IMPORT, + FETCH_ORGANIZATION_IDS, IMPORT_EXTERNAL_USERS, IMPORT_EXTERNAL_GROUP_OF_USERS, SYNC_EXTERNAL_GROUP_OF_USERS, diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/IndexStatus.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/IndexStatus.java index 83836d0f7d4..99c35d2b029 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/IndexStatus.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/IndexStatus.java @@ -19,7 +19,7 @@ public class IndexStatus extends InternalStatus { public IndexStatus(String status, String message) { if (isValid(status)) { - init(status, status, message); + init(status, message); } else { throw new IllegalArgumentException("Unknown status " + status); } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/InternalStatus.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/InternalStatus.java index a934efdc3ee..a1d28c6914b 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/InternalStatus.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/InternalStatus.java @@ -57,8 +57,8 @@ public InternalStatus(String id, String description) { } } - public InternalStatus(String id, String name, String description, String date, String version, String commit) { - super(id, name, description, date); + public InternalStatus(String id, String description, String date, String version, String commit) { + super(id, description, date); if (!isValid(id)) { throw new IllegalArgumentException("Unknown status id '" + id + "'"); } @@ -112,12 +112,7 @@ private static void fillPositiveNegativeList(String[] statusList, List p } protected void init(String statusId, String description) { - init(statusId, statusId, description); - } - - protected void init(String statusId, String statusName, String description) { super.id = statusId; - super.name = statusName; super.description = description; super.date = TimeUtils.getTime(); this.version = GitRepositoryState.getInstance().getBuildVersion(); @@ -150,7 +145,6 @@ public String toString() { sb.append("version='").append(version).append('\''); sb.append(", commit='").append(commit).append('\''); sb.append(", id='").append(id).append('\''); - sb.append(", name='").append(name).append('\''); sb.append(", description='").append(description).append('\''); sb.append(", date='").append(date).append('\''); sb.append('}'); @@ -184,15 +178,6 @@ public InternalStatus setId(String id) { return this; } - public String getName() { - return StringUtils.isNotEmpty(name) ? name : id; - } - - public InternalStatus setName(String name) { - this.name = name; - return this; - } - public String getDate() { return date; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/StatusParam.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/StatusParam.java index 806ef09be4d..d826354369e 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/StatusParam.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/StatusParam.java @@ -2,6 +2,7 @@ import org.opencb.biodata.models.common.Status; import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.models.clinical.ClinicalStatus; public class StatusParam { @@ -18,10 +19,18 @@ public static StatusParam of(Status status) { return status != null ? new StatusParam(status.getId()) : null; } + public static StatusParam of(ClinicalStatus status) { + return status != null ? new StatusParam(status.getId()) : null; + } + public Status toStatus() { return new Status(id, "", "", TimeUtils.getTime()); } + public ClinicalStatus toClinicalStatus() { + return new ClinicalStatus(id, "", null, "", "", "", ""); + } + @Override public String toString() { final StringBuilder sb = new StringBuilder("StatusParam{"); diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/file/VariantIndexStatus.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/file/VariantIndexStatus.java index a6592c369bf..1e44390761c 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/file/VariantIndexStatus.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/file/VariantIndexStatus.java @@ -24,7 +24,7 @@ public class VariantIndexStatus extends IndexStatus { public VariantIndexStatus(String status, String message) { if (isValid(status)) { - init(status, status, message); + init(status, message); } else { throw new IllegalArgumentException("Unknown status " + status); } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/job/Job.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/job/Job.java index be3952b1e28..802f5967f2f 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/job/Job.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/job/Job.java @@ -19,6 +19,7 @@ import org.opencb.commons.annotations.DataClass; import org.opencb.commons.annotations.DataField; import org.opencb.opencga.core.api.FieldConstants; +import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.models.PrivateStudyUid; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.file.File; @@ -73,7 +74,6 @@ public class Job extends PrivateStudyUid { description = FieldConstants.JOB_COMMAND_LINE) private String commandLine; - @DataField(id = "params", indexed = true, description = FieldConstants.JOB_PARAMS) private Map params; @@ -129,6 +129,17 @@ public class Job extends PrivateStudyUid { description = FieldConstants.JOB_DEPENDS_ON_DESCRIPTION) private List dependsOn; + @DataField(id = FieldConstants.JOB_PARENT_ID, indexed = false, since = "3.2.0", + description = FieldConstants.JOB_PARENT_ID_DESCRIPTION) + private String parentId; + + @DataField(id = FieldConstants.JOB_SCHEDULED_START_TIME, indexed = false, since = "3.2.0", + description = FieldConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) + private String scheduledStartTime; + + @DataField(id = ParamConstants.JOB_DRY_RUN, since = "3.2.0", description = ParamConstants.JOB_DRY_RUN_DESCRIPTION) + private boolean dryRun; + @DataField(id = "execution", indexed = true, description = FieldConstants.JOB_EXECUTION_DESCRIPTION) private ExecutionResult execution; @@ -168,8 +179,9 @@ public Job() { public Job(String id, String uuid, String description, ToolInfo tool, String userId, String commandLine, Map params, String creationDate, String modificationDate, Enums.Priority priority, JobInternal internal, File outDir, - List input, List output, List dependsOn, List tags, ExecutionResult execution, boolean visited, - File stdout, File stderr, int release, JobStudyParam study, Map attributes) { + List input, List output, List dependsOn, String parentId, String scheduledStartTime, boolean dryRun, + List tags, ExecutionResult execution, boolean visited, File stdout, File stderr, int release, JobStudyParam study, + Map attributes) { this.id = id; this.uuid = uuid; this.tool = tool; @@ -185,6 +197,9 @@ public Job(String id, String uuid, String description, ToolInfo tool, String use this.input = input; this.output = output; this.dependsOn = dependsOn; + this.parentId = parentId; + this.scheduledStartTime = scheduledStartTime; + this.dryRun = dryRun; this.tags = tags; this.execution = execution; this.visited = visited; @@ -214,6 +229,9 @@ public String toString() { sb.append(", output=").append(output); sb.append(", tags=").append(tags); sb.append(", dependsOn=").append(dependsOn); + sb.append(", parentId='").append(parentId).append('\''); + sb.append(", scheduledStartTime='").append(scheduledStartTime).append('\''); + sb.append(", dryRun=").append(dryRun); sb.append(", execution=").append(execution); sb.append(", stdout=").append(stdout); sb.append(", stderr=").append(stderr); @@ -375,6 +393,33 @@ public Job setDependsOn(List dependsOn) { return this; } + public String getParentId() { + return parentId; + } + + public Job setParentId(String parentId) { + this.parentId = parentId; + return this; + } + + public String getScheduledStartTime() { + return scheduledStartTime; + } + + public Job setScheduledStartTime(String scheduledStartTime) { + this.scheduledStartTime = scheduledStartTime; + return this; + } + + public boolean isDryRun() { + return dryRun; + } + + public Job setDryRun(boolean dryRun) { + this.dryRun = dryRun; + return this; + } + public List getTags() { return tags; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/job/JobCreateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/job/JobCreateParams.java index 278acfed94c..68382ca25c9 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/job/JobCreateParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/job/JobCreateParams.java @@ -271,9 +271,11 @@ public JobCreateParams setAttributes(Map attributes) { public Job toJob() { return new Job(id, null, description, tool, null, commandLine, params, creationDate, null, priority, internal != null ? new org.opencb.opencga.core.models.job.JobInternal(internal.getStatus()) : null, - outDir != null ? outDir.toFile() : null, getInput().stream().map(TinyFile::toFile).collect(Collectors.toList()), - getOutput().stream().map(TinyFile::toFile).collect(Collectors.toList()), Collections.emptyList(), - tags, result, false, stdout != null ? stdout.toFile() : null, stderr != null ? stderr.toFile() : null, 1, null, attributes); + outDir != null ? outDir.toFile() : null, + getInput() != null ? getInput().stream().map(TinyFile::toFile).collect(Collectors.toList()) : Collections.emptyList(), + getOutput() != null ? getOutput().stream().map(TinyFile::toFile).collect(Collectors.toList()) : Collections.emptyList(), + Collections.emptyList(), null, null, false, tags, result, false, stdout != null ? stdout.toFile() : null, + stderr != null ? stderr.toFile() : null, 1, null, attributes); } public static class JobInternal { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/job/JobInternal.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/job/JobInternal.java index 6bb8d08cdea..123dc8bf70e 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/job/JobInternal.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/job/JobInternal.java @@ -42,23 +42,34 @@ public class JobInternal extends Internal implements Cloneable { description = FieldConstants.JOB_INTERNAL_EVENTS_DESCRIPTION) private List events; + @DataField(id = "killJobRequested", indexed = false, since = "3.2.0", + description = FieldConstants.JOB_INTERNAL_KILL_JOB_REQUESTED_DESCRIPTION) + private boolean killJobRequested; + public JobInternal() { } public JobInternal(Enums.ExecutionStatus status) { - this(null, null, status, null, null); + this(null, null, status, null, null, false); } + @Deprecated public JobInternal(String registrationDate, String modificationDate, Enums.ExecutionStatus status, JobInternalWebhook webhook, List events) { + this(registrationDate, modificationDate, status, webhook, events, false); + } + + public JobInternal(String registrationDate, String modificationDate, Enums.ExecutionStatus status, JobInternalWebhook webhook, + List events, boolean killJobRequested) { super(null, registrationDate, modificationDate); this.status = status; this.webhook = webhook; this.events = events; + this.killJobRequested = killJobRequested; } public static JobInternal init() { - return new JobInternal(TimeUtils.getTime(), TimeUtils.getTime(), new Enums.ExecutionStatus(), null, new ArrayList<>()); + return new JobInternal(TimeUtils.getTime(), TimeUtils.getTime(), new Enums.ExecutionStatus(), null, new ArrayList<>(), false); } @Override @@ -69,13 +80,14 @@ public String toString() { sb.append(", status=").append(status); sb.append(", webhook=").append(webhook); sb.append(", events=").append(events); + sb.append(", killJobRequested=").append(killJobRequested); sb.append('}'); return sb.toString(); } @Override public JobInternal clone() throws CloneNotSupportedException { - return new JobInternal(registrationDate, lastModified, status, webhook.clone(), new LinkedList<>(events)); + return new JobInternal(registrationDate, lastModified, status, webhook.clone(), new LinkedList<>(events), killJobRequested); } public Enums.ExecutionStatus getStatus() { @@ -122,4 +134,13 @@ public JobInternal setLastModified(String lastModified) { this.lastModified = lastModified; return this; } + + public boolean isKillJobRequested() { + return killJobRequested; + } + + public JobInternal setKillJobRequested(boolean killJobRequested) { + this.killJobRequested = killJobRequested; + return this; + } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationRun.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/migration/MigrationRun.java similarity index 99% rename from opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationRun.java rename to opencga-core/src/main/java/org/opencb/opencga/core/models/migration/MigrationRun.java index c4cb97e9f7d..363021613f7 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationRun.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/migration/MigrationRun.java @@ -1,4 +1,4 @@ -package org.opencb.opencga.catalog.migration; +package org.opencb.opencga.core.models.migration; import org.opencb.commons.datastore.core.Event; import org.opencb.opencga.core.models.job.Job; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/notes/Note.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/notes/Note.java new file mode 100644 index 00000000000..ac12c5a474e --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/notes/Note.java @@ -0,0 +1,218 @@ +package org.opencb.opencga.core.models.notes; + +import org.opencb.commons.annotations.DataClass; +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.FieldConstants; +import org.opencb.opencga.core.models.PrivateStudyUid; + +import java.util.List; + +@DataClass(id = "Notes", since = "3.0", + description = "Notes data model hosts information about any kind of annotation or configuration that affects the whole " + + "Organization or Study.") +public class Note extends PrivateStudyUid { + + @DataField(id = "id", required = true, indexed = true, unique = true, immutable = true, + description = FieldConstants.NOTES_ID_DESCRIPTION) + private String id; + + @DataField(id = "uuid", indexed = true, unique = true, immutable = true, managed = true, + description = FieldConstants.GENERIC_UUID_DESCRIPTION) + private String uuid; + + @DataField(id = "scope", required = true, indexed = true, immutable = true, description = FieldConstants.NOTES_SCOPE_DESCRIPTION) + private Scope scope; + + @DataField(id = "study", indexed = true, immutable = true, description = FieldConstants.NOTES_STUDY_DESCRIPTION) + private String study; + + @DataField(id = "tags", indexed = true, description = FieldConstants.NOTES_TAGS_DESCRIPTION) + private List tags; + + @DataField(id = "userId", managed = true, immutable = true, description = FieldConstants.NOTES_USER_ID_DESCRIPTION) + private String userId; + + @DataField(id = "visibility", required = true, description = FieldConstants.NOTES_VISIBILITY_DESCRIPTION) + private Visibility visibility; + + @DataField(id = "version", indexed = true, managed = true, immutable = true, description = FieldConstants.GENERIC_VERSION_DESCRIPTION) + private int version; + + @DataField(id = "creationDate", indexed = true, managed = true, immutable = true, + description = FieldConstants.GENERIC_CREATION_DATE_DESCRIPTION) + private String creationDate; + + @DataField(id = "modificationDate", indexed = true, managed = true, immutable = true, + description = FieldConstants.GENERIC_MODIFICATION_DATE_DESCRIPTION) + private String modificationDate; + + @DataField(id = "valueType", required = true, description = FieldConstants.NOTES_VALUE_TYPE_DESCRIPTION) + private Type valueType; + + @DataField(id = "value", required = true, description = FieldConstants.NOTES_VALUE_DESCRIPTION) + private Object value; + + public enum Scope { + ORGANIZATION, + STUDY + } + + public enum Visibility { + PUBLIC, + PRIVATE + } + + public enum Type { + OBJECT, + ARRAY, + STRING, + INTEGER, + DOUBLE + } + + public Note() { + } + + public Note(String id, String uuid, Scope scope, String study, List tags, String userId, Visibility visibility, int version, + String creationDate, String modificationDate, Type valueType, Object value) { + this.id = id; + this.uuid = uuid; + this.scope = scope; + this.study = study; + this.tags = tags; + this.userId = userId; + this.visibility = visibility; + this.version = version; + this.creationDate = creationDate; + this.modificationDate = modificationDate; + this.valueType = valueType; + this.value = value; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("Notes{"); + sb.append("id='").append(id).append('\''); + sb.append(", uuid='").append(uuid).append('\''); + sb.append(", scope=").append(scope); + sb.append(", study='").append(study).append('\''); + sb.append(", tags=").append(tags); + sb.append(", userId='").append(userId).append('\''); + sb.append(", visibility=").append(visibility); + sb.append(", version=").append(version); + sb.append(", creationDate='").append(creationDate).append('\''); + sb.append(", modificationDate='").append(modificationDate).append('\''); + sb.append(", valueType=").append(valueType); + sb.append(", value=").append(value); + sb.append('}'); + return sb.toString(); + } + + public String getId() { + return id; + } + + public Note setId(String id) { + this.id = id; + return this; + } + + public String getUuid() { + return uuid; + } + + public Note setUuid(String uuid) { + this.uuid = uuid; + return this; + } + + public Scope getScope() { + return scope; + } + + public Note setScope(Scope scope) { + this.scope = scope; + return this; + } + + public String getStudy() { + return study; + } + + public Note setStudy(String study) { + this.study = study; + return this; + } + + public List getTags() { + return tags; + } + + public Note setTags(List tags) { + this.tags = tags; + return this; + } + + public String getUserId() { + return userId; + } + + public Note setUserId(String userId) { + this.userId = userId; + return this; + } + + public Visibility getVisibility() { + return visibility; + } + + public Note setVisibility(Visibility visibility) { + this.visibility = visibility; + return this; + } + + public int getVersion() { + return version; + } + + public Note setVersion(int version) { + this.version = version; + return this; + } + + public String getCreationDate() { + return creationDate; + } + + public Note setCreationDate(String creationDate) { + this.creationDate = creationDate; + return this; + } + + public String getModificationDate() { + return modificationDate; + } + + public Note setModificationDate(String modificationDate) { + this.modificationDate = modificationDate; + return this; + } + + public Type getValueType() { + return valueType; + } + + public Note setValueType(Type valueType) { + this.valueType = valueType; + return this; + } + + public Object getValue() { + return value; + } + + public Note setValue(Object value) { + this.value = value; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/notes/NoteCreateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/notes/NoteCreateParams.java new file mode 100644 index 00000000000..dfc277ccce9 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/notes/NoteCreateParams.java @@ -0,0 +1,88 @@ +package org.opencb.opencga.core.models.notes; + +import org.opencb.opencga.core.common.TimeUtils; + +import java.util.Collections; +import java.util.List; + +public class NoteCreateParams { + + private String id; + private List tags; + private Note.Visibility visibility; + private Note.Type valueType; + private Object value; + + public NoteCreateParams() { + } + + public NoteCreateParams(String id, List tags, Note.Visibility visibility, Note.Type valueType, Object value) { + this.id = id; + this.tags = tags; + this.visibility = visibility; + this.valueType = valueType; + this.value = value; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("NotesCreateParams{"); + sb.append("id='").append(id).append('\''); + sb.append(", tags=").append(tags); + sb.append(", visibility=").append(visibility); + sb.append(", valueType=").append(valueType); + sb.append(", value=").append(value); + sb.append('}'); + return sb.toString(); + } + + public Note toNote(Note.Scope scope, String userId) { + return new Note(id, null, scope, null, tags != null ? tags : Collections.emptyList(), userId, visibility, 1, TimeUtils.getTime(), + TimeUtils.getTime(), valueType, value != null ? value : Collections.emptyMap()); + } + + public String getId() { + return id; + } + + public NoteCreateParams setId(String id) { + this.id = id; + return this; + } + + public List getTags() { + return tags; + } + + public NoteCreateParams setTags(List tags) { + this.tags = tags; + return this; + } + + public Note.Visibility getVisibility() { + return visibility; + } + + public NoteCreateParams setVisibility(Note.Visibility visibility) { + this.visibility = visibility; + return this; + } + + public Note.Type getValueType() { + return valueType; + } + + public NoteCreateParams setValueType(Note.Type type) { + this.valueType = type; + return this; + } + + public Object getValue() { + return value; + } + + public NoteCreateParams setValue(Object value) { + this.value = value; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/notes/NoteUpdateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/notes/NoteUpdateParams.java new file mode 100644 index 00000000000..ba8c792add0 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/notes/NoteUpdateParams.java @@ -0,0 +1,67 @@ +package org.opencb.opencga.core.models.notes; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.core.JsonProcessingException; +import org.opencb.commons.datastore.core.ObjectMap; + +import java.util.List; + +import static org.opencb.opencga.core.common.JacksonUtils.getUpdateObjectMapper; + +public class NoteUpdateParams { + + private List tags; + private Note.Visibility visibility; + private Object value; + + public NoteUpdateParams() { + } + + public NoteUpdateParams(List tags, Note.Visibility visibility, Object value) { + this.tags = tags; + this.visibility = visibility; + this.value = value; + } + + @JsonIgnore + public ObjectMap getUpdateMap() throws JsonProcessingException { + return new ObjectMap(getUpdateObjectMapper().writeValueAsString(this)); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("NotesUpdateParams{"); + sb.append("tags=").append(tags); + sb.append(", visibility=").append(visibility); + sb.append(", value=").append(value); + sb.append('}'); + return sb.toString(); + } + + public List getTags() { + return tags; + } + + public NoteUpdateParams setTags(List tags) { + this.tags = tags; + return this; + } + + public Note.Visibility getVisibility() { + return visibility; + } + + public NoteUpdateParams setVisibility(Note.Visibility visibility) { + this.visibility = visibility; + return this; + } + + public Object getValue() { + return value; + } + + public NoteUpdateParams setValue(Object value) { + this.value = value; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantFileDeleteParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantFileDeleteParams.java index 85ba64ce78c..2234cb489e7 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantFileDeleteParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantFileDeleteParams.java @@ -16,6 +16,7 @@ package org.opencb.opencga.core.models.operations.variant; +import org.opencb.commons.annotations.DataField; import org.opencb.opencga.core.tools.ToolParams; import java.util.List; @@ -32,8 +33,13 @@ public VariantFileDeleteParams(List file, boolean resume) { this.resume = resume; } + @DataField(description = "List of file ids to delete. Use 'all' to remove the whole study", required = true) private List file; + + @DataField(description = "Resume failed delete operation.", defaultValue = "false") private boolean resume; + + @DataField(description = "Force delete operation. This would allow deleting partially loaded files.", defaultValue = "false") private boolean force; public List getFile() { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/organizations/Organization.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/organizations/Organization.java new file mode 100644 index 00000000000..55520c9b83d --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/organizations/Organization.java @@ -0,0 +1,198 @@ +package org.opencb.opencga.core.models.organizations; + +import org.opencb.commons.annotations.DataClass; +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.FieldConstants; +import org.opencb.opencga.core.models.notes.Note; +import org.opencb.opencga.core.models.project.Project; + +import java.util.List; +import java.util.Map; + +@DataClass(id = "Organization", since = "3.0", + description = "Organization data model hosts information about the organization managing the data.") +public class Organization { + + @DataField(id = "id", required = true, indexed = true, unique = true, immutable = true, + description = FieldConstants.ORGANIZATION_ID_DESCRIPTION) + private String id; + + @DataField(id = "uuid", managed = true, indexed = true, unique = true, immutable = true, + description = FieldConstants.GENERIC_UUID_DESCRIPTION) + private String uuid; + + @DataField(id = "name", description = FieldConstants.ORGANIZATION_NAME_DESCRIPTION) + private String name; + + @DataField(id = "owner", description = FieldConstants.ORGANIZATION_OWNER_DESCRIPTION) + private String owner; + + @DataField(id = "admins", description = FieldConstants.ORGANIZATION_ADMINS_DESCRIPTION) + private List admins; + + @DataField(id = "creationDate", description = FieldConstants.GENERIC_CREATION_DATE_DESCRIPTION) + private String creationDate; + + @DataField(id = "modificationDate", description = FieldConstants.GENERIC_MODIFICATION_DATE_DESCRIPTION) + private String modificationDate; + + @DataField(id = "projects", description = FieldConstants.ORGANIZATION_PROJECTS_DESCRIPTION) + private List projects; + + @DataField(id = "notes", description = FieldConstants.ORGANIZATION_NOTES_DESCRIPTION) + private List notes; + + @DataField(id = "configuration", description = FieldConstants.ORGANIZATION_CONFIGURATION_DESCRIPTION) + private OrganizationConfiguration configuration; + + @DataField(id = "internal", managed = true, description = FieldConstants.ORGANIZATION_INTERNAL_DESCRIPTION) + private OrganizationInternal internal; + + @DataField(id = "attributes", description = FieldConstants.GENERIC_ATTRIBUTES_DESCRIPTION) + private Map attributes; + + public Organization() { + } + + public Organization(String id, String name, String owner, List admins, String creationDate, String modificationDate, + List projects, OrganizationConfiguration configuration, OrganizationInternal internal, + Map attributes) { + this.id = id; + this.name = name; + this.owner = owner; + this.admins = admins; + this.creationDate = creationDate; + this.modificationDate = modificationDate; + this.projects = projects; + this.configuration = configuration; + this.internal = internal; + this.attributes = attributes; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("Organization{"); + sb.append("id='").append(id).append('\''); + sb.append(", uuid='").append(uuid).append('\''); + sb.append(", name='").append(name).append('\''); + sb.append(", owner='").append(owner).append('\''); + sb.append(", admins=").append(admins); + sb.append(", creationDate='").append(creationDate).append('\''); + sb.append(", modificationDate='").append(modificationDate).append('\''); + sb.append(", projects=").append(projects); + sb.append(", notes=").append(notes); + sb.append(", configuration=").append(configuration); + sb.append(", internal=").append(internal); + sb.append(", attributes=").append(attributes); + sb.append('}'); + return sb.toString(); + } + + public String getUuid() { + return uuid; + } + + public Organization setUuid(String uuid) { + this.uuid = uuid; + return this; + } + + public String getId() { + return id; + } + + public Organization setId(String id) { + this.id = id; + return this; + } + + public String getName() { + return name; + } + + public Organization setName(String name) { + this.name = name; + return this; + } + + public String getOwner() { + return owner; + } + + public Organization setOwner(String owner) { + this.owner = owner; + return this; + } + + public List getAdmins() { + return admins; + } + + public Organization setAdmins(List admins) { + this.admins = admins; + return this; + } + + public String getCreationDate() { + return creationDate; + } + + public Organization setCreationDate(String creationDate) { + this.creationDate = creationDate; + return this; + } + + public String getModificationDate() { + return modificationDate; + } + + public Organization setModificationDate(String modificationDate) { + this.modificationDate = modificationDate; + return this; + } + + public List getProjects() { + return projects; + } + + public Organization setProjects(List projects) { + this.projects = projects; + return this; + } + + public List getNotes() { + return notes; + } + + public Organization setNotes(List notes) { + this.notes = notes; + return this; + } + + public OrganizationConfiguration getConfiguration() { + return configuration; + } + + public Organization setConfiguration(OrganizationConfiguration configuration) { + this.configuration = configuration; + return this; + } + + public OrganizationInternal getInternal() { + return internal; + } + + public Organization setInternal(OrganizationInternal internal) { + this.internal = internal; + return this; + } + + public Map getAttributes() { + return attributes; + } + + public Organization setAttributes(Map attributes) { + this.attributes = attributes; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/organizations/OrganizationConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/organizations/OrganizationConfiguration.java new file mode 100644 index 00000000000..cda477b2acc --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/organizations/OrganizationConfiguration.java @@ -0,0 +1,72 @@ +package org.opencb.opencga.core.models.organizations; + +import org.opencb.opencga.core.config.AuthenticationOrigin; +import org.opencb.opencga.core.config.Optimizations; + +import java.util.List; + +public class OrganizationConfiguration { + + private List authenticationOrigins; + private String defaultUserExpirationDate; + private Optimizations optimizations; + private TokenConfiguration token; + + public OrganizationConfiguration() { + } + + public OrganizationConfiguration(List authenticationOrigins, String defaultUserExpirationDate, + Optimizations optimizations, TokenConfiguration token) { + this.authenticationOrigins = authenticationOrigins; + this.defaultUserExpirationDate = defaultUserExpirationDate; + this.optimizations = optimizations; + this.token = token; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("OrganizationConfiguration{"); + sb.append("authenticationOrigins=").append(authenticationOrigins); + sb.append(", defaultUserExpirationDate='").append(defaultUserExpirationDate).append('\''); + sb.append(", optimizations=").append(optimizations); + sb.append(", token=").append(token); + sb.append('}'); + return sb.toString(); + } + + public List getAuthenticationOrigins() { + return authenticationOrigins; + } + + public OrganizationConfiguration setAuthenticationOrigins(List authenticationOrigins) { + this.authenticationOrigins = authenticationOrigins; + return this; + } + + public String getDefaultUserExpirationDate() { + return defaultUserExpirationDate; + } + + public OrganizationConfiguration setDefaultUserExpirationDate(String defaultUserExpirationDate) { + this.defaultUserExpirationDate = defaultUserExpirationDate; + return this; + } + + public Optimizations getOptimizations() { + return optimizations; + } + + public OrganizationConfiguration setOptimizations(Optimizations optimizations) { + this.optimizations = optimizations; + return this; + } + + public TokenConfiguration getToken() { + return token; + } + + public OrganizationConfiguration setToken(TokenConfiguration token) { + this.token = token; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/organizations/OrganizationCreateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/organizations/OrganizationCreateParams.java new file mode 100644 index 00000000000..398a5f94d22 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/organizations/OrganizationCreateParams.java @@ -0,0 +1,118 @@ +package org.opencb.opencga.core.models.organizations; + +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.FieldConstants; + +import java.util.Map; + +public class OrganizationCreateParams { + + @DataField(id = "id", required = true, indexed = true, unique = true, immutable = true, + description = FieldConstants.ORGANIZATION_ID_DESCRIPTION) + private String id; + + @DataField(id = "name", description = FieldConstants.ORGANIZATION_NAME_DESCRIPTION) + private String name; + + @DataField(id = "creationDate", description = FieldConstants.GENERIC_CREATION_DATE_DESCRIPTION) + private String creationDate; + + @DataField(id = "modificationDate", description = FieldConstants.GENERIC_MODIFICATION_DATE_DESCRIPTION) + private String modificationDate; + + @DataField(id = "configuration", description = FieldConstants.ORGANIZATION_CONFIGURATION_DESCRIPTION) + private OrganizationConfiguration configuration; + + @DataField(id = "attributes", description = FieldConstants.GENERIC_ATTRIBUTES_DESCRIPTION) + private Map attributes; + + public OrganizationCreateParams() { + } + + public OrganizationCreateParams(String id, String name, String creationDate, String modificationDate, + OrganizationConfiguration configuration, Map attributes) { + this.id = id; + this.name = name; + this.creationDate = creationDate; + this.modificationDate = modificationDate; + this.configuration = configuration; + this.attributes = attributes; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("OrganizationCreateParams{"); + sb.append("id='").append(id).append('\''); + sb.append(", name='").append(name).append('\''); + sb.append(", creationDate='").append(creationDate).append('\''); + sb.append(", modificationDate='").append(modificationDate).append('\''); + sb.append(", configuration=").append(configuration); + sb.append(", attributes=").append(attributes); + sb.append('}'); + return sb.toString(); + } + + public static OrganizationCreateParams of(Organization organization) { + return new OrganizationCreateParams(organization.getId(), organization.getName(), + organization.getCreationDate(), organization.getModificationDate(), organization.getConfiguration(), + organization.getAttributes()); + } + + public Organization toOrganization() { + return new Organization(id, name, null, null, creationDate, modificationDate, null, configuration, null, attributes); + } + + public String getId() { + return id; + } + + public OrganizationCreateParams setId(String id) { + this.id = id; + return this; + } + + public String getName() { + return name; + } + + public OrganizationCreateParams setName(String name) { + this.name = name; + return this; + } + + public String getCreationDate() { + return creationDate; + } + + public OrganizationCreateParams setCreationDate(String creationDate) { + this.creationDate = creationDate; + return this; + } + + public String getModificationDate() { + return modificationDate; + } + + public OrganizationCreateParams setModificationDate(String modificationDate) { + this.modificationDate = modificationDate; + return this; + } + + public OrganizationConfiguration getConfiguration() { + return configuration; + } + + public OrganizationCreateParams setConfiguration(OrganizationConfiguration configuration) { + this.configuration = configuration; + return this; + } + + public Map getAttributes() { + return attributes; + } + + public OrganizationCreateParams setAttributes(Map attributes) { + this.attributes = attributes; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/organizations/OrganizationInternal.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/organizations/OrganizationInternal.java new file mode 100644 index 00000000000..e70722099c7 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/organizations/OrganizationInternal.java @@ -0,0 +1,46 @@ +package org.opencb.opencga.core.models.organizations; + +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.models.common.Internal; +import org.opencb.opencga.core.models.common.InternalStatus; +import org.opencb.opencga.core.models.migration.MigrationRun; + +import java.util.List; + +public class OrganizationInternal extends Internal { + + @DataField(id = "version", description = "OpenCGA version") + private String version; + + @DataField(id = "migrationExecutions", description = "List of migrations executions") + private List migrationExecutions; + + public OrganizationInternal() { + } + + public OrganizationInternal(InternalStatus status, String registrationDate, String lastModified, String version, + List migrationExecutions) { + super(status, registrationDate, lastModified); + this.version = version; + this.migrationExecutions = migrationExecutions; + } + + public String getVersion() { + return version; + } + + public OrganizationInternal setVersion(String version) { + this.version = version; + return this; + } + + public List getMigrationExecutions() { + return migrationExecutions; + } + + public OrganizationInternal setMigrationExecutions(List migrationExecutions) { + this.migrationExecutions = migrationExecutions; + return this; + } + +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/organizations/OrganizationMigrationSummary.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/organizations/OrganizationMigrationSummary.java new file mode 100644 index 00000000000..1014b1631cf --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/organizations/OrganizationMigrationSummary.java @@ -0,0 +1,117 @@ +package org.opencb.opencga.core.models.organizations; + +import java.util.List; + +public class OrganizationMigrationSummary { + + private int run; + private int skipped; + private int failed; + private int pending; + private List failedMigrations; + private List pendingMigrations; + private String version; + private String lastAttempt; // Date + + public OrganizationMigrationSummary() { + } + + public OrganizationMigrationSummary(int run, int skipped, int failed, int pending, List failedMigrations, + List pendingMigrations, String lastAttempt, String version) { + this.run = run; + this.skipped = skipped; + this.failed = failed; + this.pending = pending; + this.failedMigrations = failedMigrations; + this.pendingMigrations = pendingMigrations; + this.lastAttempt = lastAttempt; + this.version = version; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("OrganizationMigrationSummary{"); + sb.append("run=").append(run); + sb.append(", skipped=").append(skipped); + sb.append(", failed=").append(failed); + sb.append(", pending=").append(pending); + sb.append(", failedMigrations=").append(failedMigrations); + sb.append(", pendingMigrations=").append(pendingMigrations); + sb.append(", lastAttempt='").append(lastAttempt).append('\''); + sb.append(", version='").append(version).append('\''); + sb.append('}'); + return sb.toString(); + } + + public int getRun() { + return run; + } + + public OrganizationMigrationSummary setRun(int run) { + this.run = run; + return this; + } + + public int getSkipped() { + return skipped; + } + + public OrganizationMigrationSummary setSkipped(int skipped) { + this.skipped = skipped; + return this; + } + + public int getFailed() { + return failed; + } + + public OrganizationMigrationSummary setFailed(int failed) { + this.failed = failed; + return this; + } + + public int getPending() { + return pending; + } + + public OrganizationMigrationSummary setPending(int pending) { + this.pending = pending; + return this; + } + + public List getFailedMigrations() { + return failedMigrations; + } + + public OrganizationMigrationSummary setFailedMigrations(List failedMigrations) { + this.failedMigrations = failedMigrations; + return this; + } + + public List getPendingMigrations() { + return pendingMigrations; + } + + public OrganizationMigrationSummary setPendingMigrations(List pendingMigrations) { + this.pendingMigrations = pendingMigrations; + return this; + } + + public String getVersion() { + return version; + } + + public OrganizationMigrationSummary setVersion(String version) { + this.version = version; + return this; + } + + public String getLastAttempt() { + return lastAttempt; + } + + public OrganizationMigrationSummary setLastAttempt(String lastAttempt) { + this.lastAttempt = lastAttempt; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/organizations/OrganizationSummary.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/organizations/OrganizationSummary.java new file mode 100644 index 00000000000..05c8e38829a --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/organizations/OrganizationSummary.java @@ -0,0 +1,66 @@ +package org.opencb.opencga.core.models.organizations; + +public class OrganizationSummary { + + private String id; + private String database; + private String status; + private OrganizationMigrationSummary migration; + + public OrganizationSummary() { + } + + public OrganizationSummary(String id, String database, String status, OrganizationMigrationSummary migration) { + this.id = id; + this.database = database; + this.status = status; + this.migration = migration; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("OrganizationSummary{"); + sb.append("id='").append(id).append('\''); + sb.append(", database='").append(database).append('\''); + sb.append(", status='").append(status).append('\''); + sb.append(", migration=").append(migration); + sb.append('}'); + return sb.toString(); + } + + public String getId() { + return id; + } + + public OrganizationSummary setId(String id) { + this.id = id; + return this; + } + + public String getDatabase() { + return database; + } + + public OrganizationSummary setDatabase(String database) { + this.database = database; + return this; + } + + public String getStatus() { + return status; + } + + public OrganizationSummary setStatus(String status) { + this.status = status; + return this; + } + + public OrganizationMigrationSummary getMigration() { + return migration; + } + + public OrganizationSummary setMigration(OrganizationMigrationSummary migration) { + this.migration = migration; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/organizations/OrganizationUpdateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/organizations/OrganizationUpdateParams.java new file mode 100644 index 00000000000..8c4d42ea570 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/organizations/OrganizationUpdateParams.java @@ -0,0 +1,118 @@ +package org.opencb.opencga.core.models.organizations; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.core.JsonProcessingException; +import org.opencb.commons.annotations.DataField; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.opencga.core.api.FieldConstants; + +import java.util.List; +import java.util.Map; + +import static org.opencb.opencga.core.common.JacksonUtils.getUpdateObjectMapper; + +public final class OrganizationUpdateParams { + + @DataField(id = "name", description = FieldConstants.ORGANIZATION_NAME_DESCRIPTION) + private String name; + + @DataField(id = "owner", description = FieldConstants.ORGANIZATION_OWNER_DESCRIPTION) + private String owner; + + @DataField(id = "admins", description = FieldConstants.ORGANIZATION_ADMINS_DESCRIPTION) + private List admins; + + @DataField(id = "creationDate", description = FieldConstants.GENERIC_CREATION_DATE_DESCRIPTION) + private String creationDate; + + @DataField(id = "modificationDate", description = FieldConstants.GENERIC_MODIFICATION_DATE_DESCRIPTION) + private String modificationDate; + + @DataField(id = "attributes", description = FieldConstants.GENERIC_ATTRIBUTES_DESCRIPTION) + private Map attributes; + + public OrganizationUpdateParams() { + } + + public OrganizationUpdateParams(String name, String owner, List admins, String creationDate, String modificationDate, + Map attributes) { + this.name = name; + this.owner = owner; + this.admins = admins; + this.creationDate = creationDate; + this.modificationDate = modificationDate; + this.attributes = attributes; + } + + @JsonIgnore + public ObjectMap getUpdateMap() throws JsonProcessingException { + return new ObjectMap(getUpdateObjectMapper().writeValueAsString(this)); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("OrganizationUpdateParams{"); + sb.append("name='").append(name).append('\''); + sb.append(", owner='").append(owner).append('\''); + sb.append(", admins=").append(admins); + sb.append(", creationDate='").append(creationDate).append('\''); + sb.append(", modificationDate='").append(modificationDate).append('\''); + sb.append(", attributes=").append(attributes); + sb.append('}'); + return sb.toString(); + } + + public String getName() { + return name; + } + + public OrganizationUpdateParams setName(String name) { + this.name = name; + return this; + } + + public String getOwner() { + return owner; + } + + public OrganizationUpdateParams setOwner(String owner) { + this.owner = owner; + return this; + } + + public List getAdmins() { + return admins; + } + + public OrganizationUpdateParams setAdmins(List admins) { + this.admins = admins; + return this; + } + + public String getCreationDate() { + return creationDate; + } + + public OrganizationUpdateParams setCreationDate(String creationDate) { + this.creationDate = creationDate; + return this; + } + + public String getModificationDate() { + return modificationDate; + } + + public OrganizationUpdateParams setModificationDate(String modificationDate) { + this.modificationDate = modificationDate; + return this; + } + + public Map getAttributes() { + return attributes; + } + + public OrganizationUpdateParams setAttributes(Map attributes) { + this.attributes = attributes; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/organizations/TokenConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/organizations/TokenConfiguration.java new file mode 100644 index 00000000000..8ca89f2f285 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/organizations/TokenConfiguration.java @@ -0,0 +1,61 @@ +package org.opencb.opencga.core.models.organizations; + +import io.jsonwebtoken.SignatureAlgorithm; +import org.opencb.opencga.core.common.PasswordUtils; + +public class TokenConfiguration { + + private String algorithm; + private String secretKey; + private long expiration; + + public TokenConfiguration() { + } + + public TokenConfiguration(String algorithm, String secretKey, long expiration) { + this.algorithm = algorithm; + this.secretKey = secretKey; + this.expiration = expiration; + } + + public static TokenConfiguration init() { + return new TokenConfiguration(SignatureAlgorithm.HS256.getValue(), PasswordUtils.getStrongRandomPassword(32), 3600L); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("TokenConfiguration{"); + sb.append("algorithm='").append(algorithm).append('\''); + sb.append(", secretKey='").append(secretKey).append('\''); + sb.append(", expiration=").append(expiration); + sb.append('}'); + return sb.toString(); + } + + public String getAlgorithm() { + return algorithm; + } + + public TokenConfiguration setAlgorithm(String algorithm) { + this.algorithm = algorithm; + return this; + } + + public String getSecretKey() { + return secretKey; + } + + public TokenConfiguration setSecretKey(String secretKey) { + this.secretKey = secretKey; + return this; + } + + public long getExpiration() { + return expiration; + } + + public TokenConfiguration setExpiration(long expiration) { + this.expiration = expiration; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/project/Project.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/project/Project.java index bd5c59f39e3..0fd4c172876 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/project/Project.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/project/Project.java @@ -60,7 +60,7 @@ public class Project extends PrivateFields { private String name; /** - * Full Qualified Name (user@projectId). + * Full Qualified Name (organization@projectId). */ @DataField(id = "fqn", indexed = true, @@ -117,8 +117,7 @@ public class Project extends PrivateFields { * @apiNote Internal */ - @DataField(id = "release", indexed = true, - description = FieldConstants.GENERIC_RELEASE_DESCRIPTION) + @DataField(id = "internal", indexed = true, description = FieldConstants.GENERIC_INTERNAL) private ProjectInternal internal; /** @@ -164,23 +163,24 @@ public Project(String id, String name, String creationDate, String modificationD // Clone a project public Project(Project project) { - this(project.getUid(), project.getId(), project.getName(), project.getUuid(), project.getFqn(), project.getCreationDate(), - project.getModificationDate(), project.getDescription(), project.getOrganism(), project.getCurrentRelease(), - project.getStudies(), project.getInternal(), project.getAttributes()); + this(project.getUid(), project.getId(), project.getUuid(), project.getName(), project.getFqn(), project.getCreationDate(), + project.getModificationDate(), project.getDescription(), project.getOrganism(), project.getCellbase(), + project.getCurrentRelease(), project.getStudies(), project.getInternal(), project.getAttributes()); } - public Project(long uid, String id, String name, String uuid, String fqn, String creationDate, String modificationDate, - String description, ProjectOrganism organism, int currentRelease, List studies, ProjectInternal internal, - Map attributes) { + public Project(long uid, String id, String uuid, String name, String fqn, String creationDate, String modificationDate, + String description, ProjectOrganism organism, CellBaseConfiguration cellbase, int currentRelease, List studies, + ProjectInternal internal, Map attributes) { super(uid); this.id = id; - this.name = name; this.uuid = uuid; + this.name = name; this.fqn = fqn; this.creationDate = creationDate; this.modificationDate = modificationDate; this.description = description; this.organism = organism; + this.cellbase = cellbase; this.currentRelease = currentRelease; this.studies = studies; this.internal = internal; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/sample/Sample.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/sample/Sample.java index f45e545a292..10b42230abe 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/sample/Sample.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/sample/Sample.java @@ -107,8 +107,7 @@ public class Sample extends Annotable { private int release; /** - * Generic: Autoincremental version assigned to the registered entry. By default, updates does not create new versions. To enable - * versioning, users must set the `incVersion` flag from the /update web service when updating the document. + * Generic: Autoincremental version assigned to the registered entry. * * @apiNote Immutable */ diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/Study.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/Study.java index 6d7f55921aa..7f36ebe0547 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/Study.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/Study.java @@ -24,17 +24,10 @@ import org.opencb.opencga.core.api.FieldConstants; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.models.PrivateFields; -import org.opencb.opencga.core.models.clinical.ClinicalAnalysis; -import org.opencb.opencga.core.models.cohort.Cohort; import org.opencb.opencga.core.models.common.AdditionalInfo; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.common.ExternalSource; -import org.opencb.opencga.core.models.family.Family; -import org.opencb.opencga.core.models.file.File; -import org.opencb.opencga.core.models.individual.Individual; -import org.opencb.opencga.core.models.job.Job; -import org.opencb.opencga.core.models.panel.Panel; -import org.opencb.opencga.core.models.sample.Sample; +import org.opencb.opencga.core.models.notes.Note; import java.net.URI; import java.util.*; @@ -69,7 +62,7 @@ public class Study extends PrivateFields { description = FieldConstants.PROJECT_FQN) private String name; - @DataField(id = "name", indexed = true, + @DataField(id = "alias", indexed = true, description = FieldConstants.STUDY_ALIAS) private String alias; @@ -122,79 +115,6 @@ public class Study extends PrivateFields { description = FieldConstants.STUDY_GROUPS) private List groups; - /** - * A List with related files. - * - * @apiNote - */ - - @DataField(id = "files", - description = FieldConstants.STUDY_FILES) - private List files; - - /** - * A List with related jobs. - * - * @apiNote - */ - @DataField(id = "jobs", - description = FieldConstants.STUDY_JOBS) - private List jobs; - - /** - * A List with related individuals. - * - * @apiNote - */ - @DataField(id = "individuals", - description = FieldConstants.STUDY_INDIVIDUALS) - private List individuals; - - /** - * A List with related families. - * - * @apiNote - */ - @DataField(id = "families", - description = FieldConstants.STUDY_FAMILIES) - private List families; - - /** - * A List with related samples. - * - * @apiNote - */ - @DataField(id = "samples", - description = FieldConstants.STUDY_SAMPLES) - private List samples; - - /** - * A List with related cohorts. - * - * @apiNote - */ - @DataField(id = "cohorts", - description = FieldConstants.STUDY_COHORTS) - private List cohorts; - - /** - * A List with related panels. - * - * @apiNote - */ - @DataField(id = "panels", - description = FieldConstants.STUDY_PANELS) - private List panels; - - /** - * A List with related clinicalAnalyses. - * - * @apiNote - */ - @DataField(id = "clinicalAnalyses", - description = FieldConstants.STUDY_ANALYSES) - private List clinicalAnalyses; - /** * A List with related variableSets. * @@ -221,6 +141,9 @@ public class Study extends PrivateFields { description = FieldConstants.STUDY_EXTERNAL_SOURCES) private List sources; + @DataField(id = "notes", description = FieldConstants.STUDY_NOTES_DESCRIPTION) + private List notes; + @DataField(id = "type", indexed = true, description = FieldConstants.STUDY_TYPE) private StudyType type; @@ -264,17 +187,14 @@ public Study() { public Study(String name, String alias, String description, StudyType type, StudyInternal internal, URI uri, int release) { this(alias, name, alias, TimeUtils.getTime(), TimeUtils.getTime(), description, type, new LinkedList<>(), null, 0, - new LinkedList<>(), new LinkedList<>(), new LinkedList<>(), new LinkedList<>(), new LinkedList<>(), new LinkedList<>(), - new LinkedList<>(), new LinkedList<>(), new LinkedList<>(), new LinkedList<>(), new HashMap<>(), uri, release, - new Status(), internal, new LinkedList<>(), new HashMap<>()); + new LinkedList<>(), new LinkedList<>(), new HashMap<>(), uri, release, new Status(), internal, new LinkedList<>(), + new HashMap<>()); } public Study(String id, String name, String alias, String creationDate, String modificationDate, String description, StudyType type, - List sources, StudyNotification notification, long size, List groups, List files, - List jobs, List individuals, List families, List samples, List cohorts, - List panels, List clinicalAnalyses, - List variableSets, Map> permissionRules, URI uri, int release, - Status status, StudyInternal internal, List additionalInfo, Map attributes) { + List sources, StudyNotification notification, long size, List groups,List variableSets, + Map> permissionRules, URI uri, int release, Status status, StudyInternal internal, + List additionalInfo, Map attributes) { this.id = id; this.name = name; this.alias = alias; @@ -285,14 +205,6 @@ public Study(String id, String name, String alias, String creationDate, String m this.notification = notification; this.size = size; this.groups = ObjectUtils.defaultIfNull(groups, new ArrayList<>()); - this.files = ObjectUtils.defaultIfNull(files, new ArrayList<>()); - this.families = ObjectUtils.defaultIfNull(families, new ArrayList<>()); - this.jobs = ObjectUtils.defaultIfNull(jobs, new ArrayList<>()); - this.individuals = ObjectUtils.defaultIfNull(individuals, new ArrayList<>()); - this.samples = ObjectUtils.defaultIfNull(samples, new ArrayList<>()); - this.cohorts = ObjectUtils.defaultIfNull(cohorts, new ArrayList<>()); - this.panels = ObjectUtils.defaultIfNull(panels, new ArrayList<>()); - this.clinicalAnalyses = ObjectUtils.defaultIfNull(clinicalAnalyses, new ArrayList<>()); this.sources = sources; this.internal = internal; this.variableSets = ObjectUtils.defaultIfNull(variableSets, new ArrayList<>()); @@ -318,19 +230,12 @@ public String toString() { sb.append(", fqn='").append(fqn).append('\''); sb.append(", notification=").append(notification); sb.append(", groups=").append(groups); - sb.append(", files=").append(files); - sb.append(", jobs=").append(jobs); - sb.append(", individuals=").append(individuals); - sb.append(", families=").append(families); - sb.append(", samples=").append(samples); - sb.append(", cohorts=").append(cohorts); - sb.append(", panels=").append(panels); - sb.append(", clinicalAnalyses=").append(clinicalAnalyses); sb.append(", variableSets=").append(variableSets); sb.append(", permissionRules=").append(permissionRules); sb.append(", uri=").append(uri); sb.append(", release=").append(release); sb.append(", sources=").append(sources); + sb.append(", notes=").append(notes); sb.append(", type=").append(type); sb.append(", status=").append(status); sb.append(", internal=").append(internal); @@ -454,78 +359,6 @@ public Study setGroups(List groups) { return this; } - public List getFiles() { - return files; - } - - public Study setFiles(List files) { - this.files = files; - return this; - } - - public List getJobs() { - return jobs; - } - - public Study setJobs(List jobs) { - this.jobs = jobs; - return this; - } - - public List getIndividuals() { - return individuals; - } - - public Study setIndividuals(List individuals) { - this.individuals = individuals; - return this; - } - - public List getFamilies() { - return families; - } - - public Study setFamilies(List families) { - this.families = families; - return this; - } - - public List getSamples() { - return samples; - } - - public Study setSamples(List samples) { - this.samples = samples; - return this; - } - - public List getCohorts() { - return cohorts; - } - - public Study setCohorts(List cohorts) { - this.cohorts = cohorts; - return this; - } - - public List getPanels() { - return panels; - } - - public Study setPanels(List panels) { - this.panels = panels; - return this; - } - - public List getClinicalAnalyses() { - return clinicalAnalyses; - } - - public Study setClinicalAnalyses(List clinicalAnalyses) { - this.clinicalAnalyses = clinicalAnalyses; - return this; - } - public List getVariableSets() { return variableSets; } @@ -544,6 +377,15 @@ public Study setInternal(StudyInternal internal) { return this; } + public List getNotes() { + return notes; + } + + public Study setNotes(List notes) { + this.notes = notes; + return this; + } + public Map> getPermissionRules() { return permissionRules; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/StudyCreateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/StudyCreateParams.java index 40b5b5f3dc1..ddbf96f5e36 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/StudyCreateParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/StudyCreateParams.java @@ -76,8 +76,7 @@ public static StudyCreateParams of(Study study) { public Study toStudy() { return new Study(id, name, alias, creationDate, modificationDate, description, type, sources, notification, 0, null, null, null, - null, null, null, null, null, null, null, null, null, 0, status != null ? status.toStatus() : null, null, - additionalInfo, attributes); + null, 0, status != null ? status.toStatus() : null, null, additionalInfo, attributes); } @Override diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/StudyPermissions.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/StudyPermissions.java index 059af8182ab..75f627af7f3 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/StudyPermissions.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/StudyPermissions.java @@ -109,7 +109,10 @@ public enum Permissions { WRITE_CLINICAL_ANNOTATIONS(Arrays.asList(VIEW_CLINICAL_ANALYSIS, VIEW_CLINICAL_ANNOTATIONS), ClinicalAnalysisPermissions.WRITE_ANNOTATIONS.name(), CLINICAL_ANALYSIS), DELETE_CLINICAL_ANNOTATIONS(Arrays.asList(VIEW_CLINICAL_ANALYSIS, VIEW_CLINICAL_ANNOTATIONS, WRITE_CLINICAL_ANNOTATIONS), - ClinicalAnalysisPermissions.DELETE_ANNOTATIONS.name(), CLINICAL_ANALYSIS); + ClinicalAnalysisPermissions.DELETE_ANNOTATIONS.name(), CLINICAL_ANALYSIS), + ADMIN_CLINICAL_ANALYSIS(Arrays.asList(VIEW_CLINICAL_ANALYSIS, WRITE_CLINICAL_ANALYSIS, DELETE_CLINICAL_ANALYSIS, + VIEW_CLINICAL_ANNOTATIONS, WRITE_CLINICAL_ANNOTATIONS, DELETE_CLINICAL_ANNOTATIONS), + ClinicalAnalysisPermissions.ADMIN.name(), CLINICAL_ANALYSIS); private final static Map map; @@ -138,6 +141,14 @@ public static Permissions getStudyPermission(String permission, int type) { return map.get(permission + "-" + type); } + public String getPermission() { + return permission; + } + + public int getType() { + return type; + } + public List getImplicitPermissions() { return implicitPermissions; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/StudyVariantEngineConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/StudyVariantEngineConfiguration.java index 3bd986b2fb4..a0a47596cd1 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/StudyVariantEngineConfiguration.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/StudyVariantEngineConfiguration.java @@ -1,5 +1,6 @@ package org.opencb.opencga.core.models.study; +import org.opencb.commons.annotations.DataField; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.core.config.storage.SampleIndexConfiguration; @@ -8,6 +9,9 @@ public class StudyVariantEngineConfiguration { private ObjectMap options; private SampleIndexConfiguration sampleIndex; + @DataField(description = "Variant setup run", since = "3.2.0") + private VariantSetupResult setup; + public StudyVariantEngineConfiguration() { } @@ -34,11 +38,21 @@ public StudyVariantEngineConfiguration setSampleIndex(SampleIndexConfiguration s return this; } + public VariantSetupResult getSetup() { + return setup; + } + + public StudyVariantEngineConfiguration setSetup(VariantSetupResult setup) { + this.setup = setup; + return this; + } + @Override public String toString() { final StringBuilder sb = new StringBuilder("StudyVariantEngineConfiguration{"); - // sb.append("options=").append(options != null ? options.toJson() : ""); + sb.append("options=").append(options != null ? options.toJson() : ""); sb.append(", sampleIndex=").append(sampleIndex); + sb.append(", setup=").append(setup); sb.append('}'); return sb.toString(); } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/Variable.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/Variable.java index b6cf603f78f..772b168ccfc 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/Variable.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/Variable.java @@ -47,11 +47,7 @@ public class Variable { private long rank; private String dependsOn; private String description; - /** - * Variables for validate internal fields. Only valid if type is OBJECT. - **/ - @Deprecated - private Set variableSet; + private Set variables; private Map attributes; @@ -90,8 +86,6 @@ public enum VariableType { CATEGORICAL, INTEGER, DOUBLE, - @Deprecated - TEXT, STRING, OBJECT, MAP_BOOLEAN, @@ -229,16 +223,6 @@ public Variable setDescription(String description) { return this; } - @Deprecated - public Set getVariableSet() { - return getVariables(); - } - - @Deprecated - public Variable setVariableSet(Set variableSet) { - return setVariables(variableSet); - } - public Set getVariables() { return variables; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/VariableSetCreateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/VariableSetCreateParams.java index 1572f04a9c4..cfb06d84e6b 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/VariableSetCreateParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/VariableSetCreateParams.java @@ -22,7 +22,6 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.stream.Collectors; public class VariableSetCreateParams { @@ -57,7 +56,8 @@ public VariableSet toVariableSet() { Set variableSet = variables != null ? new HashSet<>(variables) : new HashSet<>(); - return new VariableSet(id, name, unique, confidential, false, description, variableSet, entities, 1, new ObjectMap()); + return new VariableSet(id, name, unique != null ? unique : true, confidential != null ? confidential : false, false, description, + variableSet, entities, 1, new ObjectMap()); } @Override diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/VariantSetupResult.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/VariantSetupResult.java new file mode 100644 index 00000000000..a1127f873f8 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/VariantSetupResult.java @@ -0,0 +1,74 @@ +package org.opencb.opencga.core.models.study; + +import org.opencb.commons.annotations.DataField; +import org.opencb.commons.datastore.core.ObjectMap; + +public class VariantSetupResult { + + @DataField(description = "User ID that started the setup run") + private String userId; + @DataField(description = "Date when the variant setup was executed") + private String date; + @DataField(description = "Variant setup status") + private Status status; + @DataField(description = "Input params for the variant setup") + private ObjectMap params; + + @DataField(description = "Generated variant storage configuration options given the input params.") + private ObjectMap options; + + public enum Status { + READY, + NOT_READY + } + + public VariantSetupResult() { + } + + public String getUserId() { + return userId; + } + + public VariantSetupResult setUserId(String userId) { + this.userId = userId; + return this; + } + + public String getDate() { + return date; + } + + public VariantSetupResult setDate(String date) { + this.date = date; + return this; + } + + public Status getStatus() { + return status; + } + + public VariantSetupResult setStatus(Status status) { + this.status = status; + return this; + } + + public ObjectMap getParams() { + return params; + } + + public VariantSetupResult setParams(ObjectMap params) { + this.params = params; + return this; + } + + public ObjectMap getOptions() { + return options; + } + + public VariantSetupResult setOptions(ObjectMap options) { + this.options = options; + return this; + } + + +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/configuration/ClinicalAnalysisStudyConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/configuration/ClinicalAnalysisStudyConfiguration.java index 97d09661407..5f71fe0c6b9 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/configuration/ClinicalAnalysisStudyConfiguration.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/configuration/ClinicalAnalysisStudyConfiguration.java @@ -1,26 +1,27 @@ package org.opencb.opencga.core.models.study.configuration; -import org.opencb.opencga.core.models.clinical.ClinicalAnalysis; -import org.opencb.opencga.core.models.common.FlagValue; import org.opencb.opencga.core.models.clinical.ClinicalStatusValue; +import org.opencb.opencga.core.models.common.FlagValue; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; public class ClinicalAnalysisStudyConfiguration { - private Map> status; + private List status; private InterpretationStudyConfiguration interpretation; private List priorities; - private Map> flags; + private List flags; private ClinicalConsentConfiguration consent; public ClinicalAnalysisStudyConfiguration() { } - public ClinicalAnalysisStudyConfiguration(Map> status, - InterpretationStudyConfiguration interpretation, List priorities, - Map> flags, ClinicalConsentConfiguration consent) { + public ClinicalAnalysisStudyConfiguration(List status, InterpretationStudyConfiguration interpretation, + List priorities, List flags, + ClinicalConsentConfiguration consent) { this.status = status; this.interpretation = interpretation; this.priorities = priorities; @@ -29,20 +30,21 @@ public ClinicalAnalysisStudyConfiguration(Map> status = new HashMap<>(); - Map> interpretationStatus = new HashMap<>(); + List clinicalStatusValueList = new ArrayList<>(4); + List interpretationStatusList = new ArrayList<>(3); List priorities = new ArrayList<>(5); - Map> flags = new HashMap<>(); List clinicalConsentList = new ArrayList<>(); - List clinicalStatusValueList = new ArrayList<>(4); clinicalStatusValueList.add( new ClinicalStatusValue("READY_FOR_INTERPRETATION", "The Clinical Analysis is ready for interpretations", ClinicalStatusValue.ClinicalStatusType.NOT_STARTED) ); clinicalStatusValueList.add( new ClinicalStatusValue("READY_FOR_REPORT", "The Interpretation is finished and it is to create the report", - ClinicalStatusValue.ClinicalStatusType.IN_PROGRESS) + ClinicalStatusValue.ClinicalStatusType.ACTIVE) + ); + clinicalStatusValueList.add( + new ClinicalStatusValue("DONE", "The Clinical Analysis is done", ClinicalStatusValue.ClinicalStatusType.DONE) ); clinicalStatusValueList.add( new ClinicalStatusValue("CLOSED", "The Clinical Analysis is closed", ClinicalStatusValue.ClinicalStatusType.CLOSED) @@ -50,21 +52,12 @@ public static ClinicalAnalysisStudyConfiguration defaultConfiguration() { clinicalStatusValueList.add( new ClinicalStatusValue("REJECTED", "The Clinical Analysis is rejected", ClinicalStatusValue.ClinicalStatusType.CLOSED) ); - status.put(ClinicalAnalysis.Type.FAMILY, clinicalStatusValueList); - status.put(ClinicalAnalysis.Type.AUTOCOMPARATIVE, clinicalStatusValueList); - status.put(ClinicalAnalysis.Type.CANCER, clinicalStatusValueList); - status.put(ClinicalAnalysis.Type.COHORT, clinicalStatusValueList); - status.put(ClinicalAnalysis.Type.SINGLE, clinicalStatusValueList); - List interpretationStatusList = new ArrayList<>(3); - interpretationStatusList.add(new ClinicalStatusValue("IN_PROGRESS", "Interpretation in progress", ClinicalStatusValue.ClinicalStatusType.IN_PROGRESS)); + interpretationStatusList.add(new ClinicalStatusValue("NOT_STARTED", "Interpretation not started", ClinicalStatusValue.ClinicalStatusType.NOT_STARTED)); + interpretationStatusList.add(new ClinicalStatusValue("IN_PROGRESS", "Interpretation in progress", ClinicalStatusValue.ClinicalStatusType.ACTIVE)); + interpretationStatusList.add(new ClinicalStatusValue("DONE", "Interpretation done", ClinicalStatusValue.ClinicalStatusType.DONE)); interpretationStatusList.add(new ClinicalStatusValue("READY", "Interpretation ready", ClinicalStatusValue.ClinicalStatusType.CLOSED)); interpretationStatusList.add(new ClinicalStatusValue("REJECTED", "Interpretation rejected", ClinicalStatusValue.ClinicalStatusType.CLOSED)); - interpretationStatus.put(ClinicalAnalysis.Type.FAMILY, interpretationStatusList); - interpretationStatus.put(ClinicalAnalysis.Type.AUTOCOMPARATIVE, interpretationStatusList); - interpretationStatus.put(ClinicalAnalysis.Type.CANCER, interpretationStatusList); - interpretationStatus.put(ClinicalAnalysis.Type.COHORT, interpretationStatusList); - interpretationStatus.put(ClinicalAnalysis.Type.SINGLE, interpretationStatusList); priorities.add(new ClinicalPriorityValue("URGENT", "Highest priority of all", 1, false)); priorities.add(new ClinicalPriorityValue("HIGH", "Second highest priority of all", 2, false)); @@ -80,20 +73,15 @@ public static ClinicalAnalysisStudyConfiguration defaultConfiguration() { flagValueList.add(new FlagValue("UNUSUAL_KARYOTYPE", "")); flagValueList.add(new FlagValue("SUSPECTED_MOSAICISM", "")); flagValueList.add(new FlagValue("LOW_QUALITY_SAMPLE", "")); - flags.put(ClinicalAnalysis.Type.FAMILY, flagValueList); - flags.put(ClinicalAnalysis.Type.AUTOCOMPARATIVE, flagValueList); - flags.put(ClinicalAnalysis.Type.CANCER, flagValueList); - flags.put(ClinicalAnalysis.Type.COHORT, flagValueList); - flags.put(ClinicalAnalysis.Type.SINGLE, flagValueList); clinicalConsentList.add(new ClinicalConsent("PRIMARY_FINDINGS", "Primary findings", "")); clinicalConsentList.add(new ClinicalConsent("SECONDARY_FINDINGS", "Secondary findings", "")); clinicalConsentList.add(new ClinicalConsent("CARRIER_FINDINGS", "Carrier findings", "")); clinicalConsentList.add(new ClinicalConsent("RESEARCH_FINDINGS", "Research findings", "")); - return new ClinicalAnalysisStudyConfiguration(status, - new InterpretationStudyConfiguration(interpretationStatus, Collections.emptyList(), Collections.emptyMap(), - Collections.emptyList()), priorities, flags, new ClinicalConsentConfiguration(clinicalConsentList)); + return new ClinicalAnalysisStudyConfiguration(clinicalStatusValueList, + new InterpretationStudyConfiguration(interpretationStatusList, Collections.emptyList(), Collections.emptyMap(), + Collections.emptyList()), priorities, flagValueList, new ClinicalConsentConfiguration(clinicalConsentList)); } @Override @@ -108,11 +96,11 @@ public String toString() { return sb.toString(); } - public Map> getStatus() { + public List getStatus() { return status; } - public ClinicalAnalysisStudyConfiguration setStatus(Map> status) { + public ClinicalAnalysisStudyConfiguration setStatus(List status) { this.status = status; return this; } @@ -135,11 +123,11 @@ public ClinicalAnalysisStudyConfiguration setPriorities(List> getFlags() { + public List getFlags() { return flags; } - public ClinicalAnalysisStudyConfiguration setFlags(Map> flags) { + public ClinicalAnalysisStudyConfiguration setFlags(List flags) { this.flags = flags; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/configuration/InterpretationStudyConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/configuration/InterpretationStudyConfiguration.java index b038fd5674d..0b9c98a780a 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/configuration/InterpretationStudyConfiguration.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/configuration/InterpretationStudyConfiguration.java @@ -1,6 +1,5 @@ package org.opencb.opencga.core.models.study.configuration; -import org.opencb.opencga.core.models.clinical.ClinicalAnalysis; import org.opencb.opencga.core.models.clinical.ClinicalStatusValue; import java.util.List; @@ -8,16 +7,16 @@ public class InterpretationStudyConfiguration { - private Map> status; + private List status; private List variantCallers; + @Deprecated private Map defaultFilter; private List inclusion; public InterpretationStudyConfiguration() { } - public InterpretationStudyConfiguration(Map> status, - List variantCallers, + public InterpretationStudyConfiguration(List status, List variantCallers, Map defaultFilter, List inclusion) { this.status = status; this.variantCallers = variantCallers; @@ -36,11 +35,11 @@ public String toString() { return sb.toString(); } - public Map> getStatus() { + public List getStatus() { return status; } - public InterpretationStudyConfiguration setStatus(Map> status) { + public InterpretationStudyConfiguration setStatus(List status) { this.status = status; return this; } @@ -54,10 +53,12 @@ public InterpretationStudyConfiguration setVariantCallers(List getDefaultFilter() { return defaultFilter; } + @Deprecated public InterpretationStudyConfiguration setDefaultFilter(Map defaultFilter) { this.defaultFilter = defaultFilter; return this; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/Account.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/Account.java index 2e05785205e..293ade331be 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/Account.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/Account.java @@ -27,79 +27,77 @@ */ public class Account { - @DataField(id = "type", indexed = true, uncommentedClasses = {"AccountType"}, - description = FieldConstants.ACCOUNT_TYPE) - private AccountType type; - - @DataField(id = "creationDate", indexed = true, - description = FieldConstants.GENERIC_CREATION_DATE_DESCRIPTION) - private String creationDate; - @DataField(id = "expirationDate", indexed = true, - description = FieldConstants.ACCOUNT_EXPIRATION_DATE_DESCRIPTION) + description = FieldConstants.INTERNAL_ACCOUNT_EXPIRATION_DATE_DESCRIPTION) private String expirationDate; + @DataField(id = "password", since = "3.2.1", description = FieldConstants.INTERNAL_ACCOUNT_PASSWORD_DESCRIPTION) + private Password password; + + @DataField(id = "failedAttempts", description = FieldConstants.INTERNAL_ACCOUNT_FAILED_ATTEMPTS_DESCRIPTION) + private int failedAttempts; @DataField(id = "authentication", indexed = true, uncommentedClasses = {"AccountType"}, - description = FieldConstants.ACCOUNT_AUTHENTICATION) + description = FieldConstants.INTERNAL_ACCOUNT_AUTHENTICATION) private AuthenticationOrigin authentication; public Account() { String creationDate = TimeUtils.getTime(); + // Default 1 year Calendar cal = Calendar.getInstance(); cal.setTime(TimeUtils.toDate(creationDate)); cal.add(Calendar.YEAR, +1); String expirationDate = TimeUtils.getTime(cal.getTime()); - this.type = AccountType.FULL; - this.creationDate = creationDate; this.expirationDate = expirationDate; + this.password = new Password(); + this.failedAttempts = 0; this.authentication = null; } - public Account(AccountType type, String creationDate, String expirationDate, AuthenticationOrigin authentication) { - this.type = type; + public Account(String expirationDate, Password password, int failedAttempts, AuthenticationOrigin authentication) { this.expirationDate = expirationDate; - this.creationDate = creationDate; + this.password = password; + this.failedAttempts = failedAttempts; this.authentication = authentication; } @Override public String toString() { final StringBuilder sb = new StringBuilder("Account{"); - sb.append("type='").append(type).append('\''); - sb.append(", creationDate='").append(creationDate).append('\''); - sb.append(", expirationDate='").append(expirationDate).append('\''); - sb.append(", authentication='").append(authentication).append('\''); + sb.append("expirationDate='").append(expirationDate).append('\''); + sb.append(", password=").append(password); + sb.append(", failedAttempts=").append(failedAttempts); + sb.append(", authentication=").append(authentication); sb.append('}'); return sb.toString(); } - public AccountType getType() { - return type; + public String getExpirationDate() { + return expirationDate; } - public Account setType(AccountType type) { - this.type = type; + public Account setExpirationDate(String expirationDate) { + this.expirationDate = expirationDate; return this; } - public String getExpirationDate() { - return expirationDate; + public Password getPassword() { + return password; } - public Account setExpirationDate(String expirationDate) { - this.expirationDate = expirationDate; + public Account setPassword(Password password) { + this.password = password; return this; } - public String getCreationDate() { - return creationDate; + public int getFailedAttempts() { + return failedAttempts; } - public Account setCreationDate(String creationDate) { - this.creationDate = creationDate; + public Account setFailedAttempts(int failedAttempts) { + this.failedAttempts = failedAttempts; return this; } @@ -112,12 +110,6 @@ public Account setAuthentication(AuthenticationOrigin authentication) { return this; } - public enum AccountType { - GUEST, - FULL, - ADMINISTRATOR - } - public static class AuthenticationOrigin { private String id; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/LoginParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/LoginParams.java index fcac1bd932c..aaa31bc395e 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/LoginParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/LoginParams.java @@ -16,8 +16,12 @@ package org.opencb.opencga.core.models.user; +import com.fasterxml.jackson.annotation.JsonAlias; + public class LoginParams { + @JsonAlias({"organizationId"}) + private String organization; private String user; private String password; private String refreshToken; @@ -25,16 +29,18 @@ public class LoginParams { public LoginParams() { } - public LoginParams(String user, String password) { - this.user = user; - this.password = password; - } - public LoginParams(String refreshToken) { this.refreshToken = refreshToken; } - public LoginParams(String user, String password, String refreshToken) { + public LoginParams(String organization, String user, String password) { + this.organization = organization; + this.user = user; + this.password = password; + } + + public LoginParams(String organization, String user, String password, String refreshToken) { + this.organization = organization; this.user = user; this.password = password; this.refreshToken = refreshToken; @@ -43,13 +49,23 @@ public LoginParams(String user, String password, String refreshToken) { @Override public String toString() { final StringBuilder sb = new StringBuilder("LoginParams{"); - sb.append("user='").append(user).append('\''); - sb.append(", password='").append("********").append('\''); - sb.append(", refreshToken='").append("********").append('\''); + sb.append("organization='").append(organization).append('\''); + sb.append(", user='").append(user).append('\''); + sb.append(", password='").append(password).append('\''); + sb.append(", refreshToken='").append(refreshToken).append('\''); sb.append('}'); return sb.toString(); } + public String getOrganization() { + return organization; + } + + public LoginParams setOrganization(String organization) { + this.organization = organization; + return this; + } + public String getUser() { return user; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/OrganizationUserUpdateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/OrganizationUserUpdateParams.java new file mode 100644 index 00000000000..938ebd6dc7f --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/OrganizationUserUpdateParams.java @@ -0,0 +1,148 @@ +package org.opencb.opencga.core.models.user; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.core.JsonProcessingException; +import org.opencb.commons.datastore.core.ObjectMap; + +import java.util.Map; + +import static org.opencb.opencga.core.common.JacksonUtils.getUpdateObjectMapper; + +public class OrganizationUserUpdateParams extends UserUpdateParams { + + private UserQuota quota; + private Internal internal; + private Map attributes; + + public OrganizationUserUpdateParams() { + } + + public OrganizationUserUpdateParams(String name, String email, UserQuota quota, Internal internal, Map attributes) { + super(name, email); + this.quota = quota; + this.internal = internal; + this.attributes = attributes; + } + + @JsonIgnore + public ObjectMap getUpdateMap() throws JsonProcessingException { + return new ObjectMap(getUpdateObjectMapper().writeValueAsString(this)); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("OrganizationUserUpdateParams{"); + sb.append("quota=").append(quota); + sb.append(", internal=").append(internal); + sb.append(", attributes=").append(attributes); + sb.append('}'); + return sb.toString(); + } + + public UserQuota getQuota() { + return quota; + } + + public OrganizationUserUpdateParams setQuota(UserQuota quota) { + this.quota = quota; + return this; + } + + @Deprecated + @JsonIgnore + public Account getAccount() { + return getInternal().getAccount(); + } + + @Deprecated + public OrganizationUserUpdateParams setAccount(Account account) { + if (internal == null) { + internal = new Internal(); + } + internal.setAccount(account); + return this; + } + + public Internal getInternal() { + return internal; + } + + public OrganizationUserUpdateParams setInternal(Internal internal) { + this.internal = internal; + return this; + } + + public Map getAttributes() { + return attributes; + } + + public OrganizationUserUpdateParams setAttributes(Map attributes) { + this.attributes = attributes; + return this; + } + + @Override + public OrganizationUserUpdateParams setName(String name) { + super.setName(name); + return this; + } + + @Override + public OrganizationUserUpdateParams setEmail(String email) { + super.setEmail(email); + return this; + } + + public static class Internal { + private Account account; + + public Internal() { + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("Internal{"); + sb.append("account=").append(account); + sb.append('}'); + return sb.toString(); + } + + public Account getAccount() { + return account; + } + + public Internal setAccount(Account account) { + this.account = account; + return this; + } + } + + public static class Account { + private String expirationDate; + + public Account() { + } + + public Account(String expirationDate) { + this.expirationDate = expirationDate; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("Account{"); + sb.append("expirationDate='").append(expirationDate).append('\''); + sb.append('}'); + return sb.toString(); + } + + public String getExpirationDate() { + return expirationDate; + } + + public Account setExpirationDate(String expirationDate) { + this.expirationDate = expirationDate; + return this; + } + } + +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/Password.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/Password.java new file mode 100644 index 00000000000..5b375936bd3 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/Password.java @@ -0,0 +1,48 @@ +package org.opencb.opencga.core.models.user; + +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.FieldConstants; + +public class Password { + + @DataField(id = "expirationDate", since = "3.2.1", description = FieldConstants.INTERNAL_ACCOUNT_PASSWORD_EXPIRATION_DATE_DESCRIPTION) + private String expirationDate; + + @DataField(id = "lastModified", since = "3.2.1", description = FieldConstants.INTERNAL_ACCOUNT_PASSWORD_LAST_MODIFIED_DESCRIPTION) + private String lastModified; + + public Password() { + } + + public Password(String expirationDate, String lastModified) { + this.expirationDate = expirationDate; + this.lastModified = lastModified; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("Password{"); + sb.append("expirationDate='").append(expirationDate).append('\''); + sb.append(", lastModified='").append(lastModified).append('\''); + sb.append('}'); + return sb.toString(); + } + + public String getExpirationDate() { + return expirationDate; + } + + public Password setExpirationDate(String expirationDate) { + this.expirationDate = expirationDate; + return this; + } + + public String getLastModified() { + return lastModified; + } + + public Password setLastModified(String lastModified) { + this.lastModified = lastModified; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/PasswordChangeParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/PasswordChangeParams.java index 09e3485c33f..8fa2be47eb1 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/PasswordChangeParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/PasswordChangeParams.java @@ -20,6 +20,8 @@ public class PasswordChangeParams { + @JsonProperty(required = true) + private String organizationId; @JsonProperty(required = true) private String user; @JsonProperty(required = false) @@ -32,22 +34,41 @@ public class PasswordChangeParams { public PasswordChangeParams() { } + @Deprecated public PasswordChangeParams(String user, String oldPassword, String newPassword) { this.user = user; this.password = oldPassword; this.newPassword = newPassword; } + public PasswordChangeParams(String organizationId, String user, String password, String newPassword) { + this.organizationId = organizationId; + this.user = user; + this.password = password; + this.newPassword = newPassword; + } + @Override public String toString() { final StringBuilder sb = new StringBuilder("PasswordChangeParams{"); - sb.append("user='").append(user).append('\''); - sb.append(", password='").append("********").append('\''); - sb.append(", newPassword='").append("********").append('\''); + sb.append("organizationId='").append(organizationId).append('\''); + sb.append(", user='").append(user).append('\''); + sb.append(", password='").append(password).append('\''); + sb.append(", newPassword='").append(newPassword).append('\''); + sb.append(", reset='").append(reset).append('\''); sb.append('}'); return sb.toString(); } + public String getOrganizationId() { + return organizationId; + } + + public PasswordChangeParams setOrganizationId(String organizationId) { + this.organizationId = organizationId; + return this; + } + public String getUser() { return user; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/User.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/User.java index d84f2910cae..98273a80528 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/User.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/User.java @@ -40,34 +40,29 @@ public class User { description = FieldConstants.GENERIC_ID_DESCRIPTION) private String id; - @DataField(id = "name", indexed = true, - description = FieldConstants.USER_NAME) + @DataField(id = "name", indexed = true, description = FieldConstants.USER_NAME) private String name; - @DataField(id = "email", indexed = true, - description = FieldConstants.USER_EMAIL) + @DataField(id = "email", indexed = true, description = FieldConstants.USER_EMAIL) private String email; - @DataField(id = "organization", indexed = true, - description = FieldConstants.USER_ORGANIZATION) + @DataField(id = "organization", indexed = true, description = FieldConstants.USER_ORGANIZATION) private String organization; - @DataField(id = "account", indexed = true, - description = FieldConstants.USER_ACCOUNT) - private Account account; + @DataField(id = "creationDate", since = "3.2.1", description = FieldConstants.GENERIC_CREATION_DATE_DESCRIPTION) + private String creationDate; - /** - * An object describing the internal information of the User. This is managed by OpenCGA. - * - * @apiNote Internal - */ + @DataField(id = "modificationDate", since = "3.2.1", description = FieldConstants.GENERIC_MODIFICATION_DATE_DESCRIPTION) + private String modificationDate; - @DataField(id = "internal", indexed = true, - description = FieldConstants.GENERIC_INTERNAL) + @DataField(id = "account", description = FieldConstants.USER_ACCOUNT) + @Deprecated + private Account account; + + @DataField(id = "internal", indexed = true, description = FieldConstants.GENERIC_INTERNAL) private UserInternal internal; - @DataField(id = "quota", indexed = true, - description = FieldConstants.USER_QUOTA) + @DataField(id = "quota", indexed = true, description = FieldConstants.USER_QUOTA) private UserQuota quota; /** @@ -75,21 +70,13 @@ public class User { * * @apiNote */ - @DataField(id = "projects", indexed = true, - description = FieldConstants.USER_PROJECTS) + @DataField(id = "projects", indexed = true, description = FieldConstants.USER_PROJECTS) private List projects; - @DataField(id = "sharedProjects", indexed = true, - description = FieldConstants.USER_SHARED_PROJECTS) - private List sharedProjects; - - - @DataField(id = "configs", indexed = true, - description = FieldConstants.USER_CONFIGS) + @DataField(id = "configs", indexed = true, description = FieldConstants.USER_CONFIGS) private Map configs; - @DataField(id = "filters", indexed = true, - description = FieldConstants.USER_FILTERS) + @DataField(id = "filters", indexed = true, description = FieldConstants.USER_FILTERS) private List filters; /** @@ -97,35 +84,46 @@ public class User { * * @apiNote */ - @DataField(id = "attributes", indexed = true, - description = FieldConstants.GENERIC_ATTRIBUTES_DESCRIPTION) + @DataField(id = "attributes", indexed = true, description = FieldConstants.GENERIC_ATTRIBUTES_DESCRIPTION) private Map attributes; public User() { } - public User(String id, Account account) { - this(id, id, null, null, account, new UserInternal(new UserStatus()), null, Collections.emptyList(), Collections.emptyList(), - Collections.emptyMap(), new LinkedList<>(), Collections.emptyMap()); + public User(String id) { + this(id, id, null, null, null, null, new UserInternal(new UserStatus()), null, Collections.emptyMap(), + new LinkedList<>(), Collections.emptyMap()); } public User(String id, String name, String email, String organization, UserInternal internal) { - this(id, name, email, organization, null, internal, null, new ArrayList<>(), new ArrayList<>(), new HashMap<>(), new LinkedList<>(), - new HashMap<>()); + this(id, name, email, organization, null, null, internal, null, new HashMap<>(), new LinkedList<>(), new HashMap<>()); } - public User(String id, String name, String email, String organization, Account account, UserInternal internal, UserQuota quota, - List projects, List sharedProjects, Map configs, List filters, - Map attributes) { + public User(String id, String name, String email, String organization, UserInternal internal, UserQuota quota, List projects, + Map configs, List filters, Map attributes) { this.id = id; this.name = name; this.email = email; this.organization = organization; - this.account = account != null ? account : new Account(); this.internal = internal; this.quota = quota; this.projects = projects; - this.sharedProjects = sharedProjects; + this.configs = configs; + this.filters = filters; + this.attributes = attributes; + } + + public User(String id, String name, String email, String organization, String creationDate, String modificationDate, + UserInternal internal, UserQuota quota, Map configs, List filters, + Map attributes) { + this.id = id; + this.name = name; + this.email = email; + this.organization = organization; + this.creationDate = creationDate; + this.modificationDate = modificationDate; + this.internal = internal; + this.quota = quota; this.configs = configs; this.filters = filters; this.attributes = attributes; @@ -138,11 +136,11 @@ public String toString() { sb.append(", name='").append(name).append('\''); sb.append(", email='").append(email).append('\''); sb.append(", organization='").append(organization).append('\''); - sb.append(", account=").append(account); + sb.append(", creationDate='").append(creationDate).append('\''); + sb.append(", modificationDate='").append(modificationDate).append('\''); sb.append(", internal=").append(internal); sb.append(", quota=").append(quota); sb.append(", projects=").append(projects); - sb.append(", sharedProjects=").append(sharedProjects); sb.append(", configs=").append(configs); sb.append(", filters=").append(filters); sb.append(", attributes=").append(attributes); @@ -186,10 +184,30 @@ public User setOrganization(String organization) { return this; } + public String getCreationDate() { + return creationDate; + } + + public User setCreationDate(String creationDate) { + this.creationDate = creationDate; + return this; + } + + public String getModificationDate() { + return modificationDate; + } + + public User setModificationDate(String modificationDate) { + this.modificationDate = modificationDate; + return this; + } + + @Deprecated public Account getAccount() { return account; } + @Deprecated public User setAccount(Account account) { this.account = account; return this; @@ -222,15 +240,6 @@ public User setProjects(List projects) { return this; } - public List getSharedProjects() { - return sharedProjects; - } - - public User setSharedProjects(List sharedProjects) { - this.sharedProjects = sharedProjects; - return this; - } - public Map getConfigs() { return configs; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserInternal.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserInternal.java index 9a4b5fc9e23..55a75709bd1 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserInternal.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserInternal.java @@ -16,21 +16,43 @@ package org.opencb.opencga.core.models.user; -public class UserInternal { +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.FieldConstants; +import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.models.common.Internal; +public class UserInternal extends Internal { + + @DataField(id = "status", description = FieldConstants.INTERNAL_STATUS_DESCRIPTION) private UserStatus status; + @DataField(id = "account", since = "3.2.1", description = FieldConstants.USER_ACCOUNT) + private Account account; + public UserInternal() { } public UserInternal(UserStatus status) { - this.status = status; + this(TimeUtils.getTime(), TimeUtils.getTime(), status, new Account()); + } + + public UserInternal(UserStatus status, Account account) { + this(TimeUtils.getTime(), TimeUtils.getTime(), status, account); + } + + public UserInternal(String registrationDate, String lastModified, UserStatus status1, Account account) { + super(null, registrationDate, lastModified); + this.status = status1; + this.account = account; } @Override public String toString() { final StringBuilder sb = new StringBuilder("UserInternal{"); sb.append("status=").append(status); + sb.append(", account=").append(account); + sb.append(", registrationDate='").append(registrationDate).append('\''); + sb.append(", lastModified='").append(lastModified).append('\''); sb.append('}'); return sb.toString(); } @@ -43,4 +65,14 @@ public UserInternal setStatus(UserStatus status) { this.status = status; return this; } + + public Account getAccount() { + return account; + } + + public UserInternal setAccount(Account account) { + this.account = account; + return this; + } + } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserStatus.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserStatus.java index bf497bd00f0..b9220d8fdce 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserStatus.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserStatus.java @@ -24,8 +24,9 @@ public class UserStatus extends InternalStatus { public static final String BANNED = "BANNED"; + public static final String SUSPENDED = "SUSPENDED"; - public static final List STATUS_LIST = Arrays.asList(READY, DELETED, BANNED); + public static final List STATUS_LIST = Arrays.asList(READY, DELETED, BANNED, SUSPENDED); public UserStatus(String status, String message) { if (isValid(status)) { @@ -47,7 +48,7 @@ public static boolean isValid(String status) { if (InternalStatus.isValid(status)) { return true; } - if (status != null && (status.equals(BANNED))) { + if (status != null && (status.equals(BANNED) || status.equals(SUSPENDED))) { return true; } return false; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserStatusUpdateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserStatusUpdateParams.java new file mode 100644 index 00000000000..5c0356ded8e --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserStatusUpdateParams.java @@ -0,0 +1,30 @@ +package org.opencb.opencga.core.models.user; + +public class UserStatusUpdateParams { + + private String status; + + public UserStatusUpdateParams() { + } + + public UserStatusUpdateParams(String status) { + this.status = status; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("UserStatusUpdateParams{"); + sb.append("status='").append(status).append('\''); + sb.append('}'); + return sb.toString(); + } + + public String getStatus() { + return status; + } + + public UserStatusUpdateParams setStatus(String status) { + this.status = status; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserUpdateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserUpdateParams.java index 7b069623e44..eda30607fbb 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserUpdateParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserUpdateParams.java @@ -16,23 +16,17 @@ package org.opencb.opencga.core.models.user; -import java.util.Map; - public class UserUpdateParams { private String name; private String email; - private String organization; - private Map attributes; public UserUpdateParams() { } - public UserUpdateParams(String name, String email, String organization, Map attributes) { + public UserUpdateParams(String name, String email) { this.name = name; this.email = email; - this.organization = organization; - this.attributes = attributes; } @Override @@ -40,8 +34,6 @@ public String toString() { final StringBuilder sb = new StringBuilder("UserUpdateParams{"); sb.append("name='").append(name).append('\''); sb.append(", email='").append(email).append('\''); - sb.append(", organization='").append(organization).append('\''); - sb.append(", attributes=").append(attributes); sb.append('}'); return sb.toString(); } @@ -64,21 +56,4 @@ public UserUpdateParams setEmail(String email) { return this; } - public String getOrganization() { - return organization; - } - - public UserUpdateParams setOrganization(String organization) { - this.organization = organization; - return this; - } - - public Map getAttributes() { - return attributes; - } - - public UserUpdateParams setAttributes(Map attributes) { - this.attributes = attributes; - return this; - } } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantSetupParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantSetupParams.java new file mode 100644 index 00000000000..1a97c0de539 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantSetupParams.java @@ -0,0 +1,173 @@ +package org.opencb.opencga.core.models.variant; + +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.tools.ToolParams; + +import java.util.List; + +public class VariantSetupParams extends ToolParams { + + @DataField(description = "Expected number of samples that will be loaded. Used to infer some parameters. " + + "This number is only used as a hint. " + + "If the real number of samples is different, if it grows beyond expectation, or if , the loader should be able to handle it.", + required = true) + private Integer expectedSamples; + + @DataField(description = "Expected number of files that will be loaded. Used to infer some parameters. " + + "This number is only used as a hint. " + + "If the real number of files is different, the loader should be able to handle it.", required = true) + private Integer expectedFiles; + + @DataField(description = "Main type of the files that will be loaded. If the dataset contains multiple types of files," + + " provide the one that matches most of the files.") + private FileType fileType; + + @DataField(description = "Average size of the files that will be loaded. This number is only used as a hint. " + + "If the real size of the files is different, the loader should be able to handle it. Accepts units. e.g. 435MB, 2GB, 86KB. " + + "If not provided, the value will be inferred from the file type.") + private String averageFileSize; + + @DataField(description = "Number of variants per sample (non hom_ref variants). This number is only used as a hint. " + + "If the real number of variants per sample is different, the loader should be able to handle it. " + + "If not provided, the value will be inferred from the file type.") + private Integer variantsPerSample; + + @DataField(description = "Average number of samples per file. This number is only used as a hint. " + + "If the real number of samples per file is different, the loader should be able to handle it. " + + "If not provided, the value will be inferred from the expectedSamples and expectedFiles and dataDistribution.") + private Float averageSamplesPerFile; + + @DataField(description = "Data distribution of the files. This parameter is used to infer the number of samples per file.") + private DataDistribution dataDistribution; + + @DataField(description = "List of normalization extensions that will be used to normalize the files.") + private List normalizeExtensions; + + public VariantSetupParams(VariantSetupParams params) { + this.expectedSamples = params.expectedSamples; + this.expectedFiles = params.expectedFiles; + this.fileType = params.fileType; + this.averageFileSize = params.averageFileSize; + this.variantsPerSample = params.variantsPerSample; + this.averageSamplesPerFile = params.averageSamplesPerFile; + this.dataDistribution = params.dataDistribution; + this.normalizeExtensions = params.normalizeExtensions; + } + + public VariantSetupParams() { + } + + public enum DataDistribution { + // Single sample VCF files. One file per sample. + // e.g. + // - Platinum gVCF + // - Cancer germline + // - RD germline without family calling + @DataField(description = "Single sample VCF files. One file per sample. e.g. Platinum gVCF, Cancer germline, RD germline without family calling") + SINGLE_SAMPLE_PER_FILE, + + // Multi samples VCF files. One file with multiple samples. + // e.g. + // - Corpasome + // - RD germline with family calling + @DataField(description = "Multi samples VCF files. One file with multiple samples. e.g. Corpasome, RD germline with family calling") + MULTIPLE_SAMPLES_PER_FILE, + + // Multiple files per sample. Each file might have multiple samples. + // e.g. + // - Somatic study with multiple callers + @DataField(description = "Multiple files per sample. Each file might have multiple samples. e.g. Somatic study with multiple callers") + MULTIPLE_FILES_PER_SAMPLE, + + // Large aggregated/joined/merged files. Each file has all samples. Each file contains a specific set of chromosomes. + // e.g. + // - 1000 genomes + @DataField(description = "Large aggregated/joined/merged files. Each file has all samples. Each file contains a specific set of chromosomes. e.g. 1000 genomes") + FILES_SPLIT_BY_CHROMOSOME, + + // Large aggregated/joined/merged files. Each file has all samples. Each file contains an arbitrary region. + @DataField(description = "Large aggregated/joined/merged files. Each file has all samples. Each file contains an arbitrary region.") + FILES_SPLIT_BY_REGION, + } + + public enum FileType { + @DataField(description = "Whole genome VCF file.") + GENOME_VCF, + @DataField(description = "Whole genome gVCF file.") + GENOME_gVCF, + @DataField(description = "Exome VCF file.") + EXOME + } + + public Integer getExpectedSamples() { + return expectedSamples; + } + + public VariantSetupParams setExpectedSamples(Integer expectedSamples) { + this.expectedSamples = expectedSamples; + return this; + } + + public Integer getExpectedFiles() { + return expectedFiles; + } + + public VariantSetupParams setExpectedFiles(Integer expectedFiles) { + this.expectedFiles = expectedFiles; + return this; + } + + public FileType getFileType() { + return fileType; + } + + public VariantSetupParams setFileType(FileType fileType) { + this.fileType = fileType; + return this; + } + + public String getAverageFileSize() { + return averageFileSize; + } + + public VariantSetupParams setAverageFileSize(String averageFileSize) { + this.averageFileSize = averageFileSize; + return this; + } + + public Integer getVariantsPerSample() { + return variantsPerSample; + } + + public VariantSetupParams setVariantsPerSample(Integer variantsPerSample) { + this.variantsPerSample = variantsPerSample; + return this; + } + + public Float getAverageSamplesPerFile() { + return averageSamplesPerFile; + } + + public VariantSetupParams setAverageSamplesPerFile(Float averageSamplesPerFile) { + this.averageSamplesPerFile = averageSamplesPerFile; + return this; + } + + public DataDistribution getDataDistribution() { + return dataDistribution; + } + + public VariantSetupParams setDataDistribution(DataDistribution dataDistribution) { + this.dataDistribution = dataDistribution; + return this; + } + + public List getNormalizeExtensions() { + return normalizeExtensions; + } + + public VariantSetupParams setNormalizeExtensions(List normalizeExtensions) { + this.normalizeExtensions = normalizeExtensions; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/response/OpenCGAResult.java b/opencga-core/src/main/java/org/opencb/opencga/core/response/OpenCGAResult.java index fb331f8a326..59af48fbef8 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/response/OpenCGAResult.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/response/OpenCGAResult.java @@ -30,6 +30,10 @@ public class OpenCGAResult extends DataResult { public OpenCGAResult() { } + public OpenCGAResult(int time, List results) { + super(time, Collections.emptyList(), results.size(), results, results.size()); + } + public OpenCGAResult(int time, List events, int numResults, List results, long numMatches) { super(time, events, numResults, results, numMatches); } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/tools/OpenCgaToolExecutor.java b/opencga-core/src/main/java/org/opencb/opencga/core/tools/OpenCgaToolExecutor.java index 15cf7a51215..0b16875bebb 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/tools/OpenCgaToolExecutor.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/tools/OpenCgaToolExecutor.java @@ -17,9 +17,10 @@ package org.opencb.opencga.core.tools; import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.opencga.core.tools.annotations.ToolExecutor; +import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.exceptions.ToolExecutorException; +import org.opencb.opencga.core.tools.annotations.ToolExecutor; import org.opencb.opencga.core.tools.result.ExecutionResultManager; import java.nio.file.Path; @@ -31,6 +32,7 @@ public abstract class OpenCgaToolExecutor { private ObjectMap executorParams; private Path outDir; private ExecutionResultManager arm; + private Configuration configuration; protected OpenCgaToolExecutor() { } @@ -51,10 +53,11 @@ public final ToolExecutor.Source getSource() { return this.getClass().getAnnotation(ToolExecutor.class).source(); } - public final void setUp(ExecutionResultManager arm, ObjectMap executorParams, Path outDir) { + public final void setUp(ExecutionResultManager arm, ObjectMap executorParams, Path outDir, Configuration configuration) { this.arm = arm; this.executorParams = executorParams; this.outDir = outDir; + this.configuration = configuration; } public final void execute() throws ToolException { @@ -77,6 +80,10 @@ public final Path getOutDir() { return outDir; } + public final Configuration getConfiguration() { + return configuration; + } + protected final String getToken() { return getExecutorParams().getString("token"); } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/tools/annotations/Tool.java b/opencga-core/src/main/java/org/opencb/opencga/core/tools/annotations/Tool.java index caedbc63194..3bf38d51053 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/tools/annotations/Tool.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/tools/annotations/Tool.java @@ -52,6 +52,11 @@ */ String description() default ""; + /** + * @return Tool priority. + */ + Enums.Priority priority() default Enums.Priority.MEDIUM; + enum Type { OPERATION, ANALYSIS; @@ -59,6 +64,7 @@ enum Type { enum Scope { GLOBAL, + ORGANIZATION, PROJECT, STUDY } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/tools/variant/GwasAnalysisExecutor.java b/opencga-core/src/main/java/org/opencb/opencga/core/tools/variant/GwasAnalysisExecutor.java index 2af4d769513..980a0a8d494 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/tools/variant/GwasAnalysisExecutor.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/tools/variant/GwasAnalysisExecutor.java @@ -33,7 +33,7 @@ public abstract class GwasAnalysisExecutor extends OpenCgaToolExecutor { private String cohort1; private String cohort2; private Path outputFile; - private GwasConfiguration configuration; + private GwasConfiguration gwasConfiguration; public GwasAnalysisExecutor() { } @@ -49,7 +49,7 @@ protected List getHeaderColumns() { columns.add("gene"); columns.add("biotype"); columns.add("consequence-types"); - if (configuration.getMethod() == GwasConfiguration.Method.CHI_SQUARE_TEST) { + if (gwasConfiguration.getMethod() == GwasConfiguration.Method.CHI_SQUARE_TEST) { columns.add("chi-square"); } columns.add("p-value"); @@ -70,7 +70,7 @@ public String toString() { sb.append(", phenotype2='").append(phenotype2).append('\''); sb.append(", cohort1='").append(cohort1).append('\''); sb.append(", cohort2='").append(cohort2).append('\''); - sb.append(", configuration=").append(configuration); + sb.append(", configuration=").append(gwasConfiguration); sb.append(", executorParams=").append(getExecutorParams()); sb.append(", outDir=").append(getOutDir()); sb.append('}'); @@ -149,12 +149,12 @@ public GwasAnalysisExecutor setOutputFile(Path outputFile) { return this; } - public GwasConfiguration getConfiguration() { - return configuration; + public GwasConfiguration getGwasConfiguration() { + return gwasConfiguration; } - public GwasAnalysisExecutor setConfiguration(GwasConfiguration configuration) { - this.configuration = configuration; + public GwasAnalysisExecutor setGwasConfiguration(GwasConfiguration gwasConfiguration) { + this.gwasConfiguration = gwasConfiguration; return this; } } diff --git a/opencga-core/src/main/resources/configuration.yml b/opencga-core/src/main/resources/configuration.yml index ad99aa68f79..0b2b007817e 100644 --- a/opencga-core/src/main/resources/configuration.yml +++ b/opencga-core/src/main/resources/configuration.yml @@ -6,6 +6,11 @@ databasePrefix: ${OPENCGA.DB.PREFIX} workspace: ${OPENCGA.USER.WORKSPACE} jobDir: ${OPENCGA.USER.WORKSPACE}/jobs +# Maximum number of login attempts before banning a user account +account: + maxLoginAttempts: ${OPENCGA.ACCOUNT.MAX_LOGIN_ATTEMPTS} + passwordExpirationDays: 0 + panel: host: "http://resources.opencb.org/opencb/opencga/disease-panels" @@ -19,56 +24,6 @@ catalog: options: authenticationDatabase: ${OPENCGA.CATALOG.DB.AUTHENTICATION_DATABASE} connectionsPerHost: ${OPENCGA.CATALOG.DB.CONNECTIONS_PER_HOST} - ## Solr Search engine configuration, by default is the same than storage - searchEngine: - # List of hosts pointing either to the Solr nodes directly using a complete URL or to the zookeper nodes with HOST:PORT - # Example for Solr connection: http://opencga-solr-01.zone:8983/solr - # Example for Zookeeper connection: opencga-zookeeper-01:2181 <-- Recommended for replicated installations - hosts: - - ${OPENCGA.CATALOG.SEARCH.HOST} - user: "" - password: "" - options: - mode: "cloud" - timeout: ${OPENCGA.CATALOG.SEARCH.TIMEOUT} - insertBatchSize: ${OPENCGA.CATALOG.SEARCH.BATCH} - -## We support multiple Authentication providers, if none is provided then we use an internal authentication implementation -authentication: - # Session expiration time in seconds - expiration: 3600 - authenticationOrigins: -# Custom LDAP | LDAPS configuration example -# - id: ldaps # Any id -# type: LDAP -# host: ldaps://localhost:636 # or ldap://localhost:123 -# options: -# authUserId: "" # Auth user id and password if querying the LDAP system requires authentication. By default, empty. -# authPassword: "" # Auth user id and password if querying the LDAP system requires authentication. By default, empty. -# usersSearch: dc=ge,dc=co,dc=uk # Mandatory field. Base search to look for users. By default, empty. -# groupsSearch: ou=general,ou=groups,dc=ge,dc=co,dc=uk # Mandatory field. Base search to look for groups. By default, empty. -# fullNameKey: displayname # Key to get the user's full name when importing users. By default, 'displayname'. -# memberKey: member # Key within groups to extract the user id. By default, 'member'. -# dnKey: dn # Key to get the user's DN or RDN unique identifier. By default, 'dn'. -# dnFormat: "%s" # Formatter to extract the user's DN if it is contained within a larger string. By default, '%s' to take the whole string. -# uidKey: uid # Key to get the user id to be used in OpenCGA. By default, 'uid'. -# uidFormat: "%s" # Formatter to extract the user id if the uid is contained within a larger string. By default, '%s' to take the whole string. -# sslInvalidCertificatesAllowed: false # If using LDAPs with a custom certificate, set to true to accept any certificate. By default, false. -# connectionTimeout: 500 # Connection timeout value. Default: 500 ms -# readTimeout: 1000 # Read timeout value. Default: 1000 ms - -# Azure AD configuration example -# - id: aad # Any id -# type: AzureAD -# host: -# options: -# tenantId: xxxx # Mandatory. Tenant id -# authClientId: xxxx # Mandatory. Client id of the client with permissions to authenticate users. -# syncClientId: xxxx # Mandatory. Client id of the client with permissions to inspect active directory. -# syncSecretKey: xxxx # Mandatory: Secret key of the client with permissions to inspect active directory. -# filters: tokenField1=aa,bb,cc;tokenField2=aa,bb,cc # Optional. Filters to be applied. OpenCGA will check if tokenField1 = aa or bb -# # or cc and tokenField2 = aa or bb or cc. If any of the filters don't succeed, even if the user is properly authenticated -# # in AAD, the user will not be able to generate a token and login in OpenCGA. server: rest: @@ -88,9 +43,6 @@ server: grpc: port: ${OPENCGA.SERVER.GRPC.PORT} -optimizations: - simplifyPermissions: ${OPENCGA_OPTIMIZATIONS_SIMPLIFY_PERMISSIONS} - audit: manager: "" # Java manager of the audit implementation to be used to audit. If empty, catalog database will be used. maxDocuments: 20000000 # Maximum number of documents that will be created in the audit collection. @@ -115,7 +67,32 @@ healthCheck: analysis: - scratchDir: "${OPENCGA.ANALYSIS.SCRATCH.DIR}" # Scratch folder for the analysis. + # List of packages where to find analysis tools + packages: + - "org.opencb.opencga" + # Scratch folder for the analysis. + scratchDir: "${OPENCGA.ANALYSIS.SCRATCH.DIR}" + # Default URL for downloading analysis resources. + resourceUrl: "http://resources.opencb.org/opencb/opencga/analysis/" + # Docker used by OpenCGA analysis and containing external tools such as samtools, bcftools, tabix, fastqc, plink1.9, bwa and r-base + # You can indicate the version, e.g: opencb/opencga-ext-tools:2.12.0, otherwise the current OpenCGA version will be used + opencgaExtTools: "opencb/opencga-ext-tools" + tools: + - id: "exomiser" + version: "13.1" + dockerId: "exomiser/exomiser-cli:13.1.0" + resources: + HG19: "exomiser/2109_hg19.zip" + HG38: "exomiser/2109_hg38.zip" + PHENOTYPE: "exomiser/2109_phenotype.zip" + - id: "exomiser" + version: "14.0" + defaultVersion: true + dockerId: "exomiser/exomiser-cli:14.0.0" + resources: + HG19: "exomiser/2402_hg19.zip" + HG38: "exomiser/2402_hg38.zip" + PHENOTYPE: "exomiser/2402_phenotype.zip" execution: # Accepted values are "local", "SGE", "azure-batch", "k8s" # see org.opencb.opencga.master.monitor.executors.ExecutorFactory @@ -171,6 +148,8 @@ analysis: ## Kubernetes executor configuration example # k8s.masterUrl: "https://192.168.99.100:8443/" k8s.clientTimeout: 30000 # ms + k8s.terminationGracePeriodSeconds: 300 # s + k8s.logToStdout: true k8s.imageName: "opencb/opencga-base:${project.parent.version}-hdp3.1" k8s.imagePullPolicy: "IfNotPresent" # k8s.imagePullSecrets: diff --git a/opencga-core/src/test/java/org/opencb/opencga/core/cellbase/CellBaseValidatorTest.java b/opencga-core/src/test/java/org/opencb/opencga/core/cellbase/CellBaseValidatorTest.java index 5eee5ed2945..82535278561 100644 --- a/opencga-core/src/test/java/org/opencb/opencga/core/cellbase/CellBaseValidatorTest.java +++ b/opencga-core/src/test/java/org/opencb/opencga/core/cellbase/CellBaseValidatorTest.java @@ -5,12 +5,15 @@ import org.junit.Assume; import org.junit.Rule; import org.junit.Test; +import org.junit.experimental.categories.Category; import org.junit.rules.ExpectedException; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.config.storage.CellBaseConfiguration; +import org.opencb.opencga.core.testclassification.duration.ShortTests; import java.io.IOException; +@Category(ShortTests.class) public class CellBaseValidatorTest { @Rule diff --git a/opencga-core/src/test/java/org/opencb/opencga/core/config/ConfigurationTest.java b/opencga-core/src/test/java/org/opencb/opencga/core/config/ConfigurationTest.java index 5a65e947bf8..618797dc5c0 100644 --- a/opencga-core/src/test/java/org/opencb/opencga/core/config/ConfigurationTest.java +++ b/opencga-core/src/test/java/org/opencb/opencga/core/config/ConfigurationTest.java @@ -40,29 +40,14 @@ public void testDefault() throws IOException { configuration.setLogLevel("INFO"); configuration.setWorkspace("/opt/opencga/sessions"); - - configuration.setAdmin(new Admin()); - - Authentication authentication = new Authentication(); - configuration.setAuthentication(authentication); - configuration.setMonitor(new Monitor()); configuration.getAnalysis().setExecution(new Execution()); - configuration.setHooks(Collections.singletonMap("user@project:study", Collections.singletonMap("file", + configuration.setHooks(Collections.singletonMap("organization@project:study", Collections.singletonMap("file", Collections.singletonList( new HookConfiguration("name", "~*SV*", HookConfiguration.Stage.CREATE, HookConfiguration.Action.ADD, "tags", "SV") )))); - List authenticationOriginList = new ArrayList<>(); - authenticationOriginList.add(new AuthenticationOrigin("opencga", AuthenticationOrigin.AuthenticationType.OPENCGA.toString(), - "localhost", Collections.emptyMap())); - Map myMap = new HashMap<>(); - myMap.put("ou", "People"); - authenticationOriginList.add(new AuthenticationOrigin("opencga", AuthenticationOrigin.AuthenticationType.LDAP.toString(), - "ldap://10.10.0.20:389", myMap)); - configuration.getAuthentication().setAuthenticationOrigins(authenticationOriginList); - Email emailServer = new Email("localhost", "", "", "", "", false); configuration.setEmail(emailServer); diff --git a/opencga-core/src/test/resources/configuration-test.yml b/opencga-core/src/test/resources/configuration-test.yml index dad67f8c5dc..2c3736938b7 100644 --- a/opencga-core/src/test/resources/configuration-test.yml +++ b/opencga-core/src/test/resources/configuration-test.yml @@ -4,6 +4,7 @@ logDir: null databasePrefix: "opencga_test" workspace: "/tmp/opencga/sessions" +maxLoginAttempts: 5 audit: manager: "" # Java manager of the audit implementation to be used to audit. If empty, catalog database will be used. @@ -32,16 +33,6 @@ catalog: password: "" options: authenticationDatabase: "" -authentication: - expiration: 1000 -#LDAP configuration example - authenticationOrigins: - - id: ldap # Any id - type: LDAP # At the moment, we only support LDAP - host: ldap://localhost:9000 - options: - usersSearch: dc=ge,dc=co,dc=uk # Base search to look for the users - groupsSearch: ou=general,ou=groups,dc=ge,dc=co,dc=uk # Base search to look for the groups server: rest: diff --git a/opencga-master/pom.xml b/opencga-master/pom.xml index 8c8ab880796..73192c29e58 100644 --- a/opencga-master/pom.xml +++ b/opencga-master/pom.xml @@ -62,6 +62,11 @@ org.opencb.opencga opencga-catalog + + org.opencb.opencga + opencga-storage-core + test + org.eclipse.jetty @@ -71,6 +76,11 @@ org.eclipse.jetty jetty-servlet + + org.mortbay.jetty + servlet-api + test + com.microsoft.azure @@ -133,6 +143,18 @@ io.fabric8 kubernetes-model + + io.fabric8 + kubernetes-model-core + + + io.fabric8 + kubernetes-client-api + + + io.fabric8 + kubernetes-model-batch + commons-codec commons-codec diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/MonitorService.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/MonitorService.java index 5b63b01078a..86b801b0390 100644 --- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/MonitorService.java +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/MonitorService.java @@ -26,6 +26,7 @@ import org.glassfish.jersey.servlet.ServletContainer; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.CatalogManager; +import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.config.storage.StorageConfiguration; import org.opencb.opencga.master.monitor.daemons.ExecutionDaemon; @@ -63,7 +64,6 @@ public class MonitorService { protected static Logger logger; - public MonitorService(Configuration configuration, StorageConfiguration storageConfiguration, String appHome, String token) throws CatalogException { this.configuration = configuration; @@ -95,14 +95,16 @@ private void init(String token) throws CatalogException { logger = LoggerFactory.getLogger(this.getClass()); this.catalogManager = new CatalogManager(this.configuration); - String nonExpiringToken = this.catalogManager.getUserManager().getNonExpiringToken(OPENCGA, Collections.emptyMap(), token); + String nonExpiringToken = this.catalogManager.getUserManager().getNonExpiringToken(ParamConstants.ADMIN_ORGANIZATION, OPENCGA, + Collections.emptyMap(), token); executionDaemon = new ExecutionDaemon( configuration.getMonitor().getExecutionDaemonInterval(), nonExpiringToken, catalogManager, storageConfiguration, - appHome); + appHome, + configuration.getAnalysis().getPackages()); // fileDaemon = new FileDaemon(configuration.getMonitor().getFileDaemonInterval(), // configuration.getMonitor().getDaysToRemove(), nonExpiringToken, catalogManager); diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/AuthorizationDaemon.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/AuthorizationDaemon.java index 00ba6d18a68..c4c2ab43ae5 100644 --- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/AuthorizationDaemon.java +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/AuthorizationDaemon.java @@ -16,24 +16,12 @@ package org.opencb.opencga.master.monitor.daemons; -import org.opencb.commons.datastore.core.Query; -import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager; -import org.opencb.opencga.catalog.db.api.DBIterator; import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogDBException; -import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.CatalogManager; -import org.opencb.opencga.core.common.TimeUtils; -import org.opencb.opencga.core.models.common.Enums; -import org.opencb.opencga.core.models.study.PermissionRule; -import org.opencb.opencga.core.models.study.Study; -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -public class AuthorizationDaemon extends MonitorParentDaemon { +public abstract class AuthorizationDaemon extends MonitorParentDaemon { private final String INTERNAL_DELIMITER = "__"; @@ -44,81 +32,83 @@ public class AuthorizationDaemon extends MonitorParentDaemon { public AuthorizationDaemon(int interval, String sessionId, CatalogManager catalogManager) throws CatalogDBException { super(interval, sessionId, catalogManager); - this.studyDBAdaptor = dbAdaptorFactory.getCatalogStudyDBAdaptor(); +// this.studyDBAdaptor = dbAdaptorFactory.getCatalogStudyDBAdaptor(); this.authorizationManager = catalogManager.getAuthorizationManager(); } @Override public void run() { - - Query allStudies = new Query(); - QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, - Arrays.asList(StudyDBAdaptor.QueryParams.PERMISSION_RULES.key(), StudyDBAdaptor.QueryParams.ID.key(), - StudyDBAdaptor.QueryParams.UID.key())); - - while (!exit) { - try { - Thread.sleep(interval); - } catch (InterruptedException e) { - if (!exit) { - e.printStackTrace(); - } - } - logger.info("----- AUTHORIZATION DAEMON -----", TimeUtils.getTimeMillis()); - - try (DBIterator iterator = studyDBAdaptor.iterator(allStudies, options)) { - while (iterator.hasNext()) { - applyPermissionRules(iterator.next()); - } - } catch (Exception e) { - logger.error("{}", e.getMessage(), e); - } - } +// +// Query allStudies = new Query(); +// QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, +// Arrays.asList(StudyDBAdaptor.QueryParams.PERMISSION_RULES.key(), StudyDBAdaptor.QueryParams.ID.key(), +// StudyDBAdaptor.QueryParams.UID.key())); +// +// while (!exit) { +// try { +// Thread.sleep(interval); +// } catch (InterruptedException e) { +// if (!exit) { +// e.printStackTrace(); +// } +// } +// logger.info("----- AUTHORIZATION DAEMON -----", TimeUtils.getTimeMillis()); +// +// try (DBIterator iterator = studyDBAdaptor.iterator(allStudies, options)) { +// while (iterator.hasNext()) { +// applyPermissionRules(iterator.next()); +// } +// } catch (Exception e) { +// logger.error("{}", e.getMessage(), e); +// } +// } } - private void applyPermissionRules(Study study) { - if (study.getPermissionRules().isEmpty()) { - return; - } - - logger.info("Analysing study {} ({})", study.getId(), study.getUid()); - - for (Map.Entry> myMap : study.getPermissionRules().entrySet()) { - Enums.Entity entry = myMap.getKey(); - for (PermissionRule permissionRule : myMap.getValue()) { - try { - String[] split = permissionRule.getId().split(INTERNAL_DELIMITER, 2); - if (split.length == 1) { - // Apply rules - logger.info("Attempting to apply permission rule {} in {}", permissionRule.getId(), entry); - authorizationManager.applyPermissionRule(study.getUid(), permissionRule, entry); - } else { - // Remove permission rule - PermissionRule.DeleteAction deleteAction = PermissionRule.DeleteAction.valueOf(split[1].split("_")[1]); - switch (deleteAction) { - case NONE: - logger.info("Removing permission rule {}", permissionRule.getId().split(INTERNAL_DELIMITER)[0], - entry); - authorizationManager.removePermissionRule(study.getUid(), permissionRule.getId(), entry); - break; - case REVERT: - logger.info("Removing permission rule {} and reverting applied permissions for {}", - permissionRule.getId().split(INTERNAL_DELIMITER)[0], entry); - authorizationManager.removePermissionRuleAndRestorePermissions(study, permissionRule.getId(), entry); - break; - case REMOVE: - default: - logger.info("Removing permission rule {} and removing applied permissions for {}", - permissionRule.getId().split(INTERNAL_DELIMITER)[0], entry); - authorizationManager.removePermissionRuleAndRemovePermissions(study, permissionRule.getId(), entry); - break; - } - } - - } catch (CatalogException e) { - logger.error("{}", e.getMessage(), e); - } - } - } - } +// private void applyPermissionRules(Study study) { +// if (study.getPermissionRules().isEmpty()) { +// return; +// } +// +// logger.info("Analysing study {} ({})", study.getId(), study.getUid()); +// +// for (Map.Entry> myMap : study.getPermissionRules().entrySet()) { +// Enums.Entity entry = myMap.getKey(); +// for (PermissionRule permissionRule : myMap.getValue()) { +// try { +// String[] split = permissionRule.getId().split(INTERNAL_DELIMITER, 2); +// if (split.length == 1) { +// // Apply rules +// logger.info("Attempting to apply permission rule {} in {}", permissionRule.getId(), entry); +// authorizationManager.applyPermissionRule(organizationId, study.getUid(), permissionRule, entry); +// } else { +// // Remove permission rule +// PermissionRule.DeleteAction deleteAction = PermissionRule.DeleteAction.valueOf(split[1].split("_")[1]); +// switch (deleteAction) { +// case NONE: +// logger.info("Removing permission rule {}", permissionRule.getId().split(INTERNAL_DELIMITER)[0], +// entry); +// authorizationManager.removePermissionRule(organizationId, study.getUid(), permissionRule.getId(), entry); +// break; +// case REVERT: +// logger.info("Removing permission rule {} and reverting applied permissions for {}", +// permissionRule.getId().split(INTERNAL_DELIMITER)[0], entry); +// authorizationManager.removePermissionRuleAndRestorePermissions(organizationId, study, +// permissionRule.getId(), entry); +// break; +// case REMOVE: +// default: +// logger.info("Removing permission rule {} and removing applied permissions for {}", +// permissionRule.getId().split(INTERNAL_DELIMITER)[0], entry); +// authorizationManager.removePermissionRuleAndRemovePermissions(organizationId, study, +// permissionRule.getId(), entry); +// break; +// } +// } +// +// } catch (CatalogException e) { +// logger.error("{}", e.getMessage(), e); +// } +// } +// } +// } } diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java index 5ab2fee0840..01c02bde4ad 100644 --- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java @@ -34,18 +34,13 @@ import org.opencb.opencga.analysis.clinical.tiering.CancerTieringInterpretationAnalysis; import org.opencb.opencga.analysis.clinical.tiering.TieringInterpretationAnalysis; import org.opencb.opencga.analysis.clinical.zetta.ZettaInterpretationAnalysis; -import org.opencb.opencga.analysis.cohort.CohortIndexTask; import org.opencb.opencga.analysis.cohort.CohortTsvAnnotationLoader; -import org.opencb.opencga.analysis.family.FamilyIndexTask; import org.opencb.opencga.analysis.family.FamilyTsvAnnotationLoader; import org.opencb.opencga.analysis.family.qc.FamilyQcAnalysis; import org.opencb.opencga.analysis.file.*; -import org.opencb.opencga.analysis.individual.IndividualIndexTask; import org.opencb.opencga.analysis.individual.IndividualTsvAnnotationLoader; import org.opencb.opencga.analysis.individual.qc.IndividualQcAnalysis; -import org.opencb.opencga.analysis.job.JobIndexTask; import org.opencb.opencga.analysis.panel.PanelImportTask; -import org.opencb.opencga.analysis.sample.SampleIndexTask; import org.opencb.opencga.analysis.sample.SampleTsvAnnotationLoader; import org.opencb.opencga.analysis.sample.qc.SampleQcAnalysis; import org.opencb.opencga.analysis.templates.TemplateRunner; @@ -74,6 +69,7 @@ import org.opencb.opencga.analysis.wrappers.plink.PlinkWrapperAnalysis; import org.opencb.opencga.analysis.wrappers.rvtests.RvtestsWrapperAnalysis; import org.opencb.opencga.analysis.wrappers.samtools.SamtoolsWrapperAnalysis; +import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager; import org.opencb.opencga.catalog.db.api.DBIterator; import org.opencb.opencga.catalog.db.api.FileDBAdaptor; import org.opencb.opencga.catalog.db.api.JobDBAdaptor; @@ -87,6 +83,7 @@ import org.opencb.opencga.catalog.managers.FileManager; import org.opencb.opencga.catalog.managers.JobManager; import org.opencb.opencga.catalog.managers.StudyManager; +import org.opencb.opencga.catalog.utils.CatalogFqn; import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.api.ParamConstants; @@ -95,6 +92,7 @@ import org.opencb.opencga.core.config.Execution; import org.opencb.opencga.core.config.storage.StorageConfiguration; import org.opencb.opencga.core.models.AclEntryList; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.file.File; import org.opencb.opencga.core.models.file.FileAclParams; @@ -108,6 +106,7 @@ import org.opencb.opencga.core.tools.result.ExecutionResultManager; import org.opencb.opencga.core.tools.result.Status; import org.opencb.opencga.master.monitor.models.PrivateJobUpdateParams; +import org.opencb.opencga.master.monitor.schedulers.JobScheduler; import javax.ws.rs.ProcessingException; import javax.ws.rs.client.Client; @@ -115,6 +114,7 @@ import javax.ws.rs.client.Entity; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import java.io.Closeable; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; @@ -137,19 +137,19 @@ /** * Created by imedina on 16/06/16. */ -public class ExecutionDaemon extends MonitorParentDaemon { +public class ExecutionDaemon extends MonitorParentDaemon implements Closeable { public static final String OUTDIR_PARAM = "outdir"; public static final int EXECUTION_RESULT_FILE_EXPIRATION_SECONDS = (int) TimeUnit.MINUTES.toSeconds(10); public static final String REDACTED_TOKEN = "xxxxxxxxxxxxxxxxxxxxx"; private final StorageConfiguration storageConfiguration; - private String internalCli; - private JobManager jobManager; - private FileManager fileManager; + private final String internalCli; + private final JobManager jobManager; + private final FileManager fileManager; private final Map jobsCountByType = new HashMap<>(); private final Map retainedLogsTime = new HashMap<>(); - private Path defaultJobDir; + private List packages; private static final Map TOOL_CLI_MAP; @@ -158,39 +158,33 @@ public class ExecutionDaemon extends MonitorParentDaemon { // On first iteration, it will queue 50 out of the 100 pending jobs. It will check up to 50 queue-running changes out of the 65 // (15 + 50 from pending), and it will check up to 50 finished jobs from the running ones. // On second iteration, it will queue the remaining 50 pending jobs, and so on... - private static final int NUM_JOBS_HANDLED = 50; + private static final int MAX_NUM_JOBS = 50; private final Query pendingJobsQuery; private final Query queuedJobsQuery; private final Query runningJobsQuery; private final QueryOptions queryOptions; private final ExecutorService executor = Executors.newSingleThreadExecutor(); + private final JobScheduler jobScheduler; static { TOOL_CLI_MAP = new HashMap() {{ put(FileUnlinkTask.ID, "files unlink"); put(FileDeleteTask.ID, "files delete"); put(FetchAndRegisterTask.ID, "files fetch"); - put(FileIndexTask.ID, "files secondary-index"); put(FileTsvAnnotationLoader.ID, "files tsv-load"); put(PostLinkSampleAssociation.ID, "files postlink"); - put(SampleIndexTask.ID, "samples secondary-index"); put(SampleTsvAnnotationLoader.ID, "samples tsv-load"); - put(IndividualIndexTask.ID, "individuals secondary-index"); put(IndividualTsvAnnotationLoader.ID, "individuals tsv-load"); - put(CohortIndexTask.ID, "cohorts secondary-index"); put(CohortTsvAnnotationLoader.ID, "cohorts tsv-load"); - put(FamilyIndexTask.ID, "families secondary-index"); put(FamilyTsvAnnotationLoader.ID, "families tsv-load"); put(PanelImportTask.ID, "panels import"); - put(JobIndexTask.ID, "jobs secondary-index"); - put("alignment-index-run", "alignment index-run"); put("coverage-index-run", "alignment coverage-index-run"); put("alignment-stats-run", "alignment stats-run"); @@ -211,7 +205,6 @@ public class ExecutionDaemon extends MonitorParentDaemon { put(RvtestsWrapperAnalysis.ID, "variant " + RvtestsWrapperAnalysis.ID + "-run"); put(GatkWrapperAnalysis.ID, "variant " + GatkWrapperAnalysis.ID + "-run"); put(ExomiserWrapperAnalysis.ID, "variant " + ExomiserWrapperAnalysis.ID + "-run"); - put(VariantFileDeleteOperationTool.ID, "variant file-delete"); put(VariantSecondaryAnnotationIndexOperationTool.ID, "variant secondary-index"); put(VariantSecondaryIndexSamplesDeleteOperationTool.ID, "variant secondary-index-delete"); put(VariantScoreDeleteOperationTool.ID, "variant score-delete"); @@ -251,9 +244,13 @@ public class ExecutionDaemon extends MonitorParentDaemon { }}; } - public ExecutionDaemon(int interval, String token, - CatalogManager catalogManager, StorageConfiguration storageConfiguration, String appHome) - throws CatalogDBException { + public ExecutionDaemon(int interval, String token, CatalogManager catalogManager, StorageConfiguration storageConfiguration, + String appHome) throws CatalogDBException { + this(interval, token, catalogManager, storageConfiguration, appHome, Collections.singletonList(ToolFactory.DEFAULT_PACKAGE)); + } + + public ExecutionDaemon(int interval, String token, CatalogManager catalogManager, StorageConfiguration storageConfiguration, + String appHome, List packages) throws CatalogDBException { super(interval, token, catalogManager); this.jobManager = catalogManager.getJobManager(); @@ -261,35 +258,34 @@ public ExecutionDaemon(int interval, String token, this.storageConfiguration = storageConfiguration; this.internalCli = appHome + "/bin/opencga-internal.sh"; - this.defaultJobDir = Paths.get(catalogManager.getConfiguration().getJobDir()); - pendingJobsQuery = new Query(JobDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), Enums.ExecutionStatus.PENDING); queuedJobsQuery = new Query(JobDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), Enums.ExecutionStatus.QUEUED); runningJobsQuery = new Query(JobDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), Enums.ExecutionStatus.RUNNING); // Sort jobs by priority and creation date queryOptions = new QueryOptions() - .append(QueryOptions.SORT, Arrays.asList(JobDBAdaptor.QueryParams.PRIORITY.key(), - JobDBAdaptor.QueryParams.CREATION_DATE.key())) - .append(QueryOptions.ORDER, QueryOptions.ASCENDING); + .append(QueryOptions.SORT, + Arrays.asList(JobDBAdaptor.QueryParams.PRIORITY.key(), JobDBAdaptor.QueryParams.CREATION_DATE.key())) + .append(QueryOptions.ORDER, QueryOptions.ASCENDING) + .append(QueryOptions.LIMIT, MAX_NUM_JOBS); + + this.jobScheduler = new JobScheduler(catalogManager, token); + + if (CollectionUtils.isEmpty(packages)) { + this.packages = Collections.singletonList(ToolFactory.DEFAULT_PACKAGE); + } else { + this.packages = packages; + } + logger.info("Packages where to find tools/analyses: " + StringUtils.join(this.packages, ", ")); } @Override - public void run() { - while (!exit) { - try { - Thread.sleep(interval); - } catch (InterruptedException e) { - if (!exit) { - e.printStackTrace(); - } - } + public void apply() throws Exception { + checkJobs(); + } - try { - checkJobs(); - } catch (Exception e) { - logger.error("Catch exception " + e.getMessage(), e); - } - } + @Override + public void close() throws IOException { + batchExecutor.close(); try { logger.info("Attempt to shutdown webhook executor"); @@ -306,54 +302,84 @@ public void run() { } } - protected void checkJobs() { - long pendingJobs = -1; - long queuedJobs = -1; - long runningJobs = -1; - try { - pendingJobs = jobManager.count(pendingJobsQuery, token).getNumMatches(); - queuedJobs = jobManager.count(queuedJobsQuery, token).getNumMatches(); - runningJobs = jobManager.count(runningJobsQuery, token).getNumMatches(); - } catch (CatalogException e) { - logger.error("{}", e.getMessage(), e); + protected void checkJobs() throws CatalogException { + List organizationIds = catalogManager.getAdminManager().getOrganizationIds(token); + + /* + RUNNING JOBS + */ + checkRunningJobs(organizationIds); + + /* + QUEUED JOBS + */ + checkQueuedJobs(organizationIds); + + long totalPendingJobs = 0; + long totalQueuedJobs = 0; + long totalRunningJobs = 0; + for (String organizationId : organizationIds) { + long pendingJobs = -1; + long queuedJobs = -1; + long runningJobs = -1; + try { + pendingJobs = jobManager.countInOrganization(organizationId, pendingJobsQuery, token).getNumMatches(); + queuedJobs = jobManager.countInOrganization(organizationId, queuedJobsQuery, token).getNumMatches(); + runningJobs = jobManager.countInOrganization(organizationId, runningJobsQuery, token).getNumMatches(); + } catch (CatalogException e) { + logger.error("{}", e.getMessage(), e); + } + logger.info("----- EXECUTION DAEMON ----- Organization={} --> pending={}, queued={}, running={}", organizationId, pendingJobs, + queuedJobs, runningJobs); + totalPendingJobs += pendingJobs; + totalQueuedJobs += queuedJobs; + totalRunningJobs += runningJobs; + } + + if (totalQueuedJobs == 0) { + // Check PENDING jobs + checkPendingJobs(organizationIds); } - logger.info("----- EXECUTION DAEMON ----- pending={}, queued={}, running={}", pendingJobs, queuedJobs, runningJobs); - - /* - PENDING JOBS - */ - checkPendingJobs(); - - /* - QUEUED JOBS - */ - checkQueuedJobs(); - - /* - RUNNING JOBS - */ - checkRunningJobs(); } - protected void checkRunningJobs() { - int handledRunningJobs = 0; - try (DBIterator iterator = jobManager.iterator(runningJobsQuery, queryOptions, token)) { - while (handledRunningJobs < NUM_JOBS_HANDLED && iterator.hasNext()) { - try { - Job job = iterator.next(); - handledRunningJobs += checkRunningJob(job); - } catch (Exception e) { - logger.error("{}", e.getMessage(), e); + protected void checkRunningJobs(List organizationIds) { + for (String organizationId : organizationIds) { + int handledRunningJobs = 0; + try (DBIterator iterator = jobManager.iteratorInOrganization(organizationId, runningJobsQuery, queryOptions, token)) { + while (handledRunningJobs < MAX_NUM_JOBS && iterator.hasNext()) { + try { + Job job = iterator.next(); + handledRunningJobs += checkRunningJob(job); + } catch (Exception e) { + logger.error("{}", e.getMessage(), e); + } } + } catch (Exception e) { + logger.error("{}", e.getMessage(), e); } - } catch (Exception e) { - logger.error("{}", e.getMessage(), e); } } protected int checkRunningJob(Job job) { Enums.ExecutionStatus jobStatus = getCurrentStatus(job); + if (killSignalSent(job)) { + logger.info("[{}] - Kill signal request received for job with status='{}'. Attempting to abort execution.", job.getId(), + job.getInternal().getStatus().getId()); + try { + if (batchExecutor.kill(job.getId())) { + return abortKillJob(job, "Job was already in execution. Job killed by the user."); + } else { + logger.info("[{}] - Kill signal send. Waiting for job to finish.", job.getId()); + return 0; + } + } catch (Exception e) { + // Skip this job. Will be retried next loop iteration + logger.error("[{}] - Error trying to kill the job: {}", job.getId(), e.getMessage(), e); + return 0; + } + } + switch (jobStatus.getId()) { case Enums.ExecutionStatus.RUNNING: ExecutionResult result = readExecutionResult(job); @@ -392,19 +418,21 @@ protected int checkRunningJob(Job job) { } } - protected void checkQueuedJobs() { - int handledQueuedJobs = 0; - try (DBIterator iterator = jobManager.iterator(queuedJobsQuery, queryOptions, token)) { - while (handledQueuedJobs < NUM_JOBS_HANDLED && iterator.hasNext()) { - try { - Job job = iterator.next(); - handledQueuedJobs += checkQueuedJob(job); - } catch (Exception e) { - logger.error("{}", e.getMessage(), e); + protected void checkQueuedJobs(List organizationIds) { + for (String organizationId : organizationIds) { + int handledQueuedJobs = 0; + try (DBIterator iterator = jobManager.iteratorInOrganization(organizationId, queuedJobsQuery, queryOptions, token)) { + while (handledQueuedJobs < MAX_NUM_JOBS && iterator.hasNext()) { + try { + Job job = iterator.next(); + handledQueuedJobs += checkQueuedJob(job); + } catch (Exception e) { + logger.error("{}", e.getMessage(), e); + } } + } catch (Exception e) { + logger.error("{}", e.getMessage(), e); } - } catch (Exception e) { - logger.error("{}", e.getMessage(), e); } } @@ -417,6 +445,24 @@ protected void checkQueuedJobs() { protected int checkQueuedJob(Job job) { Enums.ExecutionStatus status = getCurrentStatus(job); + // If the job is already running, let the running-jobs step check it + if (killSignalSent(job) && !status.getId().equals(Enums.ExecutionStatus.RUNNING)) { + logger.info("[{}] - Kill signal request received for job with status='{}'. Attempting to avoid execution.", job.getId(), + job.getInternal().getStatus().getId()); + try { + if (batchExecutor.kill(job.getId())) { + return abortKillJob(job, "Job was already queued. Job killed by the user."); + } else { + logger.info("[{}] - Kill signal send. Waiting for job to finish.", job.getId()); + return 0; + } + } catch (Exception e) { + logger.error("[{}] - Error trying to kill the job: {}", job.getId(), e.getMessage(), e); + // Skip this job. Will be retried next loop iteration + return 0; + } + } + switch (status.getId()) { case Enums.ExecutionStatus.QUEUED: // Job is still queued @@ -444,22 +490,43 @@ protected int checkQueuedJob(Job job) { } } - protected void checkPendingJobs() { + protected void checkPendingJobs(List organizationIds) { // Clear job counts each cycle jobsCountByType.clear(); - int handledPendingJobs = 0; - try (DBIterator iterator = jobManager.iterator(pendingJobsQuery, queryOptions, token)) { - while (handledPendingJobs < NUM_JOBS_HANDLED && iterator.hasNext()) { - try { - Job job = iterator.next(); - handledPendingJobs += checkPendingJob(job); - } catch (Exception e) { - logger.error("{}", e.getMessage(), e); + // If there are no queued jobs, we can queue new jobs + List pendingJobs = new LinkedList<>(); + List runningJobs = new LinkedList<>(); + + for (String organizationId : organizationIds) { + try (DBIterator iterator = jobManager.iteratorInOrganization(organizationId, pendingJobsQuery, queryOptions, token)) { + while (iterator.hasNext()) { + pendingJobs.add(iterator.next()); } + } catch (Exception e) { + logger.error("Error listing pending jobs from organization {}", organizationId, e); + return; + } + + try (DBIterator iterator = jobManager.iteratorInOrganization(organizationId, runningJobsQuery, queryOptions, token)) { + while (iterator.hasNext()) { + runningJobs.add(iterator.next()); + } + } catch (Exception e) { + logger.error("Error listing running jobs from organization {}", organizationId, e); + return; + } + } + + Iterator iterator = jobScheduler.schedule(pendingJobs, Collections.emptyList(), runningJobs); + boolean success = false; + while (iterator.hasNext() && !success) { + Job job = iterator.next(); + try { + success = checkPendingJob(job) > 0; + } catch (Exception e) { + logger.error("{}", e.getMessage(), e); } - } catch (Exception e) { - logger.error("{}", e.getMessage(), e); } } @@ -471,27 +538,35 @@ protected void checkPendingJobs() { */ protected int checkPendingJob(Job job) { if (StringUtils.isEmpty(job.getStudy().getId())) { - return abortJob(job, "Missing mandatory 'studyUuid' field"); + return abortJob(job, "Missing mandatory 'study' field"); } + CatalogFqn catalogFqn = CatalogFqn.extractFqnFromStudyFqn(job.getStudy().getId()); + String organizationId = catalogFqn.getOrganizationId(); if (StringUtils.isEmpty(job.getTool().getId())) { return abortJob(job, "Tool id '" + job.getTool().getId() + "' not found."); } - if (!canBeQueued(job)) { + if (killSignalSent(job)) { + logger.info("[{}] - Kill signal request received for job with status='{}'. Job did not start the execution.", job.getId(), + job.getInternal().getStatus().getId()); + return abortJob(job, "Job killed by the user."); + } + + if (!canBeQueued(organizationId, job)) { return 0; } Tool tool; try { - tool = new ToolFactory().getTool(job.getTool().getId()); + tool = new ToolFactory().getTool(job.getTool().getId(), packages); } catch (Exception e) { logger.error(e.getMessage(), e); return abortJob(job, "Tool " + job.getTool().getId() + " not found", e); } try { - checkToolExecutionPermission(job); + checkToolExecutionPermission(organizationId, job); } catch (Exception e) { return abortJob(job, e); } @@ -520,7 +595,7 @@ protected int checkPendingJob(Job job) { String userToken; try { - userToken = catalogManager.getUserManager().getNonExpiringToken(job.getUserId(), Collections.emptyMap(), token); + userToken = catalogManager.getUserManager().getNonExpiringToken(organizationId, job.getUserId(), Collections.emptyMap(), token); } catch (CatalogException e) { return abortJob(job, "Internal error. Could not obtain token for user '" + job.getUserId() + "'", e); } @@ -535,7 +610,6 @@ protected int checkPendingJob(Job job) { } } - Map params = job.getParams(); String outDirPathParam = (String) params.get(OUTDIR_PARAM); if (!StringUtils.isEmpty(outDirPathParam)) { @@ -548,8 +622,8 @@ protected int checkPendingJob(Job job) { } } else { try { - // JOBS/user/job_id/ - updateParams.setOutDir(getValidDefaultOutDir(job)); + // JOBS/organizationId/user/job_id/ + updateParams.setOutDir(getValidDefaultOutDir(organizationId, job)); } catch (CatalogException e) { return abortJob(job, "Cannot create output directory.", e); } @@ -593,35 +667,52 @@ protected int checkPendingJob(Job job) { return 1; } - protected void checkToolExecutionPermission(Job job) throws Exception { - Tool tool = new ToolFactory().getTool(job.getTool().getId()); + private boolean killSignalSent(Job job) { + return job.getInternal().isKillJobRequested(); + } + + protected void checkToolExecutionPermission(String organizationId, Job job) throws Exception { + Tool tool = new ToolFactory().getTool(job.getTool().getId(), packages); + + AuthorizationManager authorizationManager = catalogManager.getAuthorizationManager(); + String user = job.getUserId(); + String userToken = catalogManager.getUserManager().getNonExpiringToken(organizationId, user, null, token); + JwtPayload jwtPayload = catalogManager.getUserManager().validateToken(userToken); + + if (storageConfiguration.getMode() == StorageConfiguration.Mode.READ_ONLY) { + // Hard check. Within READ_ONLY mode, if the tool is an operation on variant, rga or alignment, we forbid it. + // Neither opencga administrators can run it. + if (tool.type() == Tool.Type.OPERATION + && (tool.resource() == Enums.Resource.VARIANT + || tool.resource() == Enums.Resource.RGA + || tool.resource() == Enums.Resource.ALIGNMENT)) { + // Forbid storage operations! + throw new CatalogAuthorizationException("Unable to execute tool '" + tool.id() + "', " + + "which is an " + Tool.Type.OPERATION + " on resource " + tool.resource() + ". " + + "The storage engine is in mode=" + storageConfiguration.getMode()); + } + } - if (catalogManager.getAuthorizationManager().isInstallationAdministrator(job.getUserId())) { + if (authorizationManager.isOpencgaAdministrator(jwtPayload)) { // Installation administrator user can run everything return; } - if (tool.scope().equals(Tool.Scope.GLOBAL)) { - throw new CatalogAuthorizationException("Only user '" + ParamConstants.OPENCGA_USER_ID + "' " - + "can run tools with scope '" + Tool.Scope.GLOBAL + "'"); + + if (tool.scope() == Tool.Scope.GLOBAL) { + // Only installation administrators can run tools with scope global + authorizationManager.checkIsOpencgaAdministrator(jwtPayload, "run tools with scope '" + Tool.Scope.GLOBAL + "'"); + } else if (tool.scope() == Tool.Scope.ORGANIZATION) { + // Only organization owners or administrators can run tools with scope organization + authorizationManager.checkIsAtLeastOrganizationOwnerOrAdmin(organizationId, user); } else { - if (job.getStudy().getId().startsWith(job.getUserId() + ParamConstants.USER_PROJECT_SEPARATOR)) { - // If the user is the owner of the project, accept all. + if (authorizationManager.isAtLeastOrganizationOwnerOrAdmin(organizationId, user)) { + // Organization administrators can run tools with scope study or project return; } // Validate user is owner or belongs to the right group String requiredGroup; if (tool.type() == Tool.Type.OPERATION) { - if (storageConfiguration.getMode() == StorageConfiguration.Mode.READ_ONLY) { - if (tool.resource() == Enums.Resource.VARIANT - || tool.resource() == Enums.Resource.RGA - || tool.resource() == Enums.Resource.ALIGNMENT) { - // Forbid storage operations! - abortJob(job, "Unable to execute tool '" + tool.id() + "', " - + "which is an " + Tool.Type.OPERATION + " on resource " + tool.resource() + ". " - + "The storage engine is in mode=" + storageConfiguration.getMode()); - } - } requiredGroup = ParamConstants.ADMINS_GROUP; } else { requiredGroup = ParamConstants.MEMBERS_GROUP; @@ -720,26 +811,21 @@ private File getValidInternalOutDir(String study, Job job, String outDirPath, St return outDir; } - private File getValidDefaultOutDir(Job job) throws CatalogException { - File folder = fileManager.createFolder(job.getStudy().getId(), "JOBS/" + job.getUserId() + "/" + TimeUtils.getDay() + "/" - + job.getId(), true, "Job " + job.getTool().getId(), job.getId(), QueryOptions.empty(), token).first(); + private File getValidDefaultOutDir(String organizationId, Job job) throws CatalogException { + File folder = fileManager.createFolder(job.getStudy().getId(), "JOBS/" + organizationId + "/" + job.getUserId() + "/" + + TimeUtils.getDay() + "/" + job.getId(), true, "Job " + job.getTool().getId(), job.getId(), QueryOptions.empty(), token) + .first(); // By default, OpenCGA will not create the physical folders until there is a file, so we need to create it manually try { catalogManager.getIoManagerFactory().get(folder.getUri()).createDirectory(folder.getUri(), true); } catch (CatalogIOException | IOException e) { - // Submit job to delete job folder - ObjectMap params = new ObjectMap() - .append("files", folder.getUuid()) - .append("study", job.getStudy().getId()) - .append(Constants.SKIP_TRASH, true); - jobManager.submit(job.getStudy().getId(), FileDeleteTask.ID, Enums.Priority.LOW, params, token); throw new CatalogException("Cannot create job directory '" + folder.getUri() + "' for path '" + folder.getPath() + "'"); } // Check if the user already has permissions set in his folder OpenCGAResult> result = fileManager.getAcls(job.getStudy().getId(), - Collections.singletonList("JOBS/" + job.getUserId() + "/"), job.getUserId(), true, token); + Collections.singletonList("JOBS/" + organizationId + "/" + job.getUserId() + "/"), job.getUserId(), true, token); if (result.getNumResults() == 0 || result.first().getAcl().isEmpty() || CollectionUtils.isEmpty(result.first().getAcl().get(0).getPermissions())) { // Add permissions to do anything under that path to the user launching the job @@ -747,10 +833,10 @@ private File getValidDefaultOutDir(Job job) throws CatalogException { .stream() .map(FilePermissions::toString) .collect(Collectors.joining(",")); - fileManager.updateAcl(job.getStudy().getId(), Collections.singletonList("JOBS/" + job.getUserId() + "/"), job.getUserId(), - new FileAclParams(null, allFilePermissions), SET, token); + fileManager.updateAcl(job.getStudy().getId(), Collections.singletonList("JOBS/" + organizationId + "/" + job.getUserId() + "/"), + job.getUserId(), new FileAclParams(null, allFilePermissions), SET, token); // Remove permissions to the @members group - fileManager.updateAcl(job.getStudy().getId(), Collections.singletonList("JOBS/" + job.getUserId() + "/"), + fileManager.updateAcl(job.getStudy().getId(), Collections.singletonList("JOBS/" + organizationId + "/" + job.getUserId() + "/"), StudyManager.MEMBERS, new FileAclParams(null, ""), SET, token); } @@ -760,13 +846,14 @@ private File getValidDefaultOutDir(Job job) throws CatalogException { public static String buildCli(String internalCli, Job job) { String toolId = job.getTool().getId(); String internalCommand = TOOL_CLI_MAP.get(toolId); - if (StringUtils.isEmpty(internalCommand)) { + if (StringUtils.isEmpty(internalCommand) || job.isDryRun()) { ObjectMap params = new ObjectMap() .append(JOB_PARAM, job.getId()) .append(STUDY_PARAM, job.getStudy().getId()); return buildCli(internalCli, "tools execute-job", params); } else { - return buildCli(internalCli, internalCommand, job.getParams()); + ObjectMap params = new ObjectMap(job.getParams()); + return buildCli(internalCli, internalCommand, params); } } @@ -830,7 +917,7 @@ public static void escapeCliArg(StringBuilder cliBuilder, String value) { } } - private boolean canBeQueued(Job job) { + private boolean canBeQueued(String organizationId, Job job) { if (job.getDependsOn() != null && !job.getDependsOn().isEmpty()) { for (Job tmpJob : job.getDependsOn()) { if (!Enums.ExecutionStatus.DONE.equals(tmpJob.getInternal().getStatus().getId())) { @@ -847,23 +934,31 @@ private boolean canBeQueued(Job job) { return false; } + if (StringUtils.isNotEmpty(job.getScheduledStartTime())) { + Date date = TimeUtils.toDate(job.getScheduledStartTime()); + if (date.after(new Date())) { + logger.debug("Job '{}' can't be queued yet. It is scheduled to start at '{}'.", job.getId(), job.getScheduledStartTime()); + return false; + } + } + Integer maxJobs = catalogManager.getConfiguration().getAnalysis().getExecution().getMaxConcurrentJobs().get(job.getTool().getId()); if (maxJobs == null) { // No limit for this tool return true; } else { - return canBeQueued(job.getTool().getId(), maxJobs); + return canBeQueued(organizationId, job.getTool().getId(), maxJobs); } } - private boolean canBeQueued(String toolId, int maxJobs) { + private boolean canBeQueued(String organizationId, String toolId, int maxJobs) { Query query = new Query() .append(JobDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), Enums.ExecutionStatus.QUEUED + "," + Enums.ExecutionStatus.RUNNING) .append(JobDBAdaptor.QueryParams.TOOL_ID.key(), toolId); long currentJobs = jobsCountByType.computeIfAbsent(toolId, k -> { try { - return catalogManager.getJobManager().count(query, token).getNumMatches(); + return catalogManager.getJobManager().countInOrganization(organizationId, query, token).getNumMatches(); } catch (CatalogException e) { logger.error("Error counting the current number of running and queued \"" + toolId + "\" jobs", e); return 0L; @@ -899,10 +994,15 @@ private int abortJob(Job job, String message, Exception e) { } private int abortJob(Job job, String description) { - logger.info("Aborting job: {} - Reason: '{}'", job.getId(), description); + logger.info("[{}] - Aborting job - Reason: '{}'", job.getId(), description); return setStatus(job, new Enums.ExecutionStatus(Enums.ExecutionStatus.ABORTED, description)); } + private int abortKillJob(Job job, String description) { + logger.info("[{}] -Aborting job - Reason: '{}'", job.getId(), description); + return processFinishedJob(job, new Enums.ExecutionStatus(Enums.ExecutionStatus.ABORTED, description)); + } + private int setStatus(Job job, Enums.ExecutionStatus status) { PrivateJobUpdateParams updateParams = new PrivateJobUpdateParams().setInternal(new JobInternal(status)); @@ -962,7 +1062,9 @@ private int processFinishedJob(Job job, Enums.ExecutionStatus status) { logger.info("[{}] - Registering job results from '{}'", job.getId(), Paths.get(job.getOutDir().getUri())); ExecutionResult execution = readExecutionResult(job); - if (execution != null) { + if (execution == null) { + logger.warn("[{}] - Execution result not found", job.getId()); + } else { if (execution.getEnd() == null) { // This could happen if the job finished abruptly logger.info("[{}] Missing end date at ExecutionResult", job.getId()); @@ -1031,29 +1133,35 @@ private int processFinishedJob(Job job, Enums.ExecutionStatus status) { updateParams.setInternal(new JobInternal()); // Check status of analysis result or if there are files that could not be moved to outdir to decide the final result - if (execution == null) { - updateParams.getInternal().setStatus(new Enums.ExecutionStatus(Enums.ExecutionStatus.ERROR, - "Job could not finish successfully. Missing execution result")); - } else if (execution.getStatus().getName().equals(Status.Type.ERROR)) { - updateParams.getInternal().setStatus(new Enums.ExecutionStatus(Enums.ExecutionStatus.ERROR, - "Job could not finish successfully")); - } else { - switch (status.getId()) { - case Enums.ExecutionStatus.DONE: - case Enums.ExecutionStatus.READY: - updateParams.getInternal().setStatus(new Enums.ExecutionStatus(Enums.ExecutionStatus.DONE)); - break; - case Enums.ExecutionStatus.ABORTED: - updateParams.getInternal().setStatus(new Enums.ExecutionStatus(Enums.ExecutionStatus.ERROR, "Job aborted!")); - break; - case Enums.ExecutionStatus.ERROR: - default: + switch (status.getId()) { + case Enums.ExecutionStatus.DONE: + case Enums.ExecutionStatus.READY: + if (execution == null) { + // Regardless of the status, without execution result, we will set the status to ERROR updateParams.getInternal().setStatus(new Enums.ExecutionStatus(Enums.ExecutionStatus.ERROR, - "Job could not finish successfully")); - break; - } + "Job could not finish successfully. Missing execution result")); + } else if (execution.getStatus().getName() == Status.Type.ERROR) { + // Discrepancy between the status in the execution result and the status of the job + status.setDescription("Job could not finish successfully." + + " Execution result status: " + execution.getStatus().getName()); + updateParams.getInternal().setStatus(status); + } else { + updateParams.getInternal().setStatus(new Enums.ExecutionStatus(Enums.ExecutionStatus.DONE)); + } + break; + case Enums.ExecutionStatus.ABORTED: + updateParams.getInternal().setStatus(status); + break; + case Enums.ExecutionStatus.ERROR: + default: + if (StringUtils.isEmpty(status.getDescription())) { + status.setDescription("Job could not finish successfully"); + } + updateParams.getInternal().setStatus(status); + break; } + logger.info("[{}] - Updating job information", job.getId()); // We update the job information try { @@ -1108,7 +1216,7 @@ private void notifyStatusChange(Job job) { } private void sendWebhookNotification(Job job, URL url) throws URISyntaxException, CatalogException, CloneNotSupportedException { - JobInternal jobInternal = new JobInternal(null, null, null, job.getInternal().getWebhook().clone(), null); + JobInternal jobInternal = new JobInternal(null, null, null, job.getInternal().getWebhook().clone(), null, false); PrivateJobUpdateParams updateParams = new PrivateJobUpdateParams() .setInternal(jobInternal); diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/MonitorParentDaemon.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/MonitorParentDaemon.java index 247b7a81534..393cb2afdd6 100644 --- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/MonitorParentDaemon.java +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/MonitorParentDaemon.java @@ -16,41 +16,34 @@ package org.opencb.opencga.master.monitor.daemons; -import org.opencb.opencga.catalog.db.DBAdaptorFactory; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.managers.CatalogManager; import org.opencb.opencga.master.monitor.executors.BatchExecutor; import org.opencb.opencga.master.monitor.executors.ExecutorFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.nio.file.Path; +import java.io.Closeable; +import java.io.IOException; /** * Created by imedina on 16/06/16. */ -public abstract class MonitorParentDaemon implements Runnable { +public abstract class MonitorParentDaemon implements Runnable, Closeable { - protected int interval; - protected CatalogManager catalogManager; - // FIXME: This should not be used directly! All the queries MUST go through the CatalogManager - @Deprecated - protected DBAdaptorFactory dbAdaptorFactory; + protected final int interval; + protected final CatalogManager catalogManager; protected BatchExecutor batchExecutor; - protected boolean exit = false; + private volatile boolean exit = false; - protected String token; + protected final String token; + protected final Logger logger; - protected Logger logger; - - public MonitorParentDaemon(int interval, String token, CatalogManager catalogManager) throws CatalogDBException { + public MonitorParentDaemon(int interval, String token, CatalogManager catalogManager) { this.interval = interval; this.catalogManager = catalogManager; this.token = token; logger = LoggerFactory.getLogger(this.getClass()); - dbAdaptorFactory = new MongoDBAdaptorFactory(catalogManager.getConfiguration(), catalogManager.getIoManagerFactory()); ExecutorFactory executorFactory = new ExecutorFactory(catalogManager.getConfiguration()); this.batchExecutor = executorFactory.getExecutor(); } @@ -63,12 +56,42 @@ public void setExit(boolean exit) { this.exit = exit; } - static Path getJobTemporaryFolder(long jobId, Path tempJobFolder) { - return tempJobFolder.resolve(getJobTemporaryFolderName(jobId)); + public void run() { + try { + init(); + } catch (Exception e) { + logger.error("Error initializing daemon", e); + throw new RuntimeException(e); + } + + while (!exit) { + try { + Thread.sleep(interval); + } catch (InterruptedException e) { + if (!exit) { + logger.warn("Interrupted while sleeping", e); + } + // If interrupted, stop the daemon + break; + } + + try { + apply(); + } catch (Exception e) { + logger.error("Catch exception " + e.getMessage(), e); + } + } + + try { + close(); + } catch (IOException e) { + logger.error("Error closing daemon", e); + } } - static String getJobTemporaryFolderName(long jobId) { - return "J_" + jobId; + public void init() throws Exception { + } + public abstract void apply() throws Exception; } diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/AzureBatchExecutor.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/AzureBatchExecutor.java index 05c6a10be19..28900481a81 100644 --- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/AzureBatchExecutor.java +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/AzureBatchExecutor.java @@ -23,6 +23,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.nio.file.Path; /** @@ -58,7 +59,11 @@ public AzureBatchExecutor(Execution execution) { this.poolInformation = new PoolInformation().withPoolId(batchPoolId); } -// public void submitAzureTask(Job job, String token) throws IOException { + @Override + public void close() throws IOException { + } + + // public void submitAzureTask(Job job, String token) throws IOException { // String jobId = getOrCreateAzureJob(job.getType()); // TaskAddParameter taskToAdd = new TaskAddParameter(); // taskToAdd.withId(job.getId()).withCommandLine(job.getCommandLine()).withContainerSettings( diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/BatchExecutor.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/BatchExecutor.java index 58a811f1538..05c38e18545 100644 --- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/BatchExecutor.java +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/BatchExecutor.java @@ -16,12 +16,13 @@ package org.opencb.opencga.master.monitor.executors; +import java.io.Closeable; import java.nio.file.Path; /** * Created by pfurio on 22/08/16. */ -public interface BatchExecutor { +public interface BatchExecutor extends Closeable { String TIMEOUT = "timeout"; String STDOUT = "stdout"; @@ -29,9 +30,7 @@ public interface BatchExecutor { String OUTDIR = "outdir"; String NUM_THREADS = "num_threads"; String MAX_MEM = "max_mem"; - @Deprecated - String JOB_STATUS_FILE = "status.json"; - String OUT_LOG_EXTENSION = ".out"; + String OUT_LOG_EXTENSION = ".log"; String ERR_LOG_EXTENSION = ".err"; void execute(String jobId, String queue, String commandLine, Path stdout, Path stderr) throws Exception; diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/K8SExecutor.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/K8SExecutor.java index 4b55a6acd69..b96dcf6f7e4 100644 --- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/K8SExecutor.java +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/K8SExecutor.java @@ -18,9 +18,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.fabric8.kubernetes.api.model.*; -import io.fabric8.kubernetes.api.model.batch.Job; -import io.fabric8.kubernetes.api.model.batch.JobBuilder; -import io.fabric8.kubernetes.api.model.batch.JobSpecBuilder; +import io.fabric8.kubernetes.api.model.batch.v1.Job; +import io.fabric8.kubernetes.api.model.batch.v1.JobBuilder; +import io.fabric8.kubernetes.api.model.batch.v1.JobSpecBuilder; import io.fabric8.kubernetes.client.Config; import io.fabric8.kubernetes.client.ConfigBuilder; import io.fabric8.kubernetes.client.*; @@ -36,6 +36,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.nio.file.Path; import java.time.Instant; import java.time.temporal.ChronoUnit; @@ -51,6 +52,8 @@ public class K8SExecutor implements BatchExecutor { public static final String K8S_IMAGE_PULL_POLICY = "k8s.imagePullPolicy"; public static final String K8S_IMAGE_PULL_SECRETS = "k8s.imagePullSecrets"; public static final String K8S_TTL_SECONDS_AFTER_FINISHED = "k8s.ttlSecondsAfterFinished"; + public static final String K8S_TERMINATION_GRACE_PERIOD_SECONDS = "k8s.terminationGracePeriodSeconds"; + public static final String K8S_LOG_TO_STDOUT = "k8s.logToStdout"; public static final String K8S_DIND_ROOTLESS = "k8s.dind.rootless"; public static final String K8S_DIND_IMAGE_NAME = "k8s.dind.imageName"; public static final String K8S_REQUESTS = "k8s.requests"; @@ -87,6 +90,7 @@ public class K8SExecutor implements BatchExecutor { .withMountPath("/usr/share/pod") .build(); private static final String DIND_DONE_FILE = "/usr/share/pod/done"; + public static final String JOB_NAME = "job-name"; private final String namespace; private final String imageName; @@ -109,6 +113,8 @@ public class K8SExecutor implements BatchExecutor { private final String imagePullPolicy; private final List imagePullSecrets; private final int ttlSecondsAfterFinished; + private final boolean logToStdout; + private long terminationGracePeriodSeconds; public K8SExecutor(Configuration configuration) { Execution execution = configuration.getAnalysis().getExecution(); @@ -129,6 +135,8 @@ public K8SExecutor(Configuration configuration) { this.imagePullPolicy = execution.getOptions().getString(K8S_IMAGE_PULL_POLICY, "IfNotPresent"); this.imagePullSecrets = buildLocalObjectReference(execution.getOptions().get(K8S_IMAGE_PULL_SECRETS)); this.ttlSecondsAfterFinished = execution.getOptions().getInt(K8S_TTL_SECONDS_AFTER_FINISHED, 3600); + this.terminationGracePeriodSeconds = execution.getOptions().getInt(K8S_TERMINATION_GRACE_PERIOD_SECONDS, 5 * 60); + this.logToStdout = execution.getOptions().getBoolean(K8S_LOG_TO_STDOUT, true); nodeSelector = getMap(execution.getOptions(), K8S_NODE_SELECTOR); if (execution.getOptions().containsKey(K8S_SECURITY_CONTEXT)) { securityContext = buildObject(execution.getOptions().get(K8S_SECURITY_CONTEXT), SecurityContext.class); @@ -216,7 +224,12 @@ public K8SExecutor(Configuration configuration) { .withEnv(new EnvVar("DOCKER_TLS_CERTDIR", "", null)) // .withResources(resources) // TODO: Should we add resources here? .withCommand("/bin/sh", "-c") - .addToArgs("dockerd-entrypoint.sh & while ! test -f " + DIND_DONE_FILE + "; do sleep 5; done; exit 0") + .addToArgs("dockerd-entrypoint.sh & " + // Add trap to capture TERM signal and finish main process + + "trap '" + + "echo \"Container terminated! ;\n" + + "touch " + DIND_DONE_FILE + " ' TERM ;" + + "while ! test -f " + DIND_DONE_FILE + "; do sleep 5; done; exit 0") .addToVolumeMounts(DOCKER_GRAPH_VOLUMEMOUNT) .addToVolumeMounts(TMP_POD_VOLUMEMOUNT) .addAllToVolumeMounts(volumeMounts) @@ -236,10 +249,9 @@ public void eventReceived(Action action, Job k8Job) { } @Override - public void onClose(KubernetesClientException e) { + public void onClose(WatcherException e) { if (e != null) { logger.error("Catch exception at jobs watcher", e); - throw e; } } }); @@ -261,19 +273,25 @@ public void eventReceived(Action action, Pod pod) { } @Override - public void onClose(KubernetesClientException e) { + public void onClose(WatcherException e) { if (e != null) { logger.error("Catch exception at pods watcher", e); - throw e; } } }); } + @Override + public void close() throws IOException { + podsWatcher.close(); + jobsWatcher.close(); + kubernetesClient.close(); + } + @Override public void execute(String jobId, String queue, String commandLine, Path stdout, Path stderr) throws Exception { String jobName = buildJobName(jobId); - final io.fabric8.kubernetes.api.model.batch.Job k8sJob = new JobBuilder() + final io.fabric8.kubernetes.api.model.batch.v1.Job k8sJob = new JobBuilder() .withApiVersion("batch/v1") .withKind("Job") .withMetadata(new ObjectMetaBuilder() @@ -290,6 +308,7 @@ public void execute(String jobId, String queue, String commandLine, Path stdout, .addToAnnotations("cluster-autoscaler.kubernetes.io/safe-to-evict", "false") .build()) .withSpec(new PodSpecBuilder() + .withTerminationGracePeriodSeconds(terminationGracePeriodSeconds) .withImagePullSecrets(imagePullSecrets) .addToContainers(new ContainerBuilder() .withName("opencga") @@ -297,9 +316,8 @@ public void execute(String jobId, String queue, String commandLine, Path stdout, .withImagePullPolicy(imagePullPolicy) .withResources(resources) .addAllToEnv(envVars) - .withCommand("/bin/sh", "-c") - .withArgs("trap 'touch " + DIND_DONE_FILE + "' EXIT ; " - + getCommandLine(commandLine, stdout, stderr)) + .withCommand("/bin/bash", "-c") + .withArgs(getCommandLine(commandLine, stdout, stderr)) .withVolumeMounts(volumeMounts) .addToVolumeMounts(TMP_POD_VOLUMEMOUNT) .withSecurityContext(securityContext) @@ -330,19 +348,20 @@ private boolean shouldAddDockerDaemon(String queue) { /** * Build a valid K8S job name. - * + *

* DNS-1123 label must consist of lower case alphanumeric characters or '-', and must start and * end with an alphanumeric character (e.g. 'my-name', or '123-abc', regex used for validation * is '[a-z0-9]([-a-z0-9]*[a-z0-9])?' - * + *

* Max length = 63 - * + *

* DNS-1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must * start and end with an alphanumeric character (e.g. 'example.com', regex used for validation * is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*') + * * @param jobIdInput job Is - * @link https://github.com/kubernetes/kubernetes/blob/c560907/staging/src/k8s.io/apimachinery/pkg/util/validation/validation.go#L135 * @return valid name + * @link https://github.com/kubernetes/kubernetes/blob/c560907/staging/src/k8s.io/apimachinery/pkg/util/validation/validation.go#L135 */ protected static String buildJobName(String jobIdInput) { String jobId = jobIdInput.replace("_", "-"); @@ -404,7 +423,53 @@ public boolean resume(String jobId) throws Exception { @Override public boolean kill(String jobId) throws Exception { - return false; + String k8sJobName = buildJobName(jobId); + + switch (getStatus(jobId)) { + case Enums.ExecutionStatus.DONE: + case Enums.ExecutionStatus.ERROR: + case Enums.ExecutionStatus.ABORTED: { + return jobPodExists(k8sJobName); + } + case Enums.ExecutionStatus.UNKNOWN: { + logger.warn("K8s Job '" + k8sJobName + "' not found!"); + return jobPodExists(k8sJobName); + } + case Enums.ExecutionStatus.QUEUED: + case Enums.ExecutionStatus.PENDING: + case Enums.ExecutionStatus.RUNNING: { + deleteJobIfAny(k8sJobName); + return jobPodExists(k8sJobName); + } + default: + return false; + } + } + + private void deleteJobIfAny(String k8sJobName) { + Job k8Job = getKubernetesClient() + .batch() + .v1() + .jobs() + .inNamespace(namespace) + .withName(k8sJobName) + .get(); + + if (k8Job != null) { + logger.info("Deleting kubernetes job '" + k8Job.getMetadata().getName() + "'"); + getKubernetesClient() + .batch() + .v1() + .jobs() + .inNamespace(namespace) + .withName(k8sJobName) + .withGracePeriod(terminationGracePeriodSeconds) + .delete(); + } + } + + private boolean jobPodExists(String k8sJobName) { + return getJobPod(k8sJobName) == null; } @Override @@ -412,6 +477,58 @@ public boolean isExecutorAlive() { return false; } + /** + * We do it this way to avoid writing the session id in the command line (avoid display/monitor/logs) attribute of Job. + * + * @param commandLine Basic command line + * @param stdout File where the standard output will be redirected + * @param stderr File where the standard error will be redirected + * @return The complete command line + */ + @Override + public String getCommandLine(String commandLine, Path stdout, Path stderr) { + // Do "exec" the main command to keep the PID 1 on the main process to allow grace kills. + commandLine = "exec " + commandLine; + // https://stackoverflow.com/questions/692000/how-do-i-write-standard-error-to-a-file-while-using-tee-with-a-pipe + if (stderr != null) { + if (logToStdout) { + commandLine = commandLine + " 2> >( tee -a \"" + stderr.toAbsolutePath() + "\" >&2 )"; + } else { + commandLine = commandLine + " 2>> " + stderr.toAbsolutePath(); + } + } + if (stdout != null) { + if (logToStdout) { + commandLine = commandLine + " > >( tee -a \"" + stdout.toAbsolutePath() + "\")"; + } else { + commandLine = commandLine + " >> " + stdout.toAbsolutePath(); + } + } + + // Add trap to capture TERM signal and kill the main process + String trapTerm = "trap '" + + "echo \"Job terminated! Run time : ${SECONDS}s\" ;\n" + + "touch INTERRUPTED ;\n" + + "if [ -s PID ] && ps -p $(cat PID) > /dev/null; then\n" + + " kill -15 $(cat PID) ;\n" + + "fi' TERM ;"; + + // Launch the main process in background. + String mainProcess = commandLine + " &"; + + String wait = "PID=$! ; \n" + + "echo $PID > PID ; \n" + + "while ps -p \"$PID\" >/dev/null; do \n" + + " sleep 1 ; \n" + + "done \n" + + "touch '" + DIND_DONE_FILE + "' \n" + + "if [ -f INTERRUPTED ]; then\n" + + " exit 1\n" + + "fi"; + + return trapTerm + " " + mainProcess + " " + wait; + } + private String getStatusForce(String k8sJobName) { Job k8Job = getKubernetesClient() .batch() @@ -421,7 +538,7 @@ private String getStatusForce(String k8sJobName) { .get(); if (k8Job == null) { - logger.warn("Job " + k8sJobName + " not found!"); + logger.warn("Job '" + k8sJobName + "' not found!"); // Job has been deleted. manually? return Enums.ExecutionStatus.ABORTED; } @@ -430,15 +547,23 @@ private String getStatusForce(String k8sJobName) { if (statusFromK8sJob.equalsIgnoreCase(Enums.ExecutionStatus.UNKNOWN) || statusFromK8sJob.equalsIgnoreCase(Enums.ExecutionStatus.QUEUED)) { logger.warn("Job status " + statusFromK8sJob + " . Fetch POD info"); - List pods = getKubernetesClient().pods().withLabel("job-name", k8sJobName).list(1, null).getItems(); - if (!pods.isEmpty()) { - Pod pod = pods.get(0); + Pod pod = getJobPod(k8sJobName); + if (pod != null) { return getStatusFromPod(pod); } } return statusFromK8sJob; } + private Pod getJobPod(String k8sJobName) { + List pods = getKubernetesClient().pods().withLabel(JOB_NAME, k8sJobName).list(1, null).getItems(); + if (pods.isEmpty()) { + return null; + } else { + return pods.get(0); + } + } + private String getJobName(Pod pod) { if (pod.getMetadata() == null || pod.getMetadata().getLabels() == null @@ -446,7 +571,7 @@ private String getJobName(Pod pod) { || pod.getStatus().getPhase() == null) { return null; } - return pod.getMetadata().getLabels().get("job-name"); + return pod.getMetadata().getLabels().get(JOB_NAME); } private String getStatusFromPod(Pod pod) { @@ -486,7 +611,7 @@ private String getStatusFromK8sJob(Job k8Job, String k8sJobName) { } else if (k8Job.getStatus().getActive() != null && k8Job.getStatus().getActive() > 0) { // status = Enums.ExecutionStatus.RUNNING; status = Enums.ExecutionStatus.QUEUED; - } else { + } else { status = Enums.ExecutionStatus.UNKNOWN; } logger.debug("k8Job '{}}' status = '{}'", k8sJobName, status); @@ -514,6 +639,7 @@ private List buildObjects(List list, Class clazz) { } return ts; } + private T buildObject(Object o, Class clazz) { if (o == null) { return null; diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/LocalExecutor.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/LocalExecutor.java index 5fd46ba866e..901dd7cb3c1 100644 --- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/LocalExecutor.java +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/LocalExecutor.java @@ -44,6 +44,7 @@ public class LocalExecutor implements BatchExecutor { private static Logger logger; private final ExecutorService threadPool; private final Map jobStatus; + private final Map runningJobs; private final int maxConcurrentJobs; public LocalExecutor(Execution execution) { @@ -56,6 +57,17 @@ protected boolean removeEldestEntry(Map.Entry eldest) { return size() > 1000; } }); + runningJobs = Collections.synchronizedMap(new LinkedHashMap(1000) { + @Override + protected boolean removeEldestEntry(Map.Entry eldest) { + return size() > 1000; + } + }); + } + + @Override + public void close() throws IOException { + threadPool.shutdown(); } @Override @@ -65,8 +77,9 @@ public void execute(String jobId, String queue, String commandLine, Path stdout, try { Thread.currentThread().setName("LocalExecutor-" + nextThreadNum()); logger.info("Ready to run - {}", commandLine); - jobStatus.put(jobId, Enums.ExecutionStatus.RUNNING); Command com = new Command(commandLine); + runningJobs.put(jobId, com); + jobStatus.put(jobId, Enums.ExecutionStatus.RUNNING); DataOutputStream dataOutputStream = new DataOutputStream(new FileOutputStream(stdout.toFile())); com.setOutputOutputStream(dataOutputStream); @@ -78,6 +91,7 @@ public void execute(String jobId, String queue, String commandLine, Path stdout, logger.info("Running ShutdownHook. Job {id: " + jobId + "} has being aborted."); com.setStatus(RunnableProcess.Status.KILLED); com.setExitValue(-2); + runningJobs.remove(jobId); closeOutputStreams(com); jobStatus.put(jobId, Enums.ExecutionStatus.ERROR); }); @@ -90,9 +104,11 @@ public void execute(String jobId, String queue, String commandLine, Path stdout, try { Runtime.getRuntime().addShutdownHook(hook); + runningJobs.put(jobId, com); com.run(); } finally { Runtime.getRuntime().removeShutdownHook(hook); + runningJobs.remove(jobId); closeOutputStreams(com); } @@ -135,7 +151,13 @@ public boolean resume(String jobId) throws Exception { @Override public boolean kill(String jobId) throws Exception { - return false; + Command command = runningJobs.get(jobId); + if (command != null) { + command.destroy(); + return true; + } else { + return false; + } } @Override diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/SGEExecutor.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/SGEExecutor.java index 1d46dd5b0a0..0d0734deaca 100644 --- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/SGEExecutor.java +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/SGEExecutor.java @@ -21,6 +21,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.nio.file.Path; /** @@ -36,6 +37,10 @@ public SGEExecutor(Execution execution) { sgeManager = new SGEManager(execution); } + @Override + public void close() throws IOException { + } + @Override public void execute(String jobId, String queue, String commandLine, Path stdout, Path stderr) throws Exception { sgeManager.queueJob(jobId, "", -1, getCommandLine(commandLine, stdout, stderr), null); diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/schedulers/JobScheduler.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/schedulers/JobScheduler.java new file mode 100644 index 00000000000..e17fc25e0ab --- /dev/null +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/schedulers/JobScheduler.java @@ -0,0 +1,232 @@ +package org.opencb.opencga.master.monitor.schedulers; + +import org.apache.commons.lang3.time.StopWatch; +import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; +import org.opencb.opencga.catalog.db.api.UserDBAdaptor; +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.managers.CatalogManager; +import org.opencb.opencga.catalog.managers.OrganizationManager; +import org.opencb.opencga.catalog.utils.CatalogFqn; +import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.models.job.Job; +import org.opencb.opencga.core.models.organizations.Organization; +import org.opencb.opencga.core.models.study.Group; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; + +public class JobScheduler { + + private final CatalogManager catalogManager; + private final String token; + + private Map userRoles; + + private static final float PRIORITY_WEIGHT = 0.6F; + private static final float IDLE_TIME_WEIGHT = 0.4F; +// private static final float OPERATION_WEIGHT = 0.2F; +// private static final float USER_PRIORITY_WEIGHT = 0.2F; + + private final Logger logger = LoggerFactory.getLogger(JobScheduler.class); + + + public JobScheduler(CatalogManager catalogManager, String token) { + this.catalogManager = catalogManager; + this.token = token; + } + + private void getUserRoles() throws CatalogException { + StopWatch stopWatch = StopWatch.createStarted(); + this.userRoles = new HashMap<>(); + + List organizationIds = catalogManager.getOrganizationManager().getOrganizationIds(token); + for (String organizationId : organizationIds) { + if (ParamConstants.ADMIN_ORGANIZATION.equals(organizationId)) { + QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, UserDBAdaptor.QueryParams.ID.key()); + catalogManager.getUserManager().search(organizationId, new Query(), options, token).getResults() + .forEach(user -> getUserRole(organizationId, user.getId()).setSuperAdmin(true)); + } else { + Organization organization = catalogManager.getOrganizationManager().get(organizationId, + OrganizationManager.INCLUDE_ORGANIZATION_ADMINS, token).first(); + getUserRole(organizationId, organization.getOwner()).addOrganizationOwner(organizationId); + organization.getAdmins().forEach(user -> getUserRole(organizationId, user).addOrganizationAdmin(organizationId)); + + QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(StudyDBAdaptor.QueryParams.FQN.key(), + StudyDBAdaptor.QueryParams.GROUPS.key())); + catalogManager.getStudyManager().searchInOrganization(organizationId, new Query(), options, token).getResults() + .forEach(study -> { + for (Group group : study.getGroups()) { + if (ParamConstants.ADMINS_GROUP.equals(group.getId())) { + group.getUserIds().forEach(user -> getUserRole(organizationId, user).addStudyAdmin(study.getFqn())); + } + } + }); + } + } + + logger.debug("Time spent fetching user roles: {}", TimeUtils.durationToString(stopWatch)); + } + + private UserRole getUserRole(String organizationId, String userId) { + String id = organizationId + "@" + userId; + if (!this.userRoles.containsKey(id)) { + this.userRoles.put(id, new UserRole()); + } + return this.userRoles.get(id); + } + + private float getPriorityWeight(Job job) { + float appPriority; + switch (job.getPriority()) { + case URGENT: + appPriority = 1f; + break; + case HIGH: + appPriority = 0.8f; + break; + case MEDIUM: + appPriority = 0.5f; + break; + case LOW: + case UNKNOWN: + default: + appPriority = 0.2f; + break; + } + + // Increase priority depending on the time the job has been waiting in PENDING status + Calendar todayCalendar = Calendar.getInstance(); + Date date = TimeUtils.toDate(job.getInternal().getRegistrationDate()); + Calendar calendar = Calendar.getInstance(); + calendar.setTime(date); + while (appPriority < 1f) { + calendar.add(Calendar.HOUR, 24); + if (calendar.before(todayCalendar)) { + appPriority = Math.max(1f, appPriority + 0.05f); + } else { + break; + } + } + + // User's priority + float usersPriority; + switch (job.getPriority()) { + case URGENT: + usersPriority = 1f; + break; + case HIGH: + usersPriority = 0.8f; + break; + case MEDIUM: + usersPriority = 0.5f; + break; + case LOW: + case UNKNOWN: + default: + usersPriority = 0.2f; + break; + } + + // Adjust user's priority depending on the user's role + String userId = job.getUserId(); + String organizationId = CatalogFqn.extractFqnFromStudyFqn(job.getStudy().getId()).getOrganizationId(); + + UserRole userRole = getUserRole(organizationId, userId); + if (userRole.isSuperAdmin) { + usersPriority = usersPriority * 1; + } else if (userRole.isOrganizationOwner(organizationId)) { + usersPriority = usersPriority * 0.8f; + } else if (userRole.isOrganizationAdmin(organizationId)) { + usersPriority = usersPriority * 0.75f; + } else if (userRole.isStudyAdmin(job.getStudy().getId())) { + usersPriority = usersPriority * 0.6f; + } else { + usersPriority = usersPriority * 0.4f; + } + + return appPriority * 0.6f + usersPriority * 0.4f; + } + + public Iterator schedule(List pendingJobs, List queuedJobs, List runningJobs) { + TreeMap> jobTreeMap = new TreeMap<>(); + + try { + getUserRoles(); + } catch (CatalogException e) { + throw new RuntimeException("Scheduler exception: " + e.getMessage(), e); + } + + StopWatch stopWatch = StopWatch.createStarted(); + for (Job job : pendingJobs) { + float priority = getPriorityWeight(job); + if (!jobTreeMap.containsKey(priority)) { + jobTreeMap.put(priority, new ArrayList<>()); + } + jobTreeMap.get(priority).add(job); + } + logger.debug("Time spent scheduling jobs: {}", TimeUtils.durationToString(stopWatch)); + + stopWatch.reset(); + stopWatch.start(); + // Obtain iterator + List allJobs = new ArrayList<>(); + for (Float priority : jobTreeMap.descendingKeySet()) { + allJobs.addAll(jobTreeMap.get(priority)); + } + logger.debug("Time spent creating iterator: {}", TimeUtils.durationToString(stopWatch)); + + return allJobs.iterator(); + } + + private static class UserRole { + + private boolean isSuperAdmin; + private Set organizationOwners; + private Set organizationAdmins; + private Set studyAdmins; + + UserRole() { + this.organizationOwners = new HashSet<>(); + this.organizationAdmins = new HashSet<>(); + this.studyAdmins = new HashSet<>(); + } + + public boolean isSuperAdmin() { + return isSuperAdmin; + } + + public UserRole setSuperAdmin(boolean superAdmin) { + isSuperAdmin = superAdmin; + return this; + } + + public void addOrganizationOwner(String userId) { + organizationOwners.add(userId); + } + + public void addOrganizationAdmin(String userId) { + organizationAdmins.add(userId); + } + + public void addStudyAdmin(String userId) { + studyAdmins.add(userId); + } + + public boolean isOrganizationOwner(String organizationId) { + return organizationOwners.contains(organizationId); + } + + public boolean isOrganizationAdmin(String organizationId) { + return organizationAdmins.contains(organizationId); + } + + public boolean isStudyAdmin(String studyFqn) { + return studyAdmins.contains(studyFqn); + } + } + +} diff --git a/opencga-master/src/test/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemonTest.java b/opencga-master/src/test/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemonTest.java index 2251f2b5b09..9d2de1bdb42 100644 --- a/opencga-master/src/test/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemonTest.java +++ b/opencga-master/src/test/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemonTest.java @@ -24,6 +24,7 @@ import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.TestParamConstants; +import org.opencb.opencga.analysis.tools.ToolRunner; import org.opencb.opencga.analysis.variant.operations.VariantAnnotationIndexOperationTool; import org.opencb.opencga.analysis.variant.operations.VariantIndexOperationTool; import org.opencb.opencga.catalog.db.api.FileDBAdaptor; @@ -33,6 +34,7 @@ import org.opencb.opencga.catalog.managers.FileManager; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.storage.StorageConfiguration; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.AclParams; @@ -52,6 +54,7 @@ import org.opencb.opencga.core.tools.result.ExecutionResultManager; import org.opencb.opencga.master.monitor.executors.BatchExecutor; import org.opencb.opencga.master.monitor.models.PrivateJobUpdateParams; +import org.opencb.opencga.storage.core.StorageEngineFactory; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -74,19 +77,23 @@ public class ExecutionDaemonTest extends AbstractManagerTest { private ExecutionDaemon daemon; private DummyBatchExecutor executor; + private List organizationIds; + @Override @Before public void setUp() throws Exception { super.setUp(); String expiringToken = this.catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); - String nonExpiringToken = this.catalogManager.getUserManager().getNonExpiringToken("opencga", Collections.emptyMap(), expiringToken); catalogManager.getConfiguration().getAnalysis().getExecution().getMaxConcurrentJobs().put(VariantIndexOperationTool.ID, 1); - daemon = new ExecutionDaemon(1000, nonExpiringToken, catalogManager, - new StorageConfiguration().setMode(StorageConfiguration.Mode.READ_WRITE), "/tmp"); + daemon = new ExecutionDaemon(1000, expiringToken, catalogManager, + new StorageConfiguration().setMode(StorageConfiguration.Mode.READ_WRITE), catalogManagerResource.getOpencgaHome().toString()); + executor = new DummyBatchExecutor(); daemon.batchExecutor = executor; + + this.organizationIds = Arrays.asList(organizationId, ParamConstants.ADMIN_ORGANIZATION); } @Test @@ -135,9 +142,9 @@ public void testBuildCli() { @Test public void testCreateDefaultOutDir() throws Exception { HashMap params = new HashMap<>(); - String jobId = catalogManager.getJobManager().submit(studyFqn, "files-delete", Enums.Priority.MEDIUM, params, token).first().getId(); + String jobId = catalogManager.getJobManager().submit(studyFqn, "files-delete", Enums.Priority.MEDIUM, params, ownerToken).first().getId(); - daemon.checkPendingJobs(); + daemon.checkPendingJobs(organizationIds); URI uri = getJob(jobId).getOutDir().getUri(); Assert.assertTrue(Files.exists(Paths.get(uri))); @@ -147,12 +154,12 @@ public void testCreateDefaultOutDir() throws Exception { @Test public void testWebhookNotification() throws Exception { catalogManager.getStudyManager().update(studyFqn, new StudyUpdateParams() - .setNotification(new StudyNotification(new URL("https://ptsv2.com/t/dgogf-1581523512/post"))), null, token); + .setNotification(new StudyNotification(new URL("https://ptsv2.com/t/dgogf-1581523512/post"))), null, ownerToken); HashMap params = new HashMap<>(); - String jobId = catalogManager.getJobManager().submit(studyFqn, "files-delete", Enums.Priority.MEDIUM, params, token).first().getId(); + String jobId = catalogManager.getJobManager().submit(studyFqn, "files-delete", Enums.Priority.MEDIUM, params, ownerToken).first().getId(); - daemon.checkPendingJobs(); + daemon.checkPendingJobs(organizationIds); // We sleep because there must be a thread sending notifying to the webhook url. Thread.sleep(1500); @@ -166,9 +173,9 @@ public void testWebhookNotification() throws Exception { public void testCreateOutDir() throws Exception { HashMap params = new HashMap<>(); params.put("outdir", "outputDir/"); - String jobId = catalogManager.getJobManager().submit(studyFqn, "files-delete", Enums.Priority.MEDIUM, params, token).first().getId(); + String jobId = catalogManager.getJobManager().submit(studyFqn, "files-delete", Enums.Priority.MEDIUM, params, ownerToken).first().getId(); - daemon.checkPendingJobs(); + daemon.checkPendingJobs(organizationIds); URI uri = getJob(jobId).getOutDir().getUri(); Assert.assertTrue(Files.exists(Paths.get(uri))); @@ -179,15 +186,15 @@ public void testCreateOutDir() throws Exception { public void testUseEmptyDirectory() throws Exception { // Create empty directory that is registered in OpenCGA org.opencb.opencga.core.models.file.File directory = catalogManager.getFileManager().createFolder(studyFqn, "outputDir/", - true, "", QueryOptions.empty(), token).first(); + true, "", QueryOptions.empty(), ownerToken).first(); catalogManager.getIoManagerFactory().get(directory.getUri()).createDirectory(directory.getUri(), true); HashMap params = new HashMap<>(); params.put("outdir", "outputDir/"); String jobId = catalogManager.getJobManager().submit(studyFqn, "files-delete", Enums.Priority.MEDIUM, params, - token).first().getId(); + ownerToken).first().getId(); - daemon.checkPendingJobs(); + daemon.checkPendingJobs(organizationIds); URI uri = getJob(jobId).getOutDir().getUri(); Assert.assertTrue(Files.exists(Paths.get(uri))); @@ -198,11 +205,11 @@ public void testUseEmptyDirectory() throws Exception { public void testNotEmptyOutDir() throws Exception { HashMap params = new HashMap<>(); params.put("outdir", "data/"); - String jobId = catalogManager.getJobManager().submit(studyFqn, "files-delete", Enums.Priority.MEDIUM, params, token).first().getId(); + String jobId = catalogManager.getJobManager().submit(studyFqn, "files-delete", Enums.Priority.MEDIUM, params, ownerToken).first().getId(); - daemon.checkPendingJobs(); + daemon.checkPendingJobs(organizationIds); - OpenCGAResult jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, jobId, QueryOptions.empty(), token); + OpenCGAResult jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, jobId, QueryOptions.empty(), ownerToken); assertEquals(1, jobOpenCGAResult.getNumResults()); checkStatus(getJob(jobId), Enums.ExecutionStatus.ABORTED); checkStatus(jobOpenCGAResult.first(), Enums.ExecutionStatus.ABORTED); @@ -210,90 +217,179 @@ public void testNotEmptyOutDir() throws Exception { } @Test - public void testProjectScopeTask() throws Exception { + public void dryRunExecutionTest() throws Exception { + ObjectMap params = new ObjectMap(); + String jobId1 = catalogManager.getJobManager().submit(studyFqn, VariantAnnotationIndexOperationTool.ID, Enums.Priority.MEDIUM, + params, "job1", "", null, null, null, null, true, ownerToken).first().getId(); + daemon.checkJobs(); + Job job = catalogManager.getJobManager().get(studyFqn, jobId1, null, ownerToken).first(); + + StorageConfiguration storageConfiguration = new StorageConfiguration(); + storageConfiguration.getVariant().setDefaultEngine("mongodb"); + ToolRunner toolRunner = new ToolRunner(catalogManagerResource.getOpencgaHome().toString(), catalogManager, + StorageEngineFactory.get(storageConfiguration)); + toolRunner.execute(job, ownerToken); + daemon.checkJobs(); + + job = catalogManager.getJobManager().get(studyFqn, jobId1, null, ownerToken).first(); + assertEquals(Enums.ExecutionStatus.DONE, job.getInternal().getStatus().getId()); + assertEquals(1, job.getExecution().getSteps().size()); + assertEquals("check", job.getExecution().getSteps().get(0).getId()); + } + + @Test + public void testProjectScopeTaskAndScheduler() throws Exception { // User 2 to admins group in study1 but not in study2 - catalogManager.getStudyManager().updateGroup(studyFqn, "@admins", ParamUtils.BasicUpdateAction.ADD, - new GroupUpdateParams(Collections.singletonList("user2")), token); + catalogManager.getStudyManager().updateGroup(studyFqn, ParamConstants.ADMINS_GROUP, ParamUtils.BasicUpdateAction.ADD, + new GroupUpdateParams(Collections.singletonList(normalUserId2)), ownerToken); // User 3 to admins group in both study1 and study2 - catalogManager.getStudyManager().updateGroup(studyFqn, "@admins", ParamUtils.BasicUpdateAction.ADD, - new GroupUpdateParams(Collections.singletonList("user3")), token); - catalogManager.getStudyManager().updateGroup(studyFqn2, "@admins", ParamUtils.BasicUpdateAction.ADD, - new GroupUpdateParams(Collections.singletonList("user3")), token); + catalogManager.getStudyManager().updateGroup(studyFqn, ParamConstants.ADMINS_GROUP, ParamUtils.BasicUpdateAction.ADD, + new GroupUpdateParams(Collections.singletonList(normalUserId3)), ownerToken); + catalogManager.getStudyManager().updateGroup(studyFqn2, ParamConstants.ADMINS_GROUP, ParamUtils.BasicUpdateAction.ADD, + new GroupUpdateParams(Collections.singletonList(normalUserId3)), ownerToken); HashMap params = new HashMap<>(); - String jobId = catalogManager.getJobManager().submit(studyFqn, VariantAnnotationIndexOperationTool.ID, Enums.Priority.MEDIUM, - params, "job1", "", null, null, token).first().getId(); + String jobId1 = catalogManager.getJobManager().submit(studyFqn, VariantAnnotationIndexOperationTool.ID, Enums.Priority.MEDIUM, + params, "job1", "", null, null, null, null, false, normalToken2).first().getId(); String jobId2 = catalogManager.getJobManager().submit(studyFqn, VariantAnnotationIndexOperationTool.ID, Enums.Priority.MEDIUM, - params, "job2", "", null, null, sessionIdUser2).first().getId(); + params, "job2", "", null, null, null, null, false, orgAdminToken1).first().getId(); String jobId3 = catalogManager.getJobManager().submit(studyFqn, VariantAnnotationIndexOperationTool.ID, Enums.Priority.MEDIUM, - params, "job3", "", null, null, sessionIdUser3).first().getId(); + params, "job3", "", null, null, null, null, false, ownerToken).first().getId(); + String jobId4 = catalogManager.getJobManager().submit(studyFqn, VariantAnnotationIndexOperationTool.ID, Enums.Priority.MEDIUM, + params, "job4", "", null, null, null, null, false, normalToken3).first().getId(); + + daemon.checkJobs(); + + // Job sent by the owner + OpenCGAResult jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, jobId3, QueryOptions.empty(), ownerToken); + assertEquals(1, jobOpenCGAResult.getNumResults()); + checkStatus(jobOpenCGAResult.first(), Enums.ExecutionStatus.QUEUED); - daemon.checkPendingJobs(); + // Job sent by normal user + jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, jobId1, QueryOptions.empty(), ownerToken); + assertEquals(1, jobOpenCGAResult.getNumResults()); + checkStatus(jobOpenCGAResult.first(), Enums.ExecutionStatus.PENDING); + + // Job sent by study administrator + jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, jobId4, QueryOptions.empty(), ownerToken); + assertEquals(1, jobOpenCGAResult.getNumResults()); + checkStatus(jobOpenCGAResult.first(), Enums.ExecutionStatus.PENDING); + + jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, jobId2, QueryOptions.empty(), ownerToken); + assertEquals(1, jobOpenCGAResult.getNumResults()); + checkStatus(jobOpenCGAResult.first(), Enums.ExecutionStatus.PENDING); + daemon.checkJobs(); + + // Because there's already a QUEUED job, it should not process any more jobs // Job sent by the owner - OpenCGAResult jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, jobId, QueryOptions.empty(), token); + jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, jobId3, QueryOptions.empty(), ownerToken); assertEquals(1, jobOpenCGAResult.getNumResults()); checkStatus(jobOpenCGAResult.first(), Enums.ExecutionStatus.QUEUED); - // Job sent by user2 (admin from study1 but not from study2) - jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, jobId2, QueryOptions.empty(), token); + // Job sent normal user + jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, jobId1, QueryOptions.empty(), ownerToken); + assertEquals(1, jobOpenCGAResult.getNumResults()); + checkStatus(jobOpenCGAResult.first(), Enums.ExecutionStatus.PENDING); + + // Job sent by study administrator + jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, jobId4, QueryOptions.empty(), ownerToken); + assertEquals(1, jobOpenCGAResult.getNumResults()); + checkStatus(jobOpenCGAResult.first(), Enums.ExecutionStatus.PENDING); + + jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, jobId2, QueryOptions.empty(), ownerToken); assertEquals(1, jobOpenCGAResult.getNumResults()); + checkStatus(jobOpenCGAResult.first(), Enums.ExecutionStatus.PENDING); + + // Set to DONE jobId3 so it can process more jobs + catalogManager.getJobManager().update(studyFqn, jobId3, new PrivateJobUpdateParams() + .setInternal(new JobInternal(new Enums.ExecutionStatus(Enums.ExecutionStatus.DONE))), QueryOptions.empty(), ownerToken); + + daemon.checkJobs(); + // Job sent by normal user should still be PENDING + jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, jobId1, QueryOptions.empty(), ownerToken); + assertEquals(1, jobOpenCGAResult.getNumResults()); + checkStatus(jobOpenCGAResult.first(), Enums.ExecutionStatus.PENDING); + + // Job sent by study administrator + jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, jobId4, QueryOptions.empty(), ownerToken); + assertEquals(1, jobOpenCGAResult.getNumResults()); + checkStatus(jobOpenCGAResult.first(), Enums.ExecutionStatus.PENDING); + + // Job sent by org admin + jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, jobId2, QueryOptions.empty(), ownerToken); + assertEquals(1, jobOpenCGAResult.getNumResults()); + checkStatus(jobOpenCGAResult.first(), Enums.ExecutionStatus.QUEUED); + + // Set to DONE jobId2 so it can process more jobs + catalogManager.getJobManager().update(studyFqn, jobId2, new PrivateJobUpdateParams() + .setInternal(new JobInternal(new Enums.ExecutionStatus(Enums.ExecutionStatus.DONE))), QueryOptions.empty(), ownerToken); + + daemon.checkJobs(); + jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, jobId1, QueryOptions.empty(), ownerToken); checkStatus(jobOpenCGAResult.first(), Enums.ExecutionStatus.ABORTED); assertTrue(jobOpenCGAResult.first().getInternal().getStatus().getDescription() .contains("can only be executed by the project owners or members of " + ParamConstants.ADMINS_GROUP)); - // Job sent by user3 (admin from study1 and study2) - jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, jobId3, QueryOptions.empty(), token); + // Job sent by study administrator + jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, jobId4, QueryOptions.empty(), ownerToken); + assertEquals(1, jobOpenCGAResult.getNumResults()); + checkStatus(jobOpenCGAResult.first(), Enums.ExecutionStatus.PENDING); + + daemon.checkJobs(); // to process jobId4 + + // Job sent by study administrator + jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, jobId4, QueryOptions.empty(), ownerToken); assertEquals(1, jobOpenCGAResult.getNumResults()); checkStatus(jobOpenCGAResult.first(), Enums.ExecutionStatus.QUEUED); - jobOpenCGAResult = catalogManager.getJobManager().search(studyFqn2, new Query(), QueryOptions.empty(), token); + jobOpenCGAResult = catalogManager.getJobManager().search(studyFqn2, new Query(), QueryOptions.empty(), ownerToken); assertEquals(0, jobOpenCGAResult.getNumResults()); jobOpenCGAResult = catalogManager.getJobManager().search(studyFqn2, new Query(), - new QueryOptions(ParamConstants.OTHER_STUDIES_FLAG, true), token); - assertEquals(2, jobOpenCGAResult.getNumResults()); + new QueryOptions(ParamConstants.OTHER_STUDIES_FLAG, true), ownerToken); + assertEquals(3, jobOpenCGAResult.getNumResults()); } @Test public void testDependsOnJobs() throws Exception { HashMap params = new HashMap<>(); - String job1 = catalogManager.getJobManager().submit(studyFqn, "files-delete", Enums.Priority.MEDIUM, params, token).first().getId(); + String job1 = catalogManager.getJobManager().submit(studyFqn, "files-delete", Enums.Priority.MEDIUM, params, ownerToken).first().getId(); String job2 = catalogManager.getJobManager().submit(studyFqn, "files-delete", Enums.Priority.MEDIUM, params, null, null, - Collections.singletonList(job1), null, token).first().getId(); + Collections.singletonList(job1), null, null, null, false, ownerToken).first().getId(); - daemon.checkPendingJobs(); + daemon.checkPendingJobs(organizationIds); - OpenCGAResult jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, job1, QueryOptions.empty(), token); + OpenCGAResult jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, job1, QueryOptions.empty(), ownerToken); assertEquals(1, jobOpenCGAResult.getNumResults()); checkStatus(jobOpenCGAResult.first(), Enums.ExecutionStatus.QUEUED); - jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, job2, QueryOptions.empty(), token); + jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, job2, QueryOptions.empty(), ownerToken); assertEquals(1, jobOpenCGAResult.getNumResults()); checkStatus(jobOpenCGAResult.first(), Enums.ExecutionStatus.PENDING); // Set the status of job1 to ERROR catalogManager.getJobManager().update(studyFqn, job1, new PrivateJobUpdateParams() - .setInternal(new JobInternal(new Enums.ExecutionStatus(Enums.ExecutionStatus.ERROR))), QueryOptions.empty(), token); + .setInternal(new JobInternal(new Enums.ExecutionStatus(Enums.ExecutionStatus.ERROR))), QueryOptions.empty(), ownerToken); // The job that depended on job1 should be ABORTED because job1 execution "failed" - daemon.checkPendingJobs(); - jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, job2, QueryOptions.empty(), token); + daemon.checkPendingJobs(organizationIds); + jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, job2, QueryOptions.empty(), ownerToken); assertEquals(1, jobOpenCGAResult.getNumResults()); checkStatus(jobOpenCGAResult.first(), Enums.ExecutionStatus.ABORTED); assertTrue(jobOpenCGAResult.first().getInternal().getStatus().getDescription().contains("depended on did not finish successfully")); // Set status of job1 to DONE to simulate it finished successfully catalogManager.getJobManager().update(studyFqn, job1, new PrivateJobUpdateParams() - .setInternal(new JobInternal(new Enums.ExecutionStatus(Enums.ExecutionStatus.DONE))), QueryOptions.empty(), token); + .setInternal(new JobInternal(new Enums.ExecutionStatus(Enums.ExecutionStatus.DONE))), QueryOptions.empty(), ownerToken); // And create a new job to simulate a normal successfully dependency String job3 = catalogManager.getJobManager().submit(studyFqn, "files-delete", Enums.Priority.MEDIUM, params, null, null, - Collections.singletonList(job1), null, token).first().getId(); - daemon.checkPendingJobs(); + Collections.singletonList(job1), null, null, null, false, ownerToken).first().getId(); + daemon.checkPendingJobs(organizationIds); - jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, job3, QueryOptions.empty(), token); + jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, job3, QueryOptions.empty(), ownerToken); assertEquals(1, jobOpenCGAResult.getNumResults()); checkStatus(jobOpenCGAResult.first(), Enums.ExecutionStatus.QUEUED); } @@ -301,14 +397,14 @@ public void testDependsOnJobs() throws Exception { @Test public void testDependsOnMultiStudy() throws Exception { HashMap params = new HashMap<>(); - Job firstJob = catalogManager.getJobManager().submit(studyFqn, "files-delete", Enums.Priority.MEDIUM, params, token).first(); + Job firstJob = catalogManager.getJobManager().submit(studyFqn, "files-delete", Enums.Priority.MEDIUM, params, ownerToken).first(); Job job = catalogManager.getJobManager().submit(studyFqn2, "files-delete", Enums.Priority.MEDIUM, params, null, null, - Collections.singletonList(firstJob.getUuid()), null, token).first(); + Collections.singletonList(firstJob.getUuid()), null, null, null, false, ownerToken).first(); assertEquals(1, job.getDependsOn().size()); assertEquals(firstJob.getId(), job.getDependsOn().get(0).getId()); assertEquals(firstJob.getUuid(), job.getDependsOn().get(0).getUuid()); - job = catalogManager.getJobManager().get(studyFqn2, job.getId(), QueryOptions.empty(), token).first(); + job = catalogManager.getJobManager().get(studyFqn2, job.getId(), QueryOptions.empty(), ownerToken).first(); assertEquals(1, job.getDependsOn().size()); assertEquals(firstJob.getId(), job.getDependsOn().get(0).getId()); assertEquals(firstJob.getUuid(), job.getDependsOn().get(0).getUuid()); @@ -318,9 +414,9 @@ public void testDependsOnMultiStudy() throws Exception { public void testRunJob() throws Exception { HashMap params = new HashMap<>(); params.put(ExecutionDaemon.OUTDIR_PARAM, "outDir"); - org.opencb.opencga.core.models.file.File inputFile = catalogManager.getFileManager().get(studyFqn, testFile1, null, token).first(); + org.opencb.opencga.core.models.file.File inputFile = catalogManager.getFileManager().get(studyFqn, testFile1, null, ownerToken).first(); params.put("myFile", inputFile.getPath()); - Job job = catalogManager.getJobManager().submit(studyFqn, "variant-index", Enums.Priority.MEDIUM, params, token).first(); + Job job = catalogManager.getJobManager().submit(studyFqn, "variant-index", Enums.Priority.MEDIUM, params, ownerToken).first(); String jobId = job.getId(); daemon.checkJobs(); @@ -348,9 +444,9 @@ public void testRunJob() throws Exception { public void testCheckLogs() throws Exception { HashMap params = new HashMap<>(); params.put(ExecutionDaemon.OUTDIR_PARAM, "outDir"); - org.opencb.opencga.core.models.file.File inputFile = catalogManager.getFileManager().get(studyFqn, testFile1, null, token).first(); + org.opencb.opencga.core.models.file.File inputFile = catalogManager.getFileManager().get(studyFqn, testFile1, null, ownerToken).first(); params.put("myFile", inputFile.getPath()); - Job job = catalogManager.getJobManager().submit(studyFqn, "variant-index", Enums.Priority.MEDIUM, params, token).first(); + Job job = catalogManager.getJobManager().submit(studyFqn, "variant-index", Enums.Priority.MEDIUM, params, ownerToken).first(); String jobId = job.getId(); daemon.checkJobs(); @@ -363,7 +459,7 @@ public void testCheckLogs() throws Exception { checkStatus(getJob(jobId), Enums.ExecutionStatus.QUEUED); executor.jobStatus.put(jobId, Enums.ExecutionStatus.RUNNING); - job = catalogManager.getJobManager().get(studyFqn, jobId, null, token).first(); + job = catalogManager.getJobManager().get(studyFqn, jobId, null, ownerToken).first(); daemon.checkJobs(); checkStatus(getJob(jobId), Enums.ExecutionStatus.RUNNING); @@ -372,10 +468,10 @@ public void testCheckLogs() throws Exception { catalogManager.getIoManagerFactory().getDefault().copy(inputStream, Paths.get(job.getOutDir().getUri()).resolve(job.getId() + ".log").toUri()); - OpenCGAResult fileContentResult = catalogManager.getJobManager().log(studyFqn, jobId, 0, 1, "stdout", true, token); + OpenCGAResult fileContentResult = catalogManager.getJobManager().log(studyFqn, jobId, 0, 1, "stdout", true, ownerToken); assertEquals("last line", fileContentResult.first().getContent()); - fileContentResult = catalogManager.getJobManager().log(studyFqn, jobId, 0, 1, "stdout", false, token); + fileContentResult = catalogManager.getJobManager().log(studyFqn, jobId, 0, 1, "stdout", false, ownerToken); assertEquals("my log content\n", fileContentResult.first().getContent()); } @@ -383,13 +479,13 @@ public void testCheckLogs() throws Exception { public void testCheckLogsNoPermissions() throws Exception { HashMap params = new HashMap<>(); params.put(ExecutionDaemon.OUTDIR_PARAM, "outDir"); - org.opencb.opencga.core.models.file.File inputFile = catalogManager.getFileManager().get(studyFqn, testFile1, null, token).first(); + org.opencb.opencga.core.models.file.File inputFile = catalogManager.getFileManager().get(studyFqn, testFile1, null, ownerToken).first(); params.put("myFile", inputFile.getPath()); - Job job = catalogManager.getJobManager().submit(studyFqn, "variant-index", Enums.Priority.MEDIUM, params, token).first(); + Job job = catalogManager.getJobManager().submit(studyFqn, "variant-index", Enums.Priority.MEDIUM, params, ownerToken).first(); String jobId = job.getId(); - catalogManager.getJobManager().updateAcl(studyFqn, Collections.singletonList(jobId), "user2", - new AclParams(JobPermissions.VIEW.name()), ParamUtils.AclAction.ADD, token); + catalogManager.getJobManager().updateAcl(studyFqn, Collections.singletonList(jobId), normalUserId2, + new AclParams(JobPermissions.VIEW.name()), ParamUtils.AclAction.ADD, ownerToken); daemon.checkJobs(); @@ -401,7 +497,7 @@ public void testCheckLogsNoPermissions() throws Exception { checkStatus(getJob(jobId), Enums.ExecutionStatus.QUEUED); executor.jobStatus.put(jobId, Enums.ExecutionStatus.RUNNING); - job = catalogManager.getJobManager().get(studyFqn, jobId, null, token).first(); + job = catalogManager.getJobManager().get(studyFqn, jobId, null, ownerToken).first(); daemon.checkJobs(); checkStatus(getJob(jobId), Enums.ExecutionStatus.RUNNING); @@ -411,16 +507,17 @@ public void testCheckLogsNoPermissions() throws Exception { Paths.get(job.getOutDir().getUri()).resolve(job.getId() + ".log").toUri()); thrown.expect(CatalogAuthorizationException.class); - catalogManager.getJobManager().log(studyFqn, jobId, 0, 1, "stdout", true, sessionIdUser2); + thrown.expectMessage("view log file"); + catalogManager.getJobManager().log(studyFqn, jobId, 0, 1, "stdout", true, normalToken2); } @Test public void testRegisterFilesSuccessfully() throws Exception { HashMap params = new HashMap<>(); // params.put(ExecutionDaemon.OUTDIR_PARAM, "outDir"); - org.opencb.opencga.core.models.file.File inputFile = catalogManager.getFileManager().get(studyFqn, testFile1, null, token).first(); + org.opencb.opencga.core.models.file.File inputFile = catalogManager.getFileManager().get(studyFqn, testFile1, null, ownerToken).first(); params.put("myFile", inputFile.getPath()); - Job job = catalogManager.getJobManager().submit(studyFqn, "variant-index", Enums.Priority.MEDIUM, params, token).first(); + Job job = catalogManager.getJobManager().submit(studyFqn, "variant-index", Enums.Priority.MEDIUM, params, ownerToken).first(); String jobId = job.getId(); daemon.checkJobs(); @@ -439,7 +536,7 @@ public void testRegisterFilesSuccessfully() throws Exception { createAnalysisResult(jobId, "myTest", false); executor.jobStatus.put(jobId, Enums.ExecutionStatus.READY); - job = catalogManager.getJobManager().get(studyFqn, job.getId(), QueryOptions.empty(), token).first(); + job = catalogManager.getJobManager().get(studyFqn, job.getId(), QueryOptions.empty(), ownerToken).first(); Files.createFile(Paths.get(job.getOutDir().getUri()).resolve("file1.txt")); Files.createFile(Paths.get(job.getOutDir().getUri()).resolve("file2.txt")); Files.createDirectory(Paths.get(job.getOutDir().getUri()).resolve("A")); @@ -452,7 +549,7 @@ public void testRegisterFilesSuccessfully() throws Exception { checkStatus(getJob(jobId), Enums.ExecutionStatus.DONE); - job = catalogManager.getJobManager().get(studyFqn, job.getId(), QueryOptions.empty(), token).first(); + job = catalogManager.getJobManager().get(studyFqn, job.getId(), QueryOptions.empty(), ownerToken).first(); String outDir = job.getOutDir().getPath(); @@ -469,7 +566,7 @@ public void testRegisterFilesSuccessfully() throws Exception { // Check jobId is properly populated OpenCGAResult files = catalogManager.getFileManager().search(studyFqn, - new Query(FileDBAdaptor.QueryParams.JOB_ID.key(), job.getId()), FileManager.INCLUDE_FILE_URI_PATH, token); + new Query(FileDBAdaptor.QueryParams.JOB_ID.key(), job.getId()), FileManager.INCLUDE_FILE_URI_PATH, ownerToken); List expectedFiles = Arrays.asList( outDir, outDir + "file1.txt", @@ -488,17 +585,40 @@ public void testRegisterFilesSuccessfully() throws Exception { } } - files = catalogManager.getFileManager().count(studyFqn, new Query(FileDBAdaptor.QueryParams.JOB_ID.key(), ""), token); - assertEquals(10, files.getNumMatches()); - files = catalogManager.getFileManager().count(studyFqn, new Query(FileDBAdaptor.QueryParams.JOB_ID.key(), "NonE"), token); - assertEquals(10, files.getNumMatches()); + files = catalogManager.getFileManager().count(studyFqn, new Query(FileDBAdaptor.QueryParams.JOB_ID.key(), ""), ownerToken); + assertEquals(16, files.getNumMatches()); + files = catalogManager.getFileManager().count(studyFqn, new Query(FileDBAdaptor.QueryParams.JOB_ID.key(), "NonE"), ownerToken); + assertEquals(16, files.getNumMatches()); + } + + @Test + public void scheduledJobTest() throws CatalogException, InterruptedException { + HashMap params = new HashMap<>(); + params.put(ExecutionDaemon.OUTDIR_PARAM, "outputDir/"); + Date date = new Date(); + // Create a date object with the current time + 2 seconds + date.setTime(date.getTime() + 2000); + String scheduledTime = TimeUtils.getTime(date); + System.out.println("Scheduled time: " + scheduledTime); + Job job = catalogManager.getJobManager().submit(studyFqn, "variant-index", Enums.Priority.MEDIUM, params, null, null, null, + null, null, scheduledTime, null, ownerToken).first(); + + daemon.checkJobs(); + checkStatus(getJob(job.getId()), Enums.ExecutionStatus.PENDING); + daemon.checkJobs(); + checkStatus(getJob(job.getId()), Enums.ExecutionStatus.PENDING); + + // Sleep for 2 seconds and check again + Thread.sleep(2000); + daemon.checkJobs(); + checkStatus(getJob(job.getId()), Enums.ExecutionStatus.QUEUED); } @Test public void testRunJobFail() throws Exception { HashMap params = new HashMap<>(); params.put(ExecutionDaemon.OUTDIR_PARAM, "outputDir/"); - Job job = catalogManager.getJobManager().submit(studyFqn, "variant-index", Enums.Priority.MEDIUM, params, token).first(); + Job job = catalogManager.getJobManager().submit(studyFqn, "variant-index", Enums.Priority.MEDIUM, params, ownerToken).first(); String jobId = job.getId(); daemon.checkJobs(); @@ -522,7 +642,7 @@ public void testRunJobFail() throws Exception { public void testRunJobFailMissingExecutionResult() throws Exception { HashMap params = new HashMap<>(); params.put(ExecutionDaemon.OUTDIR_PARAM, "outputDir/"); - Job job = catalogManager.getJobManager().submit(studyFqn, "variant-index", Enums.Priority.MEDIUM, params, token).first(); + Job job = catalogManager.getJobManager().submit(studyFqn, "variant-index", Enums.Priority.MEDIUM, params, ownerToken).first(); String jobId = job.getId(); daemon.checkJobs(); @@ -545,11 +665,11 @@ public void testRunJobFailMissingExecutionResult() throws Exception { public void registerMalformedVcfFromExecutedJobTest() throws CatalogException { HashMap params = new HashMap<>(); params.put(ExecutionDaemon.OUTDIR_PARAM, "outputDir/"); - Job job = catalogManager.getJobManager().submit(studyFqn, "variant-index", Enums.Priority.MEDIUM, params, token).first(); + Job job = catalogManager.getJobManager().submit(studyFqn, "variant-index", Enums.Priority.MEDIUM, params, ownerToken).first(); String jobId = job.getId(); daemon.checkJobs(); - job = catalogManager.getJobManager().get(studyFqn, jobId, QueryOptions.empty(), token).first(); + job = catalogManager.getJobManager().get(studyFqn, jobId, QueryOptions.empty(), ownerToken).first(); executor.jobStatus.put(jobId, Enums.ExecutionStatus.READY); try { // Create an empty VCF file (this will fail because OpenCGA will not be able to parse it) @@ -562,7 +682,7 @@ public void registerMalformedVcfFromExecutedJobTest() throws CatalogException { daemon.checkJobs(); // Check the file has been properly registered - job = catalogManager.getJobManager().get(studyFqn, jobId, QueryOptions.empty(), token).first(); + job = catalogManager.getJobManager().get(studyFqn, jobId, QueryOptions.empty(), ownerToken).first(); assertEquals(1, job.getOutput().size()); assertEquals("myemptyvcf.vcf", job.getOutput().get(0).getName()); assertEquals(File.Format.VCF, job.getOutput().get(0).getFormat()); @@ -575,7 +695,7 @@ private void checkStatus(Job job, String status) { } private Job getJob(String jobId) throws CatalogException { - return catalogManager.getJobManager().get(studyFqn, jobId, new QueryOptions(), token).first(); + return catalogManager.getJobManager().get(studyFqn, jobId, new QueryOptions(), ownerToken).first(); } private void createAnalysisResult(String jobId, String analysisId, boolean error) throws CatalogException, ToolException { @@ -598,6 +718,10 @@ public void execute(String jobId, String queue, String commandLine, Path stdout, jobStatus.put(jobId, Enums.ExecutionStatus.QUEUED); } + @Override + public void close() throws IOException { + } + @Override public String getStatus(String jobId) { return jobStatus.getOrDefault(jobId, Enums.ExecutionStatus.UNKNOWN); diff --git a/opencga-master/src/test/java/org/opencb/opencga/master/monitor/executors/LocalExecutorTest.java b/opencga-master/src/test/java/org/opencb/opencga/master/monitor/executors/LocalExecutorTest.java index 5a237749412..a350fdc3c77 100644 --- a/opencga-master/src/test/java/org/opencb/opencga/master/monitor/executors/LocalExecutorTest.java +++ b/opencga-master/src/test/java/org/opencb/opencga/master/monitor/executors/LocalExecutorTest.java @@ -31,14 +31,18 @@ import java.util.Collections; import java.util.Date; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + @Category(ShortTests.class) public class LocalExecutorTest { + public static final int MAX_CONCURRENT_JOBS = 5; private LocalExecutor localExecutor; private Path rootDir; @Before public void setUp() throws Exception { - localExecutor = new LocalExecutor(new Execution().setOptions(new ObjectMap(LocalExecutor.MAX_CONCURRENT_JOBS, 5))); + localExecutor = new LocalExecutor(new Execution().setOptions(new ObjectMap(LocalExecutor.MAX_CONCURRENT_JOBS, MAX_CONCURRENT_JOBS))); rootDir = Paths.get("target/test-data", "junit-opencga-" + new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss.SSS").format(new Date())); Files.createDirectories(rootDir); @@ -58,11 +62,41 @@ public void test() throws Exception { Thread.sleep(1000); } Assert.assertTrue(Files.exists(rootDir.resolve("out_" + i + ".txt"))); - Assert.assertEquals(Collections.singletonList("Hello World"), Files.readAllLines(rootDir.resolve("out_" + i + ".txt"))); + assertEquals(Collections.singletonList("Hello World"), Files.readAllLines(rootDir.resolve("out_" + i + ".txt"))); Assert.assertTrue(Files.exists(rootDir.resolve("err_" + i + ".txt"))); - Assert.assertEquals(Collections.emptyList(), Files.readAllLines(rootDir.resolve("err_" + i + ".txt"))); + assertEquals(Collections.emptyList(), Files.readAllLines(rootDir.resolve("err_" + i + ".txt"))); + } + } + + @Test(timeout = 10000) + public void testKill() throws Exception { + System.out.println(rootDir.toAbsolutePath()); + for (int i = 0; i < 10; i++) { +// System.out.println("Submitting job " + i); + localExecutor.execute("jobId-" + i, "default", "sleep 20", rootDir.resolve("out_" + i + ".txt"), rootDir.resolve("err_" + i + ".txt")); } + // Allow some time for the jobs to start + Thread.sleep(50); + for (int i = 0; i < 10; i++) { + String jobId = "jobId-" + i; +// System.out.println("Checking status of job " + jobId); + if (i < MAX_CONCURRENT_JOBS) { + assertEquals(jobId, "RUNNING", localExecutor.getStatus(jobId)); + } else { + assertEquals(jobId, "QUEUED", localExecutor.getStatus(jobId)); + } + } + for (int i = 0; i < 10; i++) { + String jobId = "jobId-" + i; +// System.out.println("Checking status of job " + jobId); + do { + Thread.sleep(100); + } while (!localExecutor.getStatus(jobId).equals("RUNNING")); + assertEquals("RUNNING", localExecutor.getStatus(jobId)); + assertTrue(localExecutor.kill(jobId)); + } } + } \ No newline at end of file diff --git a/opencga-master/src/test/java/org/opencb/opencga/master/monitor/models/PrivateJobUpdateParamsTest.java b/opencga-master/src/test/java/org/opencb/opencga/master/monitor/models/PrivateJobUpdateParamsTest.java index 8ab9f740c71..f58c461736c 100644 --- a/opencga-master/src/test/java/org/opencb/opencga/master/monitor/models/PrivateJobUpdateParamsTest.java +++ b/opencga-master/src/test/java/org/opencb/opencga/master/monitor/models/PrivateJobUpdateParamsTest.java @@ -67,11 +67,11 @@ public void test() throws JsonProcessingException { @Test public void updateJobInformation() throws CatalogException { OpenCGAResult jobResult = catalogManager.getJobManager().submit(studyFqn, "variant-index", Enums.Priority.HIGH, - new HashMap<>(), token); + new HashMap<>(), ownerToken); PrivateJobUpdateParams updateParams = new PrivateJobUpdateParams().setCommandLine("myCommandLine"); - catalogManager.getJobManager().update(studyFqn, jobResult.first().getId(), updateParams, QueryOptions.empty(), token); - jobResult = catalogManager.getJobManager().get(studyFqn, jobResult.first().getId(), QueryOptions.empty(), token); + catalogManager.getJobManager().update(studyFqn, jobResult.first().getId(), updateParams, QueryOptions.empty(), ownerToken); + jobResult = catalogManager.getJobManager().get(studyFqn, jobResult.first().getId(), QueryOptions.empty(), ownerToken); assertEquals("myCommandLine", jobResult.first().getCommandLine()); @@ -80,8 +80,8 @@ public void updateJobInformation() throws CatalogException { .setPath("/tmp/path") .setId("myJobId")) .setStudy(new JobStudyParam(studyFqn, Arrays.asList(studyFqn2, studyFqn3))); - catalogManager.getJobManager().update(studyFqn, jobResult.first().getId(), updateParams, QueryOptions.empty(), token); - jobResult = catalogManager.getJobManager().get(studyFqn, jobResult.first().getId(), QueryOptions.empty(), token); + catalogManager.getJobManager().update(studyFqn, jobResult.first().getId(), updateParams, QueryOptions.empty(), ownerToken); + jobResult = catalogManager.getJobManager().get(studyFqn, jobResult.first().getId(), QueryOptions.empty(), ownerToken); assertEquals(2, jobResult.first().getStudy().getOthers().size()); System.out.println(jobResult); diff --git a/opencga-server/pom.xml b/opencga-server/pom.xml index 2b5d462abff..eabe8a65d7c 100644 --- a/opencga-server/pom.xml +++ b/opencga-server/pom.xml @@ -106,10 +106,6 @@ com.fasterxml.jackson.core jackson-core - - org.codehaus.jackson - jackson-mapper-asl - com.google.protobuf protobuf-java @@ -375,9 +371,8 @@ storage-hadoop - storage-hadoop + !skipStorageHadoop - true @@ -387,9 +382,9 @@ org.opencb.opencga - ${opencga-storage-hadoop-deps.artifactId} + ${opencga-hadoop-lib.artifactId} ${project.parent.version} - shaded + true @@ -397,7 +392,9 @@ CodeGen - false + + !skipCodeGen + diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/generator/ClientsGenerator.java b/opencga-server/src/main/java/org/opencb/opencga/server/generator/ClientsGenerator.java index b9082449a32..f94a917e20f 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/generator/ClientsGenerator.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/generator/ClientsGenerator.java @@ -54,14 +54,18 @@ private void generateLibrary(String clientsGeneratorDir, String language, String System.out.println("outDir " + outDir); String binary = clientsGeneratorDir + "/" + language + "_client_generator.py"; ProcessBuilder processBuilder = new ProcessBuilder("python3", binary, restFilePath, outDir); + System.out.println("python3 " + binary + " " + restFilePath + " " + outDir); Process p; try { p = processBuilder.start(); BufferedReader input = new BufferedReader(new InputStreamReader(p.getInputStream())); + BufferedReader error = new BufferedReader(new InputStreamReader(p.getErrorStream())); String line; while ((line = input.readLine()) != null) { logger.info("{} library generator: {}", language, line); - System.out.println(language + " library generator: " + line); + } + while ((line = error.readLine()) != null) { + logger.error("{} library generator: {}", language, line); } p.waitFor(); input.close(); diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/generator/ClientsGeneratorMain.java b/opencga-server/src/main/java/org/opencb/opencga/server/generator/ClientsGeneratorMain.java index 74874049837..33014c26b4e 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/generator/ClientsGeneratorMain.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/generator/ClientsGeneratorMain.java @@ -34,6 +34,7 @@ public static void main(String[] args) throws URISyntaxException { Configurator.reconfigure(); List> classes = new ArrayList<>(); + classes.add(OrganizationWSServer.class); classes.add(UserWSServer.class); classes.add(ProjectWSServer.class); classes.add(StudyWSServer.class); diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/AutoCompleteWriter.java b/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/AutoCompleteWriter.java index d4f85542ddf..1d58459b343 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/AutoCompleteWriter.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/AutoCompleteWriter.java @@ -25,8 +25,6 @@ import java.io.File; import java.io.IOException; -import java.text.SimpleDateFormat; -import java.util.Date; public class AutoCompleteWriter extends ParentClientRestApiWriter { @@ -37,23 +35,6 @@ public AutoCompleteWriter(RestApi restApi, CommandLineConfiguration config) { @Override protected String getClassImports(String key) { StringBuilder sb = new StringBuilder(); - SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); - sb.append("/*\n"); - sb.append("* Copyright 2015-").append(sdf.format(new Date())).append(" OpenCB\n"); - sb.append("*\n"); - sb.append("* Licensed under the Apache License, Version 2.0 (the \"License\");\n"); - sb.append("* you may not use this file except in compliance with the License.\n"); - sb.append("* You may obtain a copy of the License at\n"); - sb.append("*\n"); - sb.append("* http://www.apache.org/licenses/LICENSE-2.0\n"); - sb.append("*\n"); - sb.append("* Unless required by applicable law or agreed to in writing, software\n"); - sb.append("* distributed under the License is distributed on an \"AS IS\" BASIS,\n"); - sb.append("* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n"); - sb.append("* See the License for the specific language governing permissions and\n"); - sb.append("* limitations under the License.\n"); - sb.append("*/\n"); - sb.append("\n"); sb.append("package ").append(config.getOptions().getParserPackage()).append(";\n"); sb.append("\n"); diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/ExecutorsCliRestApiWriter.java b/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/ExecutorsCliRestApiWriter.java index 8befc2876c5..faa3dd8626e 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/ExecutorsCliRestApiWriter.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/ExecutorsCliRestApiWriter.java @@ -401,9 +401,11 @@ private String getBodyParams(RestParameter body) { String javaCommandOptionsField = "commandOptions." + getJavaFieldName(restParameter); String label = StringUtils.isEmpty(restParameter.getParentName()) ? restParameter.getName() : restParameter.getParentName() + "." + restParameter.getName(); if (restParameter.getTypeClass().equals("java.lang.String;")) { - sb.append(" putNestedIfNotEmpty(beanParams, \"" + label.replaceAll("body_", "") + "\"," + javaCommandOptionsField + ", true);\n"); + sb.append(" putNestedIfNotEmpty(beanParams, \"" + label.replaceAll("body_", "") + "\", " + javaCommandOptionsField + ", true);\n"); + } else if (isValidMap(restParameter)) { + sb.append(" putNestedMapIfNotEmpty(beanParams, \"" + label.replaceAll("body_", "") + "\", " + javaCommandOptionsField + ", true);\n"); } else { - sb.append(" putNestedIfNotNull(beanParams, \"" + label.replaceAll("body_", "") + "\"," + javaCommandOptionsField + ", true);\n"); + sb.append(" putNestedIfNotNull(beanParams, \"" + label.replaceAll("body_", "") + "\", " + javaCommandOptionsField + ", true);\n"); } } } diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/OptionsCliRestApiWriter.java b/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/OptionsCliRestApiWriter.java index bfa784319b7..90b8977decc 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/OptionsCliRestApiWriter.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/OptionsCliRestApiWriter.java @@ -28,7 +28,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.text.SimpleDateFormat; import java.util.HashSet; import java.util.Set; @@ -45,7 +44,6 @@ protected String getClassImports(String key) { StringBuilder sb = new StringBuilder(); RestCategory restCategory = availableCategories.get(key); CategoryConfig categoryConfig = availableCategoryConfigs.get(key); - SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); sb.append("package ").append(config.getOptions().getOptionsPackage()).append(";\n\n"); sb.append("import com.beust.jcommander.JCommander;\n"); sb.append("import com.beust.jcommander.Parameter;\n"); diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/ParserCliRestApiWriter.java b/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/ParserCliRestApiWriter.java index 121166bd9d2..669a5151c61 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/ParserCliRestApiWriter.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/ParserCliRestApiWriter.java @@ -27,8 +27,6 @@ import java.io.File; import java.io.IOException; -import java.text.SimpleDateFormat; -import java.util.Date; public class ParserCliRestApiWriter extends ParentClientRestApiWriter { @@ -39,23 +37,6 @@ public ParserCliRestApiWriter(RestApi restApi, CommandLineConfiguration config) @Override protected String getClassImports(String key) { StringBuilder sb = new StringBuilder(); - SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); - sb.append("/*\n"); - sb.append("* Copyright 2015-").append(sdf.format(new Date())).append(" OpenCB\n"); - sb.append("*\n"); - sb.append("* Licensed under the Apache License, Version 2.0 (the \"License\");\n"); - sb.append("* you may not use this file except in compliance with the License.\n"); - sb.append("* You may obtain a copy of the License at\n"); - sb.append("*\n"); - sb.append("* http://www.apache.org/licenses/LICENSE-2.0\n"); - sb.append("*\n"); - sb.append("* Unless required by applicable law or agreed to in writing, software\n"); - sb.append("* distributed under the License is distributed on an \"AS IS\" BASIS,\n"); - sb.append("* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n"); - sb.append("* See the License for the specific language governing permissions and\n"); - sb.append("* limitations under the License.\n"); - sb.append("*/\n"); - sb.append("\n"); sb.append("package ").append(config.getOptions().getParserPackage()).append(";\n"); sb.append("\n"); diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/CohortWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/CohortWSServer.java index 82815df680f..e64e01e9ad1 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/CohortWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/CohortWSServer.java @@ -394,36 +394,36 @@ public Response updateAcl( } } - @GET - @Path("/aggregationStats") - @ApiOperation(value = "Fetch catalog cohort stats", response = FacetField.class) - public Response getAggregationStats( - @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) - @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, - @ApiParam(value = "Type") @QueryParam("type") String type, - @ApiParam(value = "Creation year") @QueryParam("creationYear") String creationYear, - @ApiParam(value = "Creation month (JANUARY, FEBRUARY...)") @QueryParam("creationMonth") String creationMonth, - @ApiParam(value = "Creation day") @QueryParam("creationDay") String creationDay, - @ApiParam(value = "Creation day of week (MONDAY, TUESDAY...)") @QueryParam("creationDayOfWeek") String creationDayOfWeek, - @ApiParam(value = "Number of samples") @QueryParam("numSamples") String numSamples, - @ApiParam(value = "Status") @QueryParam("status") String status, - @ApiParam(value = "Release") @QueryParam("release") String release, - @ApiParam(value = ParamConstants.ANNOTATION_DESCRIPTION) @QueryParam("annotation") String annotation, - - @ApiParam(value = "Calculate default stats", defaultValue = "false") @QueryParam("default") boolean defaultStats, - @ApiParam(value = "List of fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type;numSamples[0..10]:1") @QueryParam("field") String facet) { - try { - query.remove(ParamConstants.STUDY_PARAM); - query.remove("field"); - - queryOptions.put(QueryOptions.FACET, facet); - - DataResult queryResult = catalogManager.getCohortManager().facet(studyStr, query, queryOptions, defaultStats, - token); - return createOkResponse(queryResult); - } catch (Exception e) { - return createErrorResponse(e); - } - } +// @GET +// @Path("/aggregationStats") +// @ApiOperation(value = "Fetch catalog cohort stats", response = FacetField.class) +// public Response getAggregationStats( +// @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) +// @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, +// @ApiParam(value = "Type") @QueryParam("type") String type, +// @ApiParam(value = "Creation year") @QueryParam("creationYear") String creationYear, +// @ApiParam(value = "Creation month (JANUARY, FEBRUARY...)") @QueryParam("creationMonth") String creationMonth, +// @ApiParam(value = "Creation day") @QueryParam("creationDay") String creationDay, +// @ApiParam(value = "Creation day of week (MONDAY, TUESDAY...)") @QueryParam("creationDayOfWeek") String creationDayOfWeek, +// @ApiParam(value = "Number of samples") @QueryParam("numSamples") String numSamples, +// @ApiParam(value = "Status") @QueryParam("status") String status, +// @ApiParam(value = "Release") @QueryParam("release") String release, +// @ApiParam(value = ParamConstants.ANNOTATION_DESCRIPTION) @QueryParam("annotation") String annotation, +// +// @ApiParam(value = "Calculate default stats", defaultValue = "false") @QueryParam("default") boolean defaultStats, +// @ApiParam(value = "List of fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type;numSamples[0..10]:1") @QueryParam("field") String facet) { +// try { +// query.remove(ParamConstants.STUDY_PARAM); +// query.remove("field"); +// +// queryOptions.put(QueryOptions.FACET, facet); +// +// DataResult queryResult = catalogManager.getCohortManager().facet(studyStr, query, queryOptions, defaultStats, +// token); +// return createOkResponse(queryResult); +// } catch (Exception e) { +// return createErrorResponse(e); +// } +// } } diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/FamilyWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/FamilyWSServer.java index 621342e3753..38be27971c9 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/FamilyWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/FamilyWSServer.java @@ -16,19 +16,11 @@ package org.opencb.opencga.server.rest; -import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.ObjectUtils; -import org.apache.commons.lang3.time.StopWatch; import org.opencb.commons.datastore.core.DataResult; -import org.opencb.commons.datastore.core.FacetField; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.analysis.family.FamilyTsvAnnotationLoader; -import org.opencb.opencga.analysis.family.PedigreeGraphAnalysis; -import org.opencb.opencga.analysis.tools.ToolRunner; -import org.opencb.opencga.analysis.variant.circos.CircosAnalysis; -import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; -import org.opencb.opencga.analysis.variant.mutationalSignature.MutationalSignatureAnalysis; import org.opencb.opencga.catalog.db.api.FamilyDBAdaptor; import org.opencb.opencga.catalog.managers.FamilyManager; import org.opencb.opencga.catalog.utils.Constants; @@ -38,20 +30,15 @@ import org.opencb.opencga.core.models.common.TsvAnnotationParams; import org.opencb.opencga.core.models.family.*; import org.opencb.opencga.core.models.job.Job; -import org.opencb.opencga.core.models.variant.MutationalSignatureAnalysisParams; -import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.tools.annotations.*; -import org.opencb.opencga.storage.core.StorageEngineFactory; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.*; import javax.ws.rs.core.*; -import java.io.File; -import java.io.FileInputStream; import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Paths; -import java.util.*; +import java.util.HashMap; +import java.util.List; +import java.util.Map; /** * Created by pfurio on 03/05/17. @@ -363,38 +350,38 @@ public Response updateAcl( } } - @GET - @Path("/aggregationStats") - @ApiOperation(value = "Fetch catalog family stats", response = FacetField.class) - public Response getAggregationStats( - @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) - @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, - @ApiParam(value = "Creation year") @QueryParam("creationYear") String creationYear, - @ApiParam(value = "Creation month (JANUARY, FEBRUARY...)") @QueryParam("creationMonth") String creationMonth, - @ApiParam(value = "Creation day") @QueryParam("creationDay") String creationDay, - @ApiParam(value = "Creation day of week (MONDAY, TUESDAY...)") @QueryParam("creationDayOfWeek") String creationDayOfWeek, - @ApiParam(value = "Status") @QueryParam("status") String status, - @ApiParam(value = "Phenotypes") @QueryParam("phenotypes") String phenotypes, - @ApiParam(value = "Release") @QueryParam("release") String release, - @ApiParam(value = "Version") @QueryParam("version") String version, - @ApiParam(value = "Number of members") @QueryParam("numMembers") String numMembers, - @ApiParam(value = "Expected size") @QueryParam("expectedSize") String expectedSize, - @ApiParam(value = ParamConstants.ANNOTATION_DESCRIPTION) @QueryParam("annotation") String annotation, - - @ApiParam(value = "Calculate default stats", defaultValue = "false") @QueryParam("default") boolean defaultStats, - - @ApiParam(value = "List of fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type;numSamples[0..10]:1") @QueryParam("field") String facet) { - try { - query.remove(ParamConstants.STUDY_PARAM); - query.remove("field"); - - queryOptions.put(QueryOptions.FACET, facet); - - DataResult queryResult = catalogManager.getFamilyManager().facet(studyStr, query, queryOptions, defaultStats, - token); - return createOkResponse(queryResult); - } catch (Exception e) { - return createErrorResponse(e); - } - } +// @GET +// @Path("/aggregationStats") +// @ApiOperation(value = "Fetch catalog family stats", response = FacetField.class) +// public Response getAggregationStats( +// @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) +// @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, +// @ApiParam(value = "Creation year") @QueryParam("creationYear") String creationYear, +// @ApiParam(value = "Creation month (JANUARY, FEBRUARY...)") @QueryParam("creationMonth") String creationMonth, +// @ApiParam(value = "Creation day") @QueryParam("creationDay") String creationDay, +// @ApiParam(value = "Creation day of week (MONDAY, TUESDAY...)") @QueryParam("creationDayOfWeek") String creationDayOfWeek, +// @ApiParam(value = "Status") @QueryParam("status") String status, +// @ApiParam(value = "Phenotypes") @QueryParam("phenotypes") String phenotypes, +// @ApiParam(value = "Release") @QueryParam("release") String release, +// @ApiParam(value = "Version") @QueryParam("version") String version, +// @ApiParam(value = "Number of members") @QueryParam("numMembers") String numMembers, +// @ApiParam(value = "Expected size") @QueryParam("expectedSize") String expectedSize, +// @ApiParam(value = ParamConstants.ANNOTATION_DESCRIPTION) @QueryParam("annotation") String annotation, +// +// @ApiParam(value = "Calculate default stats", defaultValue = "false") @QueryParam("default") boolean defaultStats, +// +// @ApiParam(value = "List of fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type;numSamples[0..10]:1") @QueryParam("field") String facet) { +// try { +// query.remove(ParamConstants.STUDY_PARAM); +// query.remove("field"); +// +// queryOptions.put(QueryOptions.FACET, facet); +// +// DataResult queryResult = catalogManager.getFamilyManager().facet(studyStr, query, queryOptions, defaultStats, +// token); +// return createOkResponse(queryResult); +// } catch (Exception e) { +// return createErrorResponse(e); +// } +// } } diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/FileWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/FileWSServer.java index 893a4fea3d8..01be67e5a16 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/FileWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/FileWSServer.java @@ -98,9 +98,12 @@ public Response downloadAndRegister( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, @ApiParam(name = "body", value = "Fetch parameters", required = true) FileFetch fetchParams) { - return submitJob(FetchAndRegisterTask.ID, studyStr, fetchParams, jobId, jobDescription, dependsOn, jobTags); + return submitJob(FetchAndRegisterTask.ID, studyStr, fetchParams, jobId, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @GET @@ -400,7 +403,7 @@ public Response list(@ApiParam(value = ParamConstants.FILE_FOLDER_DESCRIPTION) @ @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr) { try { ParamUtils.checkIsSingleID(folder); - DataResult result = catalogManager.getFileManager().getFilesFromFolder(folder, studyStr, queryOptions, token); + DataResult result = catalogManager.getFileManager().getFilesFromFolder(studyStr, folder, queryOptions, token); return createOkResponse(result); } catch (Exception e) { return createErrorResponse(e); @@ -709,8 +712,11 @@ public Response postlink( @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(name = "body", value = "File parameters", required = true) PostLinkToolParams params) { - return submitJob(PostLinkSampleAssociation.ID, studyStr, params, jobId, jobDescription, dependsOn, jobTags); + return submitJob(PostLinkSampleAssociation.ID, studyStr, params, jobId, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -722,8 +728,11 @@ public Response linkAsync( @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(name = "body", value = "File parameters", required = true) FileLinkToolParams params) { - return submitJob(FileLinkTask.ID, studyStr, params, jobId, jobDescription, dependsOn, jobTags); + return submitJob(FileLinkTask.ID, studyStr, params, jobId, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @DELETE @@ -881,7 +890,7 @@ public Response refresh( files = Collections.singletonList(file); } } else { - List result = catalogManager.getFileManager().getFilesFromFolder(fileIdStr, studyStr, null, token).getResults(); + List result = catalogManager.getFileManager().getFilesFromFolder(studyStr, fileIdStr, null, token).getResults(); files = new ArrayList<>(result.size()); for (File f : result) { File file1 = fileMetadataReader.updateMetadataInformation(studyStr, f, token); @@ -904,7 +913,7 @@ public Response refresh( // + " RECOVERED)", dataType = "boolean", defaultValue = "false", paramType = "query") // }) // public Response delete( -// @ApiParam(value = "Study [[user@]project:]study where study and project can be either the id or alias") +// @ApiParam(value = "Study [[organization@]project:]study where study and project can be either the id or alias") // @QueryParam("study") String studyStr, // @ApiParam(value = "Comma separated list of file names") @QueryParam("name") String name, // @ApiParam(value = "Comma separated list of paths") @QueryParam("path") String path, @@ -1015,44 +1024,44 @@ public Response updateAcl( // } // } - @GET - @Path("/aggregationStats") - @ApiOperation(value = "Fetch catalog file stats", response = FacetField.class) - public Response getAggregationStats( - @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) - @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, - @ApiParam(value = "Name") @QueryParam("name") String name, - @ApiParam(value = "Type") @QueryParam("type") String type, - @ApiParam(value = "Format") @QueryParam("format") String format, - @ApiParam(value = "Bioformat") @QueryParam("bioformat") String bioformat, - @ApiParam(value = "Creation year") @QueryParam("creationYear") String creationYear, - @ApiParam(value = "Creation month (JANUARY, FEBRUARY...)") @QueryParam("creationMonth") String creationMonth, - @ApiParam(value = "Creation day") @QueryParam("creationDay") String creationDay, - @ApiParam(value = "Creation day of week (MONDAY, TUESDAY...)") @QueryParam("creationDayOfWeek") String creationDayOfWeek, - @ApiParam(value = "Status") @QueryParam("status") String status, - @ApiParam(value = "Release") @QueryParam("release") String release, - @ApiParam(value = "External") @QueryParam("external") Boolean external, - @ApiParam(value = "Size") @QueryParam("size") String size, - @ApiParam(value = "Software") @QueryParam("software") String software, - @ApiParam(value = "Experiment") @QueryParam("experiment") String experiment, - @ApiParam(value = "Number of samples") @QueryParam("numSamples") String numSamples, - @ApiParam(value = "Number of related files") @QueryParam("numRelatedFiles") String numRelatedFiles, - @ApiParam(value = ParamConstants.ANNOTATION_DESCRIPTION) @QueryParam("annotation") String annotation, - - @ApiParam(value = "Calculate default stats", defaultValue = "false") @QueryParam("default") boolean defaultStats, - - @ApiParam(value = "List of fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: " + - "studies>>biotype;type;numSamples[0..10]:1") @QueryParam("field") String facet) { - try { - query.remove(ParamConstants.STUDY_PARAM); - query.remove("field"); - - queryOptions.put(QueryOptions.FACET, facet); - - DataResult queryResult = catalogManager.getFileManager().facet(studyStr, query, queryOptions, defaultStats, token); - return createOkResponse(queryResult); - } catch (Exception e) { - return createErrorResponse(e); - } - } +// @GET +// @Path("/aggregationStats") +// @ApiOperation(value = "Fetch catalog file stats", response = FacetField.class) +// public Response getAggregationStats( +// @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) +// @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, +// @ApiParam(value = "Name") @QueryParam("name") String name, +// @ApiParam(value = "Type") @QueryParam("type") String type, +// @ApiParam(value = "Format") @QueryParam("format") String format, +// @ApiParam(value = "Bioformat") @QueryParam("bioformat") String bioformat, +// @ApiParam(value = "Creation year") @QueryParam("creationYear") String creationYear, +// @ApiParam(value = "Creation month (JANUARY, FEBRUARY...)") @QueryParam("creationMonth") String creationMonth, +// @ApiParam(value = "Creation day") @QueryParam("creationDay") String creationDay, +// @ApiParam(value = "Creation day of week (MONDAY, TUESDAY...)") @QueryParam("creationDayOfWeek") String creationDayOfWeek, +// @ApiParam(value = "Status") @QueryParam("status") String status, +// @ApiParam(value = "Release") @QueryParam("release") String release, +// @ApiParam(value = "External") @QueryParam("external") Boolean external, +// @ApiParam(value = "Size") @QueryParam("size") String size, +// @ApiParam(value = "Software") @QueryParam("software") String software, +// @ApiParam(value = "Experiment") @QueryParam("experiment") String experiment, +// @ApiParam(value = "Number of samples") @QueryParam("numSamples") String numSamples, +// @ApiParam(value = "Number of related files") @QueryParam("numRelatedFiles") String numRelatedFiles, +// @ApiParam(value = ParamConstants.ANNOTATION_DESCRIPTION) @QueryParam("annotation") String annotation, +// +// @ApiParam(value = "Calculate default stats", defaultValue = "false") @QueryParam("default") boolean defaultStats, +// +// @ApiParam(value = "List of fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: " + +// "studies>>biotype;type;numSamples[0..10]:1") @QueryParam("field") String facet) { +// try { +// query.remove(ParamConstants.STUDY_PARAM); +// query.remove("field"); +// +// queryOptions.put(QueryOptions.FACET, facet); +// +// DataResult queryResult = catalogManager.getFileManager().facet(studyStr, query, queryOptions, defaultStats, token); +// return createOkResponse(queryResult); +// } catch (Exception e) { +// return createErrorResponse(e); +// } +// } } diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/IndividualWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/IndividualWSServer.java index ac061670b52..089e6719372 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/IndividualWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/IndividualWSServer.java @@ -19,7 +19,6 @@ import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.opencb.commons.datastore.core.DataResult; -import org.opencb.commons.datastore.core.FacetField; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.analysis.individual.IndividualTsvAnnotationLoader; @@ -439,45 +438,45 @@ public Response updateAcl( } } - @GET - @Path("/aggregationStats") - @ApiOperation(value = "Fetch catalog individual stats", response = FacetField.class) - public Response getAggregationStats( - @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) - @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, - @ApiParam(value = "Has father") @QueryParam("hasFather") Boolean hasFather, - @ApiParam(value = "Has mother") @QueryParam("hasMother") Boolean hasMother, - @ApiParam(value = "Sex") @QueryParam("sex") String sex, - @ApiParam(value = "Karyotypic sex") @QueryParam("karyotypicSex") String karyotypicSex, - @ApiParam(value = "Ethnicity") @QueryParam("ethnicity") String ethnicity, - @ApiParam(value = "Population") @QueryParam("population") String population, - @ApiParam(value = "Creation year") @QueryParam("creationYear") String creationYear, - @ApiParam(value = "Creation month (JANUARY, FEBRUARY...)") @QueryParam("creationMonth") String creationMonth, - @ApiParam(value = "Creation day") @QueryParam("creationDay") String creationDay, - @ApiParam(value = "Creation day of week (MONDAY, TUESDAY...)") @QueryParam("creationDayOfWeek") String creationDayOfWeek, - @ApiParam(value = "Status") @QueryParam("status") String status, - @ApiParam(value = "Life status") @QueryParam("lifeStatus") String lifeStatus, - @ApiParam(value = "Phenotypes") @QueryParam("phenotypes") String phenotypes, - @ApiParam(value = "Number of samples") @QueryParam("numSamples") String numSamples, - @ApiParam(value = "Parental consanguinity") @QueryParam("parentalConsanguinity") Boolean parentalConsanguinity, - @ApiParam(value = "Release") @QueryParam("release") String release, - @ApiParam(value = "Version") @QueryParam("version") String version, - @ApiParam(value = ParamConstants.ANNOTATION_DESCRIPTION) @QueryParam("annotation") String annotation, - - @ApiParam(value = "Calculate default stats", defaultValue = "false") @QueryParam("default") boolean defaultStats, - - @ApiParam(value = "List of fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type;numSamples[0..10]:1") @QueryParam("field") String facet) { - try { - query.remove(ParamConstants.STUDY_PARAM); - query.remove("field"); - - queryOptions.put(QueryOptions.FACET, facet); - - DataResult queryResult = catalogManager.getIndividualManager().facet(studyStr, query, queryOptions, defaultStats, - token); - return createOkResponse(queryResult); - } catch (Exception e) { - return createErrorResponse(e); - } - } +// @GET +// @Path("/aggregationStats") +// @ApiOperation(value = "Fetch catalog individual stats", response = FacetField.class) +// public Response getAggregationStats( +// @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) +// @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, +// @ApiParam(value = "Has father") @QueryParam("hasFather") Boolean hasFather, +// @ApiParam(value = "Has mother") @QueryParam("hasMother") Boolean hasMother, +// @ApiParam(value = "Sex") @QueryParam("sex") String sex, +// @ApiParam(value = "Karyotypic sex") @QueryParam("karyotypicSex") String karyotypicSex, +// @ApiParam(value = "Ethnicity") @QueryParam("ethnicity") String ethnicity, +// @ApiParam(value = "Population") @QueryParam("population") String population, +// @ApiParam(value = "Creation year") @QueryParam("creationYear") String creationYear, +// @ApiParam(value = "Creation month (JANUARY, FEBRUARY...)") @QueryParam("creationMonth") String creationMonth, +// @ApiParam(value = "Creation day") @QueryParam("creationDay") String creationDay, +// @ApiParam(value = "Creation day of week (MONDAY, TUESDAY...)") @QueryParam("creationDayOfWeek") String creationDayOfWeek, +// @ApiParam(value = "Status") @QueryParam("status") String status, +// @ApiParam(value = "Life status") @QueryParam("lifeStatus") String lifeStatus, +// @ApiParam(value = "Phenotypes") @QueryParam("phenotypes") String phenotypes, +// @ApiParam(value = "Number of samples") @QueryParam("numSamples") String numSamples, +// @ApiParam(value = "Parental consanguinity") @QueryParam("parentalConsanguinity") Boolean parentalConsanguinity, +// @ApiParam(value = "Release") @QueryParam("release") String release, +// @ApiParam(value = "Version") @QueryParam("version") String version, +// @ApiParam(value = ParamConstants.ANNOTATION_DESCRIPTION) @QueryParam("annotation") String annotation, +// +// @ApiParam(value = "Calculate default stats", defaultValue = "false") @QueryParam("default") boolean defaultStats, +// +// @ApiParam(value = "List of fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type;numSamples[0..10]:1") @QueryParam("field") String facet) { +// try { +// query.remove(ParamConstants.STUDY_PARAM); +// query.remove("field"); +// +// queryOptions.put(QueryOptions.FACET, facet); +// +// DataResult queryResult = catalogManager.getIndividualManager().facet(studyStr, query, queryOptions, defaultStats, +// token); +// return createOkResponse(queryResult); +// } catch (Exception e) { +// return createErrorResponse(e); +// } +// } } diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/JobWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/JobWSServer.java index 113be86571a..536d692099e 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/JobWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/JobWSServer.java @@ -18,8 +18,6 @@ import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; -import org.opencb.commons.datastore.core.DataResult; -import org.opencb.commons.datastore.core.FacetField; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.catalog.db.api.JobDBAdaptor; @@ -83,6 +81,7 @@ public Response retryJob( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTagsStr, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String jobScheduledStartTime, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = "job", required = true) JobRetryParams params @@ -101,14 +100,24 @@ public Response retryJob( } else { jobTags = Collections.emptyList(); } - OpenCGAResult result = catalogManager.getJobManager().retry(study, params, - null, jobId, jobDescription, jobDependsOn, jobTags, token); + OpenCGAResult result = catalogManager.getJobManager().retry(study, params, null, jobId, jobDescription, jobDependsOn, + jobTags, jobScheduledStartTime, token); return createOkResponse(result); } catch (Exception e) { return createErrorResponse(e); } } + @POST + @Path("/{job}/kill") + @Consumes(MediaType.APPLICATION_JSON) + @ApiOperation(value = "Send a signal to kill a pending or running job", response = Job.class) + public Response kill( + @ApiParam(value = ParamConstants.JOB_ID_DESCRIPTION, required = true) @PathParam("job") String jobId, + @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study) { + return run(() -> catalogManager.getJobManager().kill(study, jobId, token)); + } + @GET @Path("/{jobs}/info") @ApiOperation(value = "Get job information", response = Job.class) @@ -357,39 +366,39 @@ public Response updateAcl( } - @GET - @Path("/aggregationStats") - @ApiOperation(value = "Fetch catalog job stats", response = FacetField.class) - public Response getAggregationStats( - @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, - @ApiParam(value = "Tool id") @QueryParam("toolId") String toolId, - @ApiParam(value = "Tool scope") @QueryParam("toolScope") String toolScope, - @ApiParam(value = "Tool type") @QueryParam("toolType") String toolType, - @ApiParam(value = "Tool resource") @QueryParam("toolResource") String toolResource, - @ApiParam(value = "User id") @QueryParam("userId") String userId, - @ApiParam(value = "Priority") @QueryParam("priority") String priority, - @ApiParam(value = "Tags") @QueryParam("tags") String tags, - @ApiParam(value = "Executor id") @QueryParam("executorId") String executorId, - @ApiParam(value = "Executor framework") @QueryParam("executorFramework") String executorFramework, - @ApiParam(value = "Creation year") @QueryParam("creationYear") String creationYear, - @ApiParam(value = "Creation month (JANUARY, FEBRUARY...)") @QueryParam("creationMonth") String creationMonth, - @ApiParam(value = "Creation day") @QueryParam("creationDay") String creationDay, - @ApiParam(value = "Creation day of week (MONDAY, TUESDAY...)") @QueryParam("creationDayOfWeek") String creationDayOfWeek, - @ApiParam(value = "Status") @QueryParam("status") String status, - @ApiParam(value = "Release") @QueryParam("release") String release, - @ApiParam(value = "Calculate default stats", defaultValue = "false") @QueryParam("default") boolean defaultStats, - @ApiParam(value = "List of fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type;numSamples[0..10]:1") @QueryParam("field") String facet) { - try { - query.remove(ParamConstants.STUDY_PARAM); - query.remove("field"); - - queryOptions.put(QueryOptions.FACET, facet); - - DataResult queryResult = catalogManager.getJobManager().facet(studyStr, query, queryOptions, defaultStats, - token); - return createOkResponse(queryResult); - } catch (Exception e) { - return createErrorResponse(e); - } - } +// @GET +// @Path("/aggregationStats") +// @ApiOperation(value = "Fetch catalog job stats", response = FacetField.class) +// public Response getAggregationStats( +// @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, +// @ApiParam(value = "Tool id") @QueryParam("toolId") String toolId, +// @ApiParam(value = "Tool scope") @QueryParam("toolScope") String toolScope, +// @ApiParam(value = "Tool type") @QueryParam("toolType") String toolType, +// @ApiParam(value = "Tool resource") @QueryParam("toolResource") String toolResource, +// @ApiParam(value = "User id") @QueryParam("userId") String userId, +// @ApiParam(value = "Priority") @QueryParam("priority") String priority, +// @ApiParam(value = "Tags") @QueryParam("tags") String tags, +// @ApiParam(value = "Executor id") @QueryParam("executorId") String executorId, +// @ApiParam(value = "Executor framework") @QueryParam("executorFramework") String executorFramework, +// @ApiParam(value = "Creation year") @QueryParam("creationYear") String creationYear, +// @ApiParam(value = "Creation month (JANUARY, FEBRUARY...)") @QueryParam("creationMonth") String creationMonth, +// @ApiParam(value = "Creation day") @QueryParam("creationDay") String creationDay, +// @ApiParam(value = "Creation day of week (MONDAY, TUESDAY...)") @QueryParam("creationDayOfWeek") String creationDayOfWeek, +// @ApiParam(value = "Status") @QueryParam("status") String status, +// @ApiParam(value = "Release") @QueryParam("release") String release, +// @ApiParam(value = "Calculate default stats", defaultValue = "false") @QueryParam("default") boolean defaultStats, +// @ApiParam(value = "List of fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: studies>>biotype;type;numSamples[0..10]:1") @QueryParam("field") String facet) { +// try { +// query.remove(ParamConstants.STUDY_PARAM); +// query.remove("field"); +// +// queryOptions.put(QueryOptions.FACET, facet); +// +// DataResult queryResult = catalogManager.getJobManager().facet(studyStr, query, queryOptions, defaultStats, +// token); +// return createOkResponse(queryResult); +// } catch (Exception e) { +// return createErrorResponse(e); +// } +// } } \ No newline at end of file diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/MetaWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/MetaWSServer.java index c416d0474b8..6569dd3effe 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/MetaWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/MetaWSServer.java @@ -121,6 +121,7 @@ public Response model(@ApiParam(value = "Model description") @QueryParam("model" @ApiOperation(value = "API", response = List.class) public Response api(@ApiParam(value = "List of categories to get API from") @QueryParam("category") String categoryStr, @QueryParam("summary") boolean summary) { Map> classMap = new LinkedHashMap<>(); + classMap.put("organizations", OrganizationWSServer.class); classMap.put("users", UserWSServer.class); classMap.put("projects", ProjectWSServer.class); classMap.put("studies", StudyWSServer.class); diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/OpenCGAWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/OpenCGAWSServer.java index 761c0a3bd52..3dae031c245 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/OpenCGAWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/OpenCGAWSServer.java @@ -36,7 +36,6 @@ import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.managers.AbstractManager; import org.opencb.opencga.catalog.managers.CatalogManager; -import org.opencb.opencga.catalog.migration.MigrationRun; import org.opencb.opencga.catalog.migration.MigrationSummary; import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.ParamUtils; @@ -45,8 +44,10 @@ import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.config.storage.StorageConfiguration; import org.opencb.opencga.core.exceptions.VersionException; +import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.job.Job; +import org.opencb.opencga.core.models.migration.MigrationRun; import org.opencb.opencga.core.models.study.Study; import org.opencb.opencga.core.response.FederationNode; import org.opencb.opencga.core.response.OpenCGAResult; @@ -263,8 +264,8 @@ static void init(String opencgaHomeStr) { } logger.info("| OpenCGA REST successfully started!"); - logger.info("| - Version " + GitRepositoryState.getInstance().getBuildVersion()); - logger.info("| - Git version: " + GitRepositoryState.getInstance().getBranch() + " " + GitRepositoryState.getInstance().getCommitId()); + logger.info("| - Version {}", GitRepositoryState.getInstance().getBuildVersion()); + logger.info("| - Git version: {} {}", GitRepositoryState.getInstance().getBranch(), GitRepositoryState.getInstance().getCommitId()); logger.info("========================================================================\n"); } @@ -300,12 +301,15 @@ private static void initOpenCGAObjects() { healthCheckMonitor = new OpenCGAHealthCheckMonitor(configuration, catalogManager, storageEngineFactory, variantManager); healthCheckMonitor.asyncUpdate(); - MigrationSummary migrationSummary = catalogManager.getMigrationManager().getMigrationSummary(); - if (migrationSummary.getMigrationsToBeApplied() > 0) { - logger.info("| * Pending migrations: {}", migrationSummary.getMigrationsToBeApplied()); - for (Map.Entry entry : migrationSummary.getStatusCount().entrySet()) { - if (entry.getKey().toBeApplied() && entry.getValue() > 0) { - logger.info("| * {}: {}", entry.getKey(), entry.getValue()); + Map migrationSummaryMap = catalogManager.getMigrationManager().getMigrationSummary(); + for (Map.Entry entry : migrationSummaryMap.entrySet()) { + if (entry.getValue().getMigrationsToBeApplied() > 0) { + logger.info("| * Pending migrations for organization {}: {}", entry.getKey(), + entry.getValue().getMigrationsToBeApplied()); + for (Map.Entry entry2 : entry.getValue().getStatusCount().entrySet()) { + if (entry2.getKey().toBeApplied() && entry2.getValue() > 0) { + logger.info("| * {}: {}", entry2.getKey(), entry2.getValue()); + } } } } @@ -848,43 +852,46 @@ public Response run(Callable> c) { } } - public Response submitJob(String toolId, String project, String study, Map paramsMap, - String jobName, String jobDescription, String jobDependsOne, String jobTags) { - return run(() -> submitJobRaw(toolId, project, study, paramsMap, jobName, jobDescription, jobDependsOne, jobTags)); + public Response submitJob(String toolId, String project, String study, Map paramsMap, String jobName, + String jobDescription, String jobDependsOne, String jobTags, String jobScheduledStartTime, String jobPriority, + Boolean dryRun) { + return run(() -> submitJobRaw(toolId, project, study, paramsMap, jobName, jobDescription, jobDependsOne, jobTags, jobScheduledStartTime, jobPriority, dryRun)); } public Response submitJob(String toolId, String study, ToolParams bodyParams, String jobId, String jobDescription, - String jobDependsOnStr, String jobTagsStr) { - return submitJob(toolId, null, study, bodyParams, jobId, jobDescription, jobDependsOnStr, jobTagsStr); + String jobDependsOnStr, String jobTagsStr, String jobScheduledStartTime, String jobPriority, Boolean dryRun) { + return submitJob(toolId, null, study, bodyParams, jobId, jobDescription, jobDependsOnStr, jobTagsStr, jobScheduledStartTime, jobPriority, dryRun); } public Response submitJobAdmin(String toolId, ToolParams bodyParams, String jobId, String jobDescription, - String jobDependsOnStr, String jobTagsStr) { + String jobDependsOnStr, String jobTagsStr, String jobScheduledStartTime, String jobPriority, + Boolean dryRun) { return run(() -> { - if (!catalogManager.getUserManager().getUserId(token).equals(ParamConstants.OPENCGA_USER_ID)) { - throw new CatalogAuthenticationException("Only user '" + ParamConstants.OPENCGA_USER_ID + "' can run this operation!"); - } - return submitJobRaw(toolId, null, ADMIN_STUDY_FQN, bodyParams, jobId, jobDescription, jobDependsOnStr, jobTagsStr); + JwtPayload jwtPayload = catalogManager.getUserManager().validateToken(token); + catalogManager.getAuthorizationManager().checkIsOpencgaAdministrator(jwtPayload, "submit job from tool '" + toolId + "'"); + return submitJobRaw(toolId, null, ADMIN_STUDY_FQN, bodyParams, jobId, jobDescription, jobDependsOnStr, jobTagsStr, jobScheduledStartTime, jobPriority, dryRun); }); } public Response submitJob(String toolId, String project, String study, ToolParams bodyParams, String jobId, String jobDescription, - String jobDependsOnStr, String jobTagsStr) { - return run(() -> submitJobRaw(toolId, project, study, bodyParams, jobId, jobDescription, jobDependsOnStr, jobTagsStr)); + String jobDependsOnStr, String jobTagsStr, String jobScheduledStartTime, String jobPriority, Boolean dryRun) { + return run(() -> submitJobRaw(toolId, project, study, bodyParams, jobId, jobDescription, jobDependsOnStr, jobTagsStr, jobScheduledStartTime, jobPriority, dryRun)); } - protected DataResult submitJobRaw(String toolId, String project, String study, ToolParams bodyParams, - String jobId, String jobDescription, String jobDependsOnStr, String jobTagsStr) + protected DataResult submitJobRaw(String toolId, String project, String study, ToolParams bodyParams, String jobId, + String jobDescription, String jobDependsOnStr, String jobTagsStr, String jobScheduledStartTime, + String jobPriority, Boolean dryRun) throws CatalogException { Map paramsMap = bodyParams.toParams(); if (StringUtils.isNotEmpty(study)) { paramsMap.putIfAbsent(ParamConstants.STUDY_PARAM, study); } - return submitJobRaw(toolId, project, study, paramsMap, jobId, jobDescription, jobDependsOnStr, jobTagsStr); + return submitJobRaw(toolId, project, study, paramsMap, jobId, jobDescription, jobDependsOnStr, jobTagsStr, jobScheduledStartTime, jobPriority, dryRun); } - protected DataResult submitJobRaw(String toolId, String project, String study, Map paramsMap, - String jobId, String jobDescription, String jobDependsOnStr, String jobTagsStr) + protected DataResult submitJobRaw(String toolId, String project, String study, Map paramsMap, String jobId, + String jobDescription, String jobDependsOnStr, String jobTagsStr, String jobScheduledStartTime, + String jobPriority, Boolean dryRun) throws CatalogException { if (StringUtils.isNotEmpty(project) && StringUtils.isEmpty(study)) { @@ -915,8 +922,13 @@ protected DataResult submitJobRaw(String toolId, String project, String stu } else { jobDependsOn = Collections.emptyList(); } + Enums.Priority priority = Enums.Priority.MEDIUM; + if (!StringUtils.isEmpty(jobPriority)) { + priority = Enums.Priority.getPriority(jobPriority.toUpperCase()); + } return catalogManager.getJobManager() - .submit(study, toolId, Enums.Priority.MEDIUM, paramsMap, jobId, jobDescription, jobDependsOn, jobTags, token); + .submit(study, toolId, priority, paramsMap, jobId, jobDescription, jobDependsOn, jobTags, null, + jobScheduledStartTime, dryRun, token); } public Response createPendingResponse() { diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/OrganizationWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/OrganizationWSServer.java new file mode 100644 index 00000000000..73079cf2704 --- /dev/null +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/OrganizationWSServer.java @@ -0,0 +1,283 @@ +/* + * Copyright 2015-2020 OpenCB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opencb.opencga.server.rest; + +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.catalog.db.api.NoteDBAdaptor; +import org.opencb.opencga.catalog.db.api.OrganizationDBAdaptor; +import org.opencb.opencga.catalog.utils.Constants; +import org.opencb.opencga.catalog.utils.ParamUtils; +import org.opencb.opencga.core.api.FieldConstants; +import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.exceptions.VersionException; +import org.opencb.opencga.core.models.notes.Note; +import org.opencb.opencga.core.models.notes.NoteCreateParams; +import org.opencb.opencga.core.models.notes.NoteUpdateParams; +import org.opencb.opencga.core.models.organizations.Organization; +import org.opencb.opencga.core.models.organizations.OrganizationConfiguration; +import org.opencb.opencga.core.models.organizations.OrganizationCreateParams; +import org.opencb.opencga.core.models.organizations.OrganizationUpdateParams; +import org.opencb.opencga.core.models.user.OrganizationUserUpdateParams; +import org.opencb.opencga.core.models.user.User; +import org.opencb.opencga.core.models.user.UserStatusUpdateParams; +import org.opencb.opencga.core.response.OpenCGAResult; +import org.opencb.opencga.core.tools.annotations.*; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.*; +import javax.ws.rs.core.*; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +@Path("/{apiVersion}/organizations") +@Produces(MediaType.APPLICATION_JSON) +@Api(value = "Organizations", description = "Methods for working with 'organizations' endpoint") +public class OrganizationWSServer extends OpenCGAWSServer { + + public OrganizationWSServer(@Context UriInfo uriInfo, @Context HttpServletRequest httpServletRequest, @Context HttpHeaders httpHeaders) throws IOException, VersionException { + super(uriInfo, httpServletRequest, httpHeaders); + } + + @GET + @Path("/{organization}/info") + @ApiOperation(value = "Return the organization information", response = Organization.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = QueryOptions.INCLUDE, value = ParamConstants.INCLUDE_DESCRIPTION, + dataType = "string", paramType = "query"), + @ApiImplicitParam(name = QueryOptions.EXCLUDE, value = ParamConstants.EXCLUDE_DESCRIPTION, + dataType = "string", paramType = "query"), + }) + public Response getInfo( + @ApiParam(value = ParamConstants.ORGANIZATION_DESCRIPTION, required = true) @PathParam(ParamConstants.ORGANIZATION) String organizationId + ) { + try { + OpenCGAResult result = catalogManager.getOrganizationManager().get(organizationId, queryOptions, token); + return createOkResponse(result); + } catch (Exception e) { + return createErrorResponse(e); + } + } + + @POST + @Path("/{organization}/update") + @Consumes(MediaType.APPLICATION_JSON) + @ApiOperation(value = "Update some organization attributes", response = Organization.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = QueryOptions.INCLUDE, value = ParamConstants.INCLUDE_DESCRIPTION, dataType = "string", paramType = "query"), + @ApiImplicitParam(name = QueryOptions.EXCLUDE, value = ParamConstants.EXCLUDE_DESCRIPTION, dataType = "string", paramType = "query") + }) + public Response update( + @ApiParam(value = ParamConstants.ORGANIZATION_DESCRIPTION, required = true) @PathParam(ParamConstants.ORGANIZATION) String organizationId, + @ApiParam(value = ParamConstants.INCLUDE_RESULT_DESCRIPTION, defaultValue = "false") @QueryParam(ParamConstants.INCLUDE_RESULT_PARAM) boolean includeResult, + @ApiParam(value = "Action to be performed if the array of admins is being updated.", allowableValues = "ADD,REMOVE", defaultValue = "ADD") + @QueryParam("adminsAction") ParamUtils.AddRemoveAction adminsAction, + @ApiParam(value = "JSON containing the params to be updated.", required = true) OrganizationUpdateParams parameters) { + try { + if (adminsAction == null) { + adminsAction = ParamUtils.AddRemoveAction.ADD; + } + Map actionMap = new HashMap<>(); + actionMap.put(OrganizationDBAdaptor.QueryParams.ADMINS.key(), adminsAction); + queryOptions.put(Constants.ACTIONS, actionMap); + OpenCGAResult result = catalogManager.getOrganizationManager().update(organizationId, parameters, queryOptions, token); + return createOkResponse(result); + } catch (Exception e) { + return createErrorResponse(e); + } + } + + @POST + @Path("/{organization}/configuration/update") + @Consumes(MediaType.APPLICATION_JSON) + @ApiOperation(value = "Update the Organization configuration attributes", response = OrganizationConfiguration.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = QueryOptions.INCLUDE, value = ParamConstants.INCLUDE_DESCRIPTION, dataType = "string", paramType = "query"), + @ApiImplicitParam(name = QueryOptions.EXCLUDE, value = ParamConstants.EXCLUDE_DESCRIPTION, dataType = "string", paramType = "query") + }) + public Response updateConfiguration( + @ApiParam(value = ParamConstants.ORGANIZATION_DESCRIPTION, required = true) @PathParam(ParamConstants.ORGANIZATION) String organizationId, + @ApiParam(value = ParamConstants.INCLUDE_RESULT_DESCRIPTION, defaultValue = "false") @QueryParam(ParamConstants.INCLUDE_RESULT_PARAM) boolean includeResult, + @ApiParam(value = "Action to be performed if the array of authenticationOrigins is being updated.", + allowableValues = "ADD,REMOVE,SET,REPLACE", defaultValue = "ADD") @QueryParam("authenticationOriginsAction") ParamUtils.UpdateAction authOriginsAction, + @ApiParam(value = "JSON containing the params to be updated.", required = true) OrganizationConfiguration parameters) { + try { + if (authOriginsAction == null) { + authOriginsAction = ParamUtils.UpdateAction.ADD; + } + Map actionMap = new HashMap<>(); + actionMap.put(OrganizationDBAdaptor.AUTH_ORIGINS_FIELD, authOriginsAction); + queryOptions.put(Constants.ACTIONS, actionMap); + OpenCGAResult result = catalogManager.getOrganizationManager().updateConfiguration(organizationId, parameters, queryOptions, token); + return createOkResponse(result); + } catch (Exception e) { + return createErrorResponse(e); + } + } + + @POST + @Path("/create") + @Consumes(MediaType.APPLICATION_JSON) + @ApiOperation(value = "Create a new organization", response = Organization.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = QueryOptions.INCLUDE, value = ParamConstants.INCLUDE_DESCRIPTION, dataType = "string", paramType = "query"), + @ApiImplicitParam(name = QueryOptions.EXCLUDE, value = ParamConstants.EXCLUDE_DESCRIPTION, dataType = "string", paramType = "query") + }) + public Response create( + @ApiParam(value = ParamConstants.INCLUDE_RESULT_DESCRIPTION, defaultValue = "false") @QueryParam(ParamConstants.INCLUDE_RESULT_PARAM) boolean includeResult, + @ApiParam(value = "JSON containing the organization to be created.", required = true) OrganizationCreateParams parameters) { + try { + OpenCGAResult result = catalogManager.getOrganizationManager().create(parameters, queryOptions, token); + return createOkResponse(result); + } catch (Exception e) { + return createErrorResponse(e); + } + } + + @GET + @Path("/notes/search") + @ApiOperation(value = "Search for notes of scope ORGANIZATION", response = Note.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = QueryOptions.INCLUDE, value = ParamConstants.INCLUDE_DESCRIPTION, + dataType = "string", paramType = "query"), + @ApiImplicitParam(name = QueryOptions.EXCLUDE, value = ParamConstants.EXCLUDE_DESCRIPTION, + dataType = "string", paramType = "query"), + }) + public Response noteSearch( + @ApiParam(value = ParamConstants.CREATION_DATE_DESCRIPTION) @QueryParam(ParamConstants.CREATION_DATE_PARAM) String creationDate, + @ApiParam(value = ParamConstants.MODIFICATION_DATE_DESCRIPTION) @QueryParam(ParamConstants.MODIFICATION_DATE_PARAM) String modificationDate, + @ApiParam(value = FieldConstants.NOTES_ID_DESCRIPTION) @QueryParam(FieldConstants.NOTES_ID_PARAM) String noteId, + @ApiParam(value = FieldConstants.NOTES_SCOPE_DESCRIPTION) @QueryParam(FieldConstants.NOTES_SCOPE_PARAM) String scope, + @ApiParam(value = FieldConstants.NOTES_VISIBILITY_DESCRIPTION) @QueryParam(FieldConstants.NOTES_VISIBILITY_PARAM) String visibility, + @ApiParam(value = FieldConstants.GENERIC_UUID_DESCRIPTION) @QueryParam("uuid") String uuid, + @ApiParam(value = FieldConstants.NOTES_USER_ID_DESCRIPTION) @QueryParam(FieldConstants.NOTES_USER_ID_PARAM) String userId, + @ApiParam(value = FieldConstants.NOTES_TAGS_DESCRIPTION) @QueryParam(FieldConstants.NOTES_TAGS_PARAM) String tags, + @ApiParam(value = FieldConstants.GENERIC_VERSION_DESCRIPTION) @QueryParam("version") String version + ) { + try { + OpenCGAResult result = catalogManager.getNotesManager().searchOrganizationNote(query, queryOptions, token); + return createOkResponse(result); + } catch (Exception e) { + return createErrorResponse(e); + } + } + + @POST + @Path("/notes/create") + @Consumes(MediaType.APPLICATION_JSON) + @ApiOperation(value = "Create a new note", response = Note.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = QueryOptions.INCLUDE, value = ParamConstants.INCLUDE_DESCRIPTION, dataType = "string", paramType = "query"), + @ApiImplicitParam(name = QueryOptions.EXCLUDE, value = ParamConstants.EXCLUDE_DESCRIPTION, dataType = "string", paramType = "query") + }) + public Response createNote( + @ApiParam(value = ParamConstants.INCLUDE_RESULT_DESCRIPTION, defaultValue = "false") @QueryParam(ParamConstants.INCLUDE_RESULT_PARAM) boolean includeResult, + @ApiParam(value = "JSON containing the Note to be added.", required = true) NoteCreateParams parameters) { + try { + OpenCGAResult result = catalogManager.getNotesManager().createOrganizationNote(parameters, queryOptions, token); + return createOkResponse(result); + } catch (Exception e) { + return createErrorResponse(e); + } + } + + @POST + @Path("/notes/{id}/update") + @Consumes(MediaType.APPLICATION_JSON) + @ApiOperation(value = "Update a note", response = Note.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = QueryOptions.INCLUDE, value = ParamConstants.INCLUDE_DESCRIPTION, dataType = "string", paramType = "query"), + @ApiImplicitParam(name = QueryOptions.EXCLUDE, value = ParamConstants.EXCLUDE_DESCRIPTION, dataType = "string", paramType = "query") + }) + public Response updateNote( + @ApiParam(value = FieldConstants.NOTES_ID_DESCRIPTION) @PathParam(FieldConstants.NOTES_ID_PARAM) String noteId, + @ApiParam(value = "Action to be performed if the array of tags is being updated.", allowableValues = "ADD,REMOVE,SET", defaultValue = "ADD") + @QueryParam("tagsAction") ParamUtils.BasicUpdateAction tagsAction, + @ApiParam(value = ParamConstants.INCLUDE_RESULT_DESCRIPTION, defaultValue = "false") @QueryParam(ParamConstants.INCLUDE_RESULT_PARAM) boolean includeResult, + @ApiParam(value = "JSON containing the Note fields to be updated.", required = true) NoteUpdateParams parameters) { + try { + if (tagsAction == null) { + tagsAction = ParamUtils.BasicUpdateAction.ADD; + } + Map actionMap = new HashMap<>(); + actionMap.put(NoteDBAdaptor.QueryParams.TAGS.key(), tagsAction); + queryOptions.put(Constants.ACTIONS, actionMap); + OpenCGAResult result = catalogManager.getNotesManager().updateOrganizationNote(noteId, parameters, queryOptions, token); + return createOkResponse(result); + } catch (Exception e) { + return createErrorResponse(e); + } + } + + @DELETE + @Path("/notes/{id}/delete") + @ApiOperation(value = "Delete note", response = Note.class) + public Response deleteNote( + @ApiParam(value = FieldConstants.NOTES_ID_DESCRIPTION) @PathParam(FieldConstants.NOTES_ID_PARAM) String noteId, + @ApiParam(value = ParamConstants.INCLUDE_RESULT_DESCRIPTION, defaultValue = "false") @QueryParam(ParamConstants.INCLUDE_RESULT_PARAM) boolean includeResult) { + try { + OpenCGAResult result = catalogManager.getNotesManager().deleteOrganizationNote(noteId, queryOptions, token); + return createOkResponse(result); + } catch (Exception e) { + return createErrorResponse(e); + } + } + + @POST + @Path("/user/{user}/update") + @Consumes(MediaType.APPLICATION_JSON) + @ApiOperation(value = "Update the user information", response = User.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = QueryOptions.INCLUDE, value = ParamConstants.INCLUDE_DESCRIPTION, dataType = "string", paramType = "query"), + @ApiImplicitParam(name = QueryOptions.EXCLUDE, value = ParamConstants.EXCLUDE_DESCRIPTION, dataType = "string", paramType = "query") + }) + public Response updateUserInformation( + @ApiParam(value = ParamConstants.USER_DESCRIPTION, required = true) @PathParam("user") String userId, + @ApiParam(value = ParamConstants.ORGANIZATION_DESCRIPTION) @QueryParam(ParamConstants.ORGANIZATION) String organizationId, + @ApiParam(value = ParamConstants.INCLUDE_RESULT_DESCRIPTION, defaultValue = "false") @QueryParam(ParamConstants.INCLUDE_RESULT_PARAM) boolean includeResult, + @ApiParam(value = "JSON containing the User fields to be updated.", required = true) OrganizationUserUpdateParams parameters) { + try { + OpenCGAResult result = catalogManager.getOrganizationManager().updateUser(organizationId, userId, parameters, queryOptions, token); + return createOkResponse(result); + } catch (Exception e) { + return createErrorResponse(e); + } + } + + @POST + @Path("/user/{user}/status/update") + @Consumes(MediaType.APPLICATION_JSON) + @ApiOperation(value = "Update the user status", response = User.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = QueryOptions.INCLUDE, value = ParamConstants.INCLUDE_DESCRIPTION, dataType = "string", paramType = "query"), + @ApiImplicitParam(name = QueryOptions.EXCLUDE, value = ParamConstants.EXCLUDE_DESCRIPTION, dataType = "string", paramType = "query") + }) + public Response updateUserStatus( + @ApiParam(value = ParamConstants.USER_DESCRIPTION, required = true) @PathParam("user") String userId, + @ApiParam(value = ParamConstants.ORGANIZATION_DESCRIPTION) @QueryParam(ParamConstants.ORGANIZATION) String organizationId, + @ApiParam(value = ParamConstants.INCLUDE_RESULT_DESCRIPTION, defaultValue = "false") @QueryParam(ParamConstants.INCLUDE_RESULT_PARAM) boolean includeResult, + @ApiParam(value = "JSON containing the User fields to be updated.", required = true) UserStatusUpdateParams parameters) { + try { + OpenCGAResult result = catalogManager.getUserManager().changeStatus(organizationId, userId, parameters.getStatus(), queryOptions, token); + return createOkResponse(result); + } catch (Exception e) { + return createErrorResponse(e); + } + } + + +} \ No newline at end of file diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/PanelWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/PanelWSServer.java index 2ec062bb84f..7d36dec4386 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/PanelWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/PanelWSServer.java @@ -82,8 +82,11 @@ public Response importPanel( @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(name = "body", value = "Panel parameters") PanelImportParams params) { - return submitJob(PanelImportTask.ID, studyStr, params, jobId, jobDescription, dependsOn, jobTags); + return submitJob(PanelImportTask.ID, studyStr, params, jobId, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } // @POST diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/ProjectWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/ProjectWSServer.java index 2d8a7d24c7e..6c345efe171 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/ProjectWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/ProjectWSServer.java @@ -18,7 +18,10 @@ import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; -import org.opencb.commons.datastore.core.*; +import org.opencb.commons.datastore.core.DataResult; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.catalog.db.api.ProjectDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.utils.ParamUtils; @@ -32,13 +35,10 @@ import org.opencb.opencga.core.tools.annotations.*; import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.QueryParam; import javax.ws.rs.*; import javax.ws.rs.core.*; import java.io.IOException; -import java.util.HashMap; import java.util.List; -import java.util.Map; import static org.opencb.opencga.core.common.JacksonUtils.getUpdateObjectMapper; @@ -107,7 +107,7 @@ public Response info( @ApiImplicitParam(name = QueryOptions.SKIP, value = ParamConstants.SKIP_DESCRIPTION, dataType = "integer", paramType = "query") }) public Response searchProjects( - @ApiParam(value = "Owner of the project") @QueryParam("owner") String owner, + @ApiParam(value = ParamConstants.ORGANIZATION_DESCRIPTION) @QueryParam(ParamConstants.ORGANIZATION) String organizationId, @ApiParam(value = ParamConstants.PROJECT_DESCRIPTION) @QueryParam("id") String id, @ApiParam(value = "Project name") @QueryParam("name") String name, @ApiParam(value = "Project fqn") @QueryParam("fqn") String fqn, @@ -121,51 +121,46 @@ public Response searchProjects( @ApiParam(value = ParamConstants.INTERNAL_STATUS_DESCRIPTION) @QueryParam(ParamConstants.INTERNAL_STATUS_PARAM) String internalStatus, @ApiParam(value = "Attributes") @QueryParam("attributes") String attributes) { try { - if (StringUtils.isNotEmpty(owner)) { - query.remove("owner"); - query.put(ProjectDBAdaptor.QueryParams.USER_ID.key(), owner); - } - - DataResult queryResult = catalogManager.getProjectManager().search(query, queryOptions, token); + DataResult queryResult = catalogManager.getProjectManager().search(organizationId, query, queryOptions, token); return createOkResponse(queryResult); } catch (Exception e) { return createErrorResponse(e); } } - @GET - @Path("/{projects}/aggregationStats") - @ApiOperation(value = "Fetch catalog project stats", response = FacetField.class) - public Response getAggregationStats( - @ApiParam(value = ParamConstants.PROJECTS_DESCRIPTION, required = true) @PathParam("projects") String projects, - @ApiParam(value = "Calculate default stats", defaultValue = "true") @QueryParam("default") Boolean defaultStats, - @ApiParam(value = "List of file fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: " - + "studies>>biotype;type") @QueryParam("fileFields") String fileFields, - @ApiParam(value = "List of individual fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: " - + "studies>>biotype;type") @QueryParam("individualFields") String individualFields, - @ApiParam(value = "List of family fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: " - + "studies>>biotype;type") @QueryParam("familyFields") String familyFields, - @ApiParam(value = "List of sample fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: " - + "studies>>biotype;type") @QueryParam("sampleFields") String sampleFields, - @ApiParam(value = "List of cohort fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: " - + "studies>>biotype;type") @QueryParam("cohortFields") String cohortFields, - @ApiParam(value = "List of job fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: " - + "studies>>biotype;type") @QueryParam("jobFields") String jobFields) { - try { - if (defaultStats == null) { - defaultStats = true; - } - List idList = getIdList(projects); - Map result = new HashMap<>(); - for (String project : idList) { - result.put(project, catalogManager.getProjectManager().facet(project, fileFields, sampleFields, individualFields, - cohortFields, familyFields, jobFields, defaultStats, token)); - } - return createOkResponse(result); - } catch (Exception e) { - return createErrorResponse(e); - } - } +// @GET +// @Path("/{projects}/aggregationStats") +// @ApiOperation(value = "Fetch catalog project stats", response = FacetField.class) +// public Response getAggregationStats( +// @ApiParam(value = ParamConstants.PROJECTS_DESCRIPTION, required = true) @PathParam("projects") String projects, +// @ApiParam(value = "Calculate default stats", defaultValue = "true") @QueryParam("default") Boolean defaultStats, +// @ApiParam(value = "List of file fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: " +// + "studies>>biotype;type") @QueryParam("fileFields") String fileFields, +// @ApiParam(value = "List of individual fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: " +// + "studies>>biotype;type") @QueryParam("individualFields") String individualFields, +// @ApiParam(value = "List of family fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: " +// + "studies>>biotype;type") @QueryParam("familyFields") String familyFields, +// @ApiParam(value = "List of sample fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: " +// + "studies>>biotype;type") @QueryParam("sampleFields") String sampleFields, +// @ApiParam(value = "List of cohort fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: " +// + "studies>>biotype;type") @QueryParam("cohortFields") String cohortFields, +// @ApiParam(value = "List of job fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: " +// + "studies>>biotype;type") @QueryParam("jobFields") String jobFields) { +// try { +// if (defaultStats == null) { +// defaultStats = true; +// } +// List idList = getIdList(projects); +// Map result = new HashMap<>(); +// for (String project : idList) { +// result.put(project, catalogManager.getProjectManager().facet(project, fileFields, sampleFields, individualFields, +// cohortFields, familyFields, jobFields, defaultStats, token)); +// } +// return createOkResponse(result); +// } catch (Exception e) { +// return createErrorResponse(e); +// } +// } @POST @Path("/{project}/incRelease") diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/SampleWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/SampleWSServer.java index 5ab2cc91491..0a329811db2 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/SampleWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/SampleWSServer.java @@ -19,7 +19,6 @@ import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.opencb.commons.datastore.core.DataResult; -import org.opencb.commons.datastore.core.FacetField; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.analysis.sample.SampleTsvAnnotationLoader; @@ -41,7 +40,10 @@ import javax.ws.rs.*; import javax.ws.rs.core.*; import java.io.IOException; -import java.util.*; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; /** * Created by jacobo on 15/12/14. @@ -123,7 +125,7 @@ public Response loadSamples( try { File pedigreeFile = catalogManager.getFileManager().get(studyStr, fileStr, null, token).first(); CatalogSampleAnnotationsLoader loader = new CatalogSampleAnnotationsLoader(catalogManager); - DataResult sampleQueryResult = loader.loadSampleAnnotations(pedigreeFile, variableSet, token); + DataResult sampleQueryResult = loader.loadSampleAnnotations(studyStr, pedigreeFile, variableSet, token); return createOkResponse(sampleQueryResult); } catch (Exception e) { return createErrorResponse(e); @@ -442,39 +444,39 @@ public Response updateAcl( } } - @GET - @Path("/aggregationStats") - @ApiOperation(value = "Fetch catalog sample stats", response = FacetField.class) - public Response getAggregationStats( - @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, - @ApiParam(value = "Source") @QueryParam("source") String source, - @ApiParam(value = "Creation year") @QueryParam("creationYear") String creationYear, - @ApiParam(value = "Creation month (JANUARY, FEBRUARY...)") @QueryParam("creationMonth") String creationMonth, - @ApiParam(value = "Creation day") @QueryParam("creationDay") String creationDay, - @ApiParam(value = "Creation day of week (MONDAY, TUESDAY...)") @QueryParam("creationDayOfWeek") String creationDayOfWeek, - @ApiParam(value = "Status") @QueryParam("status") String status, - @ApiParam(value = "Type") @QueryParam("type") String type, - @ApiParam(value = "Phenotypes") @QueryParam("phenotypes") String phenotypes, - @ApiParam(value = "Release") @QueryParam("release") String release, - @ApiParam(value = "Version") @QueryParam("version") String version, - @ApiParam(value = "Somatic") @QueryParam("somatic") Boolean somatic, - @ApiParam(value = ParamConstants.ANNOTATION_DESCRIPTION) @QueryParam("annotation") String annotation, - - @ApiParam(value = "Calculate default stats", defaultValue = "false") @QueryParam("default") boolean defaultStats, - - @ApiParam(value = "List of fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: " + - "studies>>biotype;type;numSamples[0..10]:1") @QueryParam("field") String facet) { - try { - query.remove(ParamConstants.STUDY_PARAM); - query.remove("field"); - - queryOptions.put(QueryOptions.FACET, facet); - - DataResult queryResult = catalogManager.getSampleManager().facet(studyStr, query, queryOptions, defaultStats, - token); - return createOkResponse(queryResult); - } catch (Exception e) { - return createErrorResponse(e); - } - } +// @GET +// @Path("/aggregationStats") +// @ApiOperation(value = "Fetch catalog sample stats", response = FacetField.class) +// public Response getAggregationStats( +// @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, +// @ApiParam(value = "Source") @QueryParam("source") String source, +// @ApiParam(value = "Creation year") @QueryParam("creationYear") String creationYear, +// @ApiParam(value = "Creation month (JANUARY, FEBRUARY...)") @QueryParam("creationMonth") String creationMonth, +// @ApiParam(value = "Creation day") @QueryParam("creationDay") String creationDay, +// @ApiParam(value = "Creation day of week (MONDAY, TUESDAY...)") @QueryParam("creationDayOfWeek") String creationDayOfWeek, +// @ApiParam(value = "Status") @QueryParam("status") String status, +// @ApiParam(value = "Type") @QueryParam("type") String type, +// @ApiParam(value = "Phenotypes") @QueryParam("phenotypes") String phenotypes, +// @ApiParam(value = "Release") @QueryParam("release") String release, +// @ApiParam(value = "Version") @QueryParam("version") String version, +// @ApiParam(value = "Somatic") @QueryParam("somatic") Boolean somatic, +// @ApiParam(value = ParamConstants.ANNOTATION_DESCRIPTION) @QueryParam("annotation") String annotation, +// +// @ApiParam(value = "Calculate default stats", defaultValue = "false") @QueryParam("default") boolean defaultStats, +// +// @ApiParam(value = "List of fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: " + +// "studies>>biotype;type;numSamples[0..10]:1") @QueryParam("field") String facet) { +// try { +// query.remove(ParamConstants.STUDY_PARAM); +// query.remove("field"); +// +// queryOptions.put(QueryOptions.FACET, facet); +// +// DataResult queryResult = catalogManager.getSampleManager().facet(studyStr, query, queryOptions, defaultStats, +// token); +// return createOkResponse(queryResult); +// } catch (Exception e) { +// return createErrorResponse(e); +// } +// } } diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/StudyWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/StudyWSServer.java index 0db3fae08ad..8492d30c490 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/StudyWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/StudyWSServer.java @@ -21,19 +21,23 @@ import org.glassfish.jersey.media.multipart.FormDataContentDisposition; import org.glassfish.jersey.media.multipart.FormDataParam; import org.opencb.commons.datastore.core.DataResult; -import org.opencb.commons.datastore.core.FacetField; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.analysis.templates.TemplateRunner; +import org.opencb.opencga.catalog.db.api.NoteDBAdaptor; import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; import org.opencb.opencga.catalog.managers.StudyManager; import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.ParamUtils; +import org.opencb.opencga.core.api.FieldConstants; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.exceptions.VersionException; import org.opencb.opencga.core.models.AclEntryList; import org.opencb.opencga.core.models.audit.AuditRecord; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.job.Job; +import org.opencb.opencga.core.models.notes.Note; +import org.opencb.opencga.core.models.notes.NoteCreateParams; +import org.opencb.opencga.core.models.notes.NoteUpdateParams; import org.opencb.opencga.core.models.study.*; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.tools.annotations.*; @@ -328,8 +332,7 @@ public Response updateAcl( try { ObjectUtils.defaultIfNull(params, new StudyAclUpdateParams()); StudyAclParams aclParams = new StudyAclParams(params.getPermissions(), params.getTemplate()); - List idList = getIdList(params.getStudy(), false); - return createOkResponse(studyManager.updateAcl(idList, memberId, aclParams, action, token)); + return createOkResponse(studyManager.updateAcl(params.getStudy(), memberId, aclParams, action, token)); } catch (Exception e) { return createErrorResponse(e); } @@ -365,40 +368,40 @@ public Response getVariableSets( } } - @GET - @Path("/{studies}/aggregationStats") - @ApiOperation(value = "Fetch catalog study stats", response = FacetField.class) - public Response getAggregationStats( - @ApiParam(value = "Comma separated list of studies [[user@]project:]study up to a maximum of 100", required = true) - @PathParam(ParamConstants.STUDIES_PARAM) String studies, - @ApiParam(value = "Calculate default stats", defaultValue = "true") @QueryParam("default") Boolean defaultStats, - @ApiParam(value = "List of file fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: " - + "studies>>biotype;type") @QueryParam("fileFields") String fileFields, - @ApiParam(value = "List of individual fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: " - + "studies>>biotype;type") @QueryParam("individualFields") String individualFields, - @ApiParam(value = "List of family fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: " - + "studies>>biotype;type") @QueryParam("familyFields") String familyFields, - @ApiParam(value = "List of sample fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: " - + "studies>>biotype;type") @QueryParam("sampleFields") String sampleFields, - @ApiParam(value = "List of cohort fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: " - + "studies>>biotype;type") @QueryParam("cohortFields") String cohortFields, - @ApiParam(value = "List of job fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: " - + "studies>>biotype;type") @QueryParam("jobFields") String jobFields) { - try { - if (defaultStats == null) { - defaultStats = true; - } - List idList = getIdList(studies); - Map result = new HashMap<>(); - for (String study : idList) { - result.put(study, catalogManager.getStudyManager().facet(study, fileFields, sampleFields, individualFields, cohortFields, - familyFields, jobFields, defaultStats, token)); - } - return createOkResponse(result); - } catch (Exception e) { - return createErrorResponse(e); - } - } +// @GET +// @Path("/{studies}/aggregationStats") +// @ApiOperation(value = "Fetch catalog study stats", response = FacetField.class) +// public Response getAggregationStats( +// @ApiParam(value = "Comma separated list of studies [[organization@]project:]study up to a maximum of 100", required = true) +// @PathParam(ParamConstants.STUDIES_PARAM) String studies, +// @ApiParam(value = "Calculate default stats", defaultValue = "true") @QueryParam("default") Boolean defaultStats, +// @ApiParam(value = "List of file fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: " +// + "studies>>biotype;type") @QueryParam("fileFields") String fileFields, +// @ApiParam(value = "List of individual fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: " +// + "studies>>biotype;type") @QueryParam("individualFields") String individualFields, +// @ApiParam(value = "List of family fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: " +// + "studies>>biotype;type") @QueryParam("familyFields") String familyFields, +// @ApiParam(value = "List of sample fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: " +// + "studies>>biotype;type") @QueryParam("sampleFields") String sampleFields, +// @ApiParam(value = "List of cohort fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: " +// + "studies>>biotype;type") @QueryParam("cohortFields") String cohortFields, +// @ApiParam(value = "List of job fields separated by semicolons, e.g.: studies;type. For nested fields use >>, e.g.: " +// + "studies>>biotype;type") @QueryParam("jobFields") String jobFields) { +// try { +// if (defaultStats == null) { +// defaultStats = true; +// } +// List idList = getIdList(studies); +// Map result = new HashMap<>(); +// for (String study : idList) { +// result.put(study, catalogManager.getStudyManager().facet(study, fileFields, sampleFields, individualFields, cohortFields, +// familyFields, jobFields, defaultStats, token)); +// } +// return createOkResponse(result); +// } catch (Exception e) { +// return createErrorResponse(e); +// } +// } @POST @Path("/{study}/variableSets/update") @@ -423,9 +426,8 @@ public Response createOrRemoveVariableSets( fixVariable(variable); } - queryResult = catalogManager.getStudyManager().createVariableSet(studyStr, params.getId(), params.getName(), - params.getUnique(), - params.getConfidential(), params.getDescription(), null, params.getVariables(), params.getEntities(), token); + VariableSet variableSet = params.toVariableSet(); + queryResult = catalogManager.getStudyManager().createVariableSet(studyStr, variableSet, token); } else { boolean force = ParamUtils.AddRemoveForceRemoveAction.FORCE_REMOVE.equals(action); queryResult = catalogManager.getStudyManager().deleteVariableSet(studyStr, params.getId(), force, token); @@ -530,7 +532,7 @@ public Response upload( @Path("/{study}/templates/{templateId}/delete") @ApiOperation(value = "Delete template", response = Boolean.class) public Response delete( - @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, + @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @PathParam(ParamConstants.STUDY_PARAM) String studyStr, @ApiParam(value = "Template id") @PathParam("templateId") String templateId) { try { return createOkResponse(studyManager.deleteTemplate(studyStr, templateId, token)); @@ -548,8 +550,103 @@ public Response executeTemplate( @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = TemplateParams.DESCRIPTION, required = true) TemplateParams params) { - return submitJob(TemplateRunner.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(TemplateRunner.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); + } + + @GET + @Path("/{study}/notes/search") + @ApiOperation(value = "Search for notes of scope STUDY", response = Note.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = QueryOptions.INCLUDE, value = ParamConstants.INCLUDE_DESCRIPTION, + dataType = "string", paramType = "query"), + @ApiImplicitParam(name = QueryOptions.EXCLUDE, value = ParamConstants.EXCLUDE_DESCRIPTION, + dataType = "string", paramType = "query"), + }) + public Response noteSearch( + @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @PathParam(ParamConstants.STUDY_PARAM) String studyStr, + @ApiParam(value = ParamConstants.CREATION_DATE_DESCRIPTION) @QueryParam(ParamConstants.CREATION_DATE_PARAM) String creationDate, + @ApiParam(value = ParamConstants.MODIFICATION_DATE_DESCRIPTION) @QueryParam(ParamConstants.MODIFICATION_DATE_PARAM) String modificationDate, + @ApiParam(value = FieldConstants.NOTES_ID_DESCRIPTION) @QueryParam(FieldConstants.NOTES_ID_PARAM) String noteId, + @ApiParam(value = FieldConstants.GENERIC_UUID_DESCRIPTION) @QueryParam("uuid") String uuid, + @ApiParam(value = FieldConstants.NOTES_USER_ID_DESCRIPTION) @QueryParam(FieldConstants.NOTES_USER_ID_PARAM) String userId, + @ApiParam(value = FieldConstants.NOTES_TAGS_DESCRIPTION) @QueryParam(FieldConstants.NOTES_TAGS_PARAM) String tags, + @ApiParam(value = FieldConstants.NOTES_VISIBILITY_DESCRIPTION) @QueryParam(FieldConstants.NOTES_VISIBILITY_PARAM) String visibility, + @ApiParam(value = FieldConstants.GENERIC_VERSION_DESCRIPTION) @QueryParam("version") String version + ) { + try { + OpenCGAResult result = catalogManager.getNotesManager().searchStudyNote(studyStr, query, queryOptions, token); + return createOkResponse(result); + } catch (Exception e) { + return createErrorResponse(e); + } + } + + @POST + @Path("/{study}/notes/create") + @Consumes(MediaType.APPLICATION_JSON) + @ApiOperation(value = "Create a new note", response = Note.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = QueryOptions.INCLUDE, value = ParamConstants.INCLUDE_DESCRIPTION, dataType = "string", paramType = "query"), + @ApiImplicitParam(name = QueryOptions.EXCLUDE, value = ParamConstants.EXCLUDE_DESCRIPTION, dataType = "string", paramType = "query") + }) + public Response createNote( + @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @PathParam(ParamConstants.STUDY_PARAM) String studyStr, + @ApiParam(value = ParamConstants.INCLUDE_RESULT_DESCRIPTION, defaultValue = "false") @QueryParam(ParamConstants.INCLUDE_RESULT_PARAM) boolean includeResult, + @ApiParam(value = "JSON containing the Note to be added.", required = true) NoteCreateParams parameters) { + try { + OpenCGAResult result = catalogManager.getNotesManager().createStudyNote(studyStr, parameters, queryOptions, token); + return createOkResponse(result); + } catch (Exception e) { + return createErrorResponse(e); + } + } + + @POST + @Path("/{study}/notes/{id}/update") + @Consumes(MediaType.APPLICATION_JSON) + @ApiOperation(value = "Update a note", response = Note.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = QueryOptions.INCLUDE, value = ParamConstants.INCLUDE_DESCRIPTION, dataType = "string", paramType = "query"), + @ApiImplicitParam(name = QueryOptions.EXCLUDE, value = ParamConstants.EXCLUDE_DESCRIPTION, dataType = "string", paramType = "query") + }) + public Response updateNote( + @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @PathParam(ParamConstants.STUDY_PARAM) String studyStr, + @ApiParam(value = FieldConstants.NOTES_ID_DESCRIPTION) @PathParam(FieldConstants.NOTES_ID_PARAM) String noteId, + @ApiParam(value = "Action to be performed if the array of tags is being updated.", allowableValues = "ADD,REMOVE,SET", defaultValue = "ADD") + @QueryParam("tagsAction") ParamUtils.BasicUpdateAction tagsAction, + @ApiParam(value = ParamConstants.INCLUDE_RESULT_DESCRIPTION, defaultValue = "false") @QueryParam(ParamConstants.INCLUDE_RESULT_PARAM) boolean includeResult, + @ApiParam(value = "JSON containing the Note fields to be updated.", required = true) NoteUpdateParams parameters) { + try { + if (tagsAction == null) { + tagsAction = ParamUtils.BasicUpdateAction.ADD; + } + Map actionMap = new HashMap<>(); + actionMap.put(NoteDBAdaptor.QueryParams.TAGS.key(), tagsAction); + queryOptions.put(Constants.ACTIONS, actionMap); + OpenCGAResult result = catalogManager.getNotesManager().updateStudyNote(studyStr, noteId, parameters, queryOptions, token); + return createOkResponse(result); + } catch (Exception e) { + return createErrorResponse(e); + } + } + + @DELETE + @Path("/{study}/notes/{id}/delete") + @ApiOperation(value = "Delete note", response = Note.class) + public Response deleteNote( + @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @PathParam(ParamConstants.STUDY_PARAM) String studyStr, + @ApiParam(value = FieldConstants.NOTES_ID_DESCRIPTION) @PathParam(FieldConstants.NOTES_ID_PARAM) String noteId, + @ApiParam(value = ParamConstants.INCLUDE_RESULT_DESCRIPTION, defaultValue = "false") @QueryParam(ParamConstants.INCLUDE_RESULT_PARAM) boolean includeResult) { + try { + OpenCGAResult result = catalogManager.getNotesManager().deleteStudyNote(studyStr, noteId, queryOptions, token); + return createOkResponse(result); + } catch (Exception e) { + return createErrorResponse(e); + } } private void fixVariable(Variable variable) { diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/UserWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/UserWSServer.java index 8b8850b3ebd..31fcd3a9720 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/UserWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/UserWSServer.java @@ -21,11 +21,10 @@ import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.catalog.db.api.ProjectDBAdaptor; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.exceptions.VersionException; -import org.opencb.opencga.core.models.project.Project; import org.opencb.opencga.core.models.user.*; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.tools.annotations.*; @@ -49,6 +48,51 @@ public UserWSServer(@Context UriInfo uriInfo, @Context HttpServletRequest httpSe super(uriInfo, httpServletRequest, httpHeaders); } + @POST + @Path("/create") + @Consumes(MediaType.APPLICATION_JSON) + @ApiOperation(value = "Create a new user", response = User.class) + public Response create( + @ApiParam(value = "JSON containing the parameters", required = true) UserCreateParams user + ) { + try { + if (!user.checkValidParams()) { + return createErrorResponse(new CatalogException("id, name, email or password not present")); + } + + OpenCGAResult queryResult = catalogManager.getUserManager() + .create(user.getId(), user.getName(), user.getEmail(), user.getPassword(), user.getOrganization(), null, token); + + return createOkResponse(queryResult); + } catch (Exception e) { + return createErrorResponse(e); + } + } + + @GET + @Path("/search") + @ApiOperation(value = "User search method", response = User.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = QueryOptions.INCLUDE, value = ParamConstants.INCLUDE_DESCRIPTION, example = "name,attributes", + dataType = "string", paramType = "query"), + @ApiImplicitParam(name = QueryOptions.EXCLUDE, value = ParamConstants.EXCLUDE_DESCRIPTION, example = "id,status", dataType = + "string", paramType = "query"), + @ApiImplicitParam(name = QueryOptions.LIMIT, value = ParamConstants.LIMIT_DESCRIPTION, dataType = "integer", paramType = + "query"), + @ApiImplicitParam(name = QueryOptions.SKIP, value = ParamConstants.SKIP_DESCRIPTION, dataType = "integer", paramType = "query"), + @ApiImplicitParam(name = QueryOptions.COUNT, value = ParamConstants.COUNT_DESCRIPTION, defaultValue = "false", dataType = + "boolean", paramType = "query") + }) + public Response search( + @ApiParam(value = ParamConstants.ORGANIZATION_DESCRIPTION) @QueryParam(ParamConstants.ORGANIZATION) String organizationId, + @ApiParam(value = ParamConstants.USER_ID_DESCRIPTION) @QueryParam(ParamConstants.USER_ID_PARAM) String userId, + @ApiParam(value = ParamConstants.USER_AUTHENTICATION_ORIGIN_DESCRIPTION) @QueryParam(ParamConstants.USER_AUTHENTICATION_ORIGIN) String authentication) { + return run(() -> { + query.remove(ParamConstants.ORGANIZATION); + return catalogManager.getUserManager().search(organizationId, query, queryOptions, token); + }); + } + @GET @Path("/{users}/info") @ApiOperation(value = "Return the user information including its projects and studies", response = User.class) @@ -58,51 +102,19 @@ public UserWSServer(@Context UriInfo uriInfo, @Context HttpServletRequest httpSe @ApiImplicitParam(name = QueryOptions.EXCLUDE, value = ParamConstants.EXCLUDE_DESCRIPTION, dataType = "string", paramType = "query"), }) - public Response getInfo(@ApiParam(value = ParamConstants.USERS_DESCRIPTION, required = true) @PathParam("users") String userIds) { + public Response getInfo( + @ApiParam(value = ParamConstants.ORGANIZATION_DESCRIPTION) @QueryParam(ParamConstants.ORGANIZATION) String organizationId, + @ApiParam(value = ParamConstants.USERS_DESCRIPTION, required = true) @PathParam("users") String userIds + ) { try { List userList = getIdList(userIds); - OpenCGAResult result = catalogManager.getUserManager().get(userList, queryOptions, token); + OpenCGAResult result = catalogManager.getUserManager().get(organizationId, userList, queryOptions, token); return createOkResponse(result); } catch (Exception e) { return createErrorResponse(e); } } -// @Deprecated -// @POST -// @Path("/{user}/login") -// @Consumes(MediaType.APPLICATION_JSON) -// @ApiOperation(value = "Get identified and gain access to the system", -// notes = "Login method is implemented using JSON Web Tokens that use the standard RFC 7519. The provided tokens are not " + -// "stored by OpenCGA so there is not a logout method anymore. Tokens are provided with an expiration time that, once " + -// "finished, will no longer be valid.\nIf password is provided it will attempt to login the user. If no password is " + -// "provided and a valid token is given, a new token will be provided extending the expiration time.", -// response = AuthenticationResponse.class, hidden = true) -// public Response deprecatedLogin( -// @ApiParam(value = ParamConstants.USER_DESCRIPTION, required = true) @PathParam("user") String userId, -// @ApiParam(value = "JSON containing the parameter 'password'") LoginParams login) { -// try { -// if (login == null) { -// login = new LoginParams(); -// } -// AuthenticationResponse authenticationResponse; -// if (StringUtils.isNotEmpty(login.getPassword())) { -// authenticationResponse = catalogManager.getUserManager().login(userId, login.getPassword()); -// } else if (StringUtils.isNotEmpty(this.token)) { -// authenticationResponse = catalogManager.getUserManager().refreshToken(this.token); -// } else { -// throw new Exception("Neither a password nor a token was provided."); -// } -// -// OpenCGAResult response = new OpenCGAResult<>(0, Collections.emptyList(), 1, -// Collections.singletonList(authenticationResponse), 1); -// -// return createOkResponse(response); -// } catch (Exception e) { -// return createErrorResponse(e); -// } -// } - @POST @Path("/login") @Consumes(MediaType.APPLICATION_JSON) @@ -120,7 +132,7 @@ public Response login( if (StringUtils.isNotEmpty(login.getRefreshToken())) { throw new Exception("Only 'user' and 'password' fields or 'refreshToken' field are allowed at the same time"); } - authenticationResponse = catalogManager.getUserManager().login(login.getUser(), login.getPassword()); + authenticationResponse = catalogManager.getUserManager().login(login.getOrganization(), login.getUser(), login.getPassword()); } else if (StringUtils.isNotEmpty(login.getRefreshToken())) { if (StringUtils.isNotEmpty(login.getPassword()) || StringUtils.isNotEmpty(login.getUser())) { throw new Exception("Only 'user' and 'password' fields or 'refreshToken' field are allowed at the same time"); @@ -139,23 +151,21 @@ public Response login( } } -// @Deprecated -// @POST -// @Path("/{user}/password") -// @Consumes(MediaType.APPLICATION_JSON) -// @ApiOperation(value = "Change the password of a user", -// notes = "Only for local users. Not available for users belonging to external authentication origins.", response = User.class, -// hidden = true) -// public Response deprecatedChangePasswordPost( -// @ApiParam(value = ParamConstants.USER_DESCRIPTION, required = true) @PathParam("user") String userId, -// @ApiParam(value = "JSON containing the change of password parameters", required = true) PasswordChangeParams params) { -// try { -// catalogManager.getUserManager().changePassword(userId, params.getPassword(), params.getNewPassword()); -// return createOkResponse(DataResult.empty()); -// } catch (Exception e) { -// return createErrorResponse(e); -// } -// } + @POST + @Path("/anonymous") + @Consumes(MediaType.APPLICATION_JSON) + @ApiOperation(value = "Get an anonymous token to gain access to the system", response = AuthenticationResponse.class) + public Response anonymous( + @ApiParam(value = ParamConstants.ORGANIZATION_DESCRIPTION, required = true) @QueryParam(ParamConstants.ORGANIZATION) String organizationId) { + try { + AuthenticationResponse authenticationResponse = catalogManager.getUserManager().loginAnonymous(organizationId); + OpenCGAResult response = new OpenCGAResult<>(0, Collections.emptyList(), 1, + Collections.singletonList(authenticationResponse), 1); + return createOkResponse(response); + } catch (Exception e) { + return createErrorResponse(e); + } + } @POST @Path("/password") @@ -165,7 +175,7 @@ public Response login( public Response changePassword( @ApiParam(value = "JSON containing the change of password parameters", required = true) PasswordChangeParams params) { try { - catalogManager.getUserManager().changePassword(params.getUser(), params.getPassword(), params.getNewPassword()); + catalogManager.getUserManager().changePassword(params.getOrganizationId(), params.getUser(), params.getPassword(), params.getNewPassword()); return createOkResponse(DataResult.empty()); } catch (Exception e) { return createErrorResponse(e); @@ -176,7 +186,8 @@ public Response changePassword( @Path("/{user}/password/reset") @ApiOperation(value = "Reset password", hidden = false, notes = "Reset the user's password and send a new random one to the e-mail stored in catalog.", response = User.class) - public Response resetPassword(@ApiParam(value = ParamConstants.USER_DESCRIPTION, required = true) @PathParam("user") String userId) { + public Response resetPassword( + @ApiParam(value = ParamConstants.USER_DESCRIPTION, required = true) @PathParam("user") String userId) { try { OpenCGAResult result = catalogManager.getUserManager().resetPassword(userId, token); return createOkResponse(result, "The new password has been sent to the user's email."); @@ -185,31 +196,6 @@ public Response resetPassword(@ApiParam(value = ParamConstants.USER_DESCRIPTION, } } - @GET - @Path("/{user}/projects") - @ApiOperation(value = "Retrieve the projects of the user", notes = "Retrieve the list of projects and studies belonging to the user" - + " performing the query. This will not fetch shared projects. To get those, please use /projects/search web service.", - response = Project.class) - @ApiImplicitParams({ - @ApiImplicitParam(name = QueryOptions.INCLUDE, value = ParamConstants.INCLUDE_DESCRIPTION, - dataType = "string", paramType = "query"), - @ApiImplicitParam(name = QueryOptions.EXCLUDE, value = ParamConstants.EXCLUDE_DESCRIPTION, - dataType = "string", paramType = "query"), - @ApiImplicitParam(name = QueryOptions.LIMIT, value = ParamConstants.LIMIT_DESCRIPTION, dataType = "integer", paramType = - "query"), - @ApiImplicitParam(name = QueryOptions.SKIP, value = ParamConstants.SKIP_DESCRIPTION, dataType = "integer", paramType = "query") - }) - public Response getAllProjects(@ApiParam(value = ParamConstants.USER_DESCRIPTION, required = true) @PathParam("user") String userId) { - try { - ParamUtils.checkIsSingleID(userId); - query.remove("user"); - query.put(ProjectDBAdaptor.QueryParams.USER_ID.key(), userId); - return createOkResponse(catalogManager.getProjectManager().search(query, queryOptions, token)); - } catch (Exception e) { - return createErrorResponse(e); - } - } - @POST @Path("/{user}/update") @Consumes(MediaType.APPLICATION_JSON) diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/admin/AdminWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/admin/AdminWSServer.java index 835c9632af1..c0345d421f5 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/admin/AdminWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/admin/AdminWSServer.java @@ -16,30 +16,23 @@ package org.opencb.opencga.server.rest.admin; -import org.apache.commons.lang3.StringUtils; import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.utils.ListUtils; -import org.opencb.opencga.analysis.cohort.CohortIndexTask; -import org.opencb.opencga.analysis.family.FamilyIndexTask; -import org.opencb.opencga.analysis.file.FileIndexTask; -import org.opencb.opencga.analysis.individual.IndividualIndexTask; -import org.opencb.opencga.analysis.job.JobIndexTask; -import org.opencb.opencga.analysis.sample.SampleIndexTask; import org.opencb.opencga.catalog.db.api.MetaDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.exceptions.VersionException; +import org.opencb.opencga.core.models.Acl; import org.opencb.opencga.core.models.admin.*; import org.opencb.opencga.core.models.common.Enums; -import org.opencb.opencga.core.models.job.Job; import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.study.Group; -import org.opencb.opencga.core.models.user.Account; import org.opencb.opencga.core.models.user.AuthenticationResponse; import org.opencb.opencga.core.models.user.User; +import org.opencb.opencga.core.models.user.UserCreateParams; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.tools.annotations.*; import org.opencb.opencga.server.rest.OpenCGAWSServer; @@ -48,12 +41,9 @@ import javax.ws.rs.*; import javax.ws.rs.core.*; import java.io.IOException; -import java.util.ArrayList; import java.util.Collections; -import java.util.List; import java.util.Map; -import static org.opencb.opencga.core.api.ParamConstants.ADMIN_STUDY_FQN; import static org.opencb.opencga.core.models.admin.UserImportParams.ResourceType.*; @Path("/{apiVersion}/admin") @@ -79,11 +69,11 @@ public AdminWSServer(@Context UriInfo uriInfo, @Context HttpServletRequest httpS @ApiImplicitParam(name = QueryOptions.COUNT, value = ParamConstants.COUNT_DESCRIPTION, defaultValue = "false", dataType = "boolean", paramType = "query") }) public Response userSearch( + @ApiParam(value = ParamConstants.ORGANIZATION_DESCRIPTION) @QueryParam(ParamConstants.ORGANIZATION) String organizationId, @ApiParam(value = ParamConstants.USER_DESCRIPTION) @QueryParam(ParamConstants.USER) String user, - @ApiParam(value = ParamConstants.USER_ACCOUNT_TYPE_DESCRIPTION) @QueryParam(ParamConstants.USER_ACCOUNT_TYPE) String account, @ApiParam(value = ParamConstants.USER_AUTHENTICATION_ORIGIN_DESCRIPTION) @QueryParam(ParamConstants.USER_AUTHENTICATION_ORIGIN) String authentication) { try { - return createOkResponse(catalogManager.getAdminManager().userSearch(query, queryOptions, token)); + return createOkResponse(catalogManager.getAdminManager().userSearch(organizationId, query, queryOptions, token)); } catch (CatalogException e) { return createErrorResponse(e); } @@ -93,19 +83,17 @@ public Response userSearch( @POST @Path("/users/create") @Consumes(MediaType.APPLICATION_JSON) - @ApiOperation(value = "Create a new user", response = User.class, notes = "Account type can only be one of 'GUEST' (default) or 'FULL'") - public Response create(@ApiParam(value = "JSON containing the parameters", required = true) UserCreateParams user) { + @ApiOperation(value = "Create a new user", response = User.class) + public Response create( + @ApiParam(value = "JSON containing the parameters", required = true) UserCreateParams user + ) { try { if (!user.checkValidParams()) { - createErrorResponse(new CatalogException("id, name, email or password not present")); - } - - if (user.getType() == null) { - user.setType(Account.AccountType.GUEST); + return createErrorResponse(new CatalogException("id, name, email or password not present")); } OpenCGAResult queryResult = catalogManager.getUserManager() - .create(user.getId(), user.getName(), user.getEmail(), user.getPassword(), user.getOrganization(), null, user.getType(), token); + .create(user.getId(), user.getName(), user.getEmail(), user.getPassword(), user.getOrganization(), null, token); return createOkResponse(queryResult); } catch (Exception e) { @@ -125,7 +113,10 @@ public Response create(@ApiParam(value = "JSON containing the parameters", requi + "configuration.
" + "type will be one of 'guest' or 'full'. If not provided, it will be considered 'guest' by default." ) - public Response remoteImport(@ApiParam(value = "JSON containing the parameters", required = true) UserImportParams remoteParams) { + public Response remoteImport( + @ApiParam(value = ParamConstants.ORGANIZATION_DESCRIPTION) @QueryParam(ParamConstants.ORGANIZATION) String organizationId, + @ApiParam(value = "JSON containing the parameters", required = true) UserImportParams remoteParams + ) { try { if (remoteParams.getResourceType() == null) { throw new CatalogException("Missing mandatory 'resourceType' field."); @@ -135,14 +126,14 @@ public Response remoteImport(@ApiParam(value = "JSON containing the parameters", } if (remoteParams.getResourceType() == USER || remoteParams.getResourceType() == APPLICATION) { - catalogManager.getUserManager().importRemoteEntities(remoteParams.getAuthenticationOriginId(), remoteParams.getId(), + catalogManager.getUserManager().importRemoteEntities(organizationId, remoteParams.getAuthenticationOriginId(), remoteParams.getId(), remoteParams.getResourceType() == APPLICATION, remoteParams.getStudyGroup(), remoteParams.getStudy(), token); } else if (remoteParams.getResourceType() == GROUP) { if (remoteParams.getId().size() > 1) { throw new CatalogException("More than one group found in 'id'. Only one group is accepted at a time"); } - catalogManager.getUserManager().importRemoteGroupOfUsers(remoteParams.getAuthenticationOriginId(), remoteParams.getId().get(0), + catalogManager.getUserManager().importRemoteGroupOfUsers(organizationId, remoteParams.getAuthenticationOriginId(), remoteParams.getId().get(0), remoteParams.getStudyGroup(), remoteParams.getStudy(), false, token); } else { throw new CatalogException("Unknown resourceType '" + remoteParams.getResourceType() + "'"); @@ -154,17 +145,36 @@ public Response remoteImport(@ApiParam(value = "JSON containing the parameters", } } + @GET + @Path("/users/permissions") + @ApiOperation(value = "User permissions", notes = "Effective permissions assigned to the users for a given list of entries.", + response = Acl.class) + public Response effectivePermissions( + @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, + @ApiParam(value = ParamConstants.ENTRY_ID_LIST_DESCRIPTION) @QueryParam(ParamConstants.ENTRY_ID_LIST) String entryIdList, + @ApiParam(value = ParamConstants.PERMISSION_LIST_DESCRIPTION) @QueryParam(ParamConstants.PERMISSION_LIST) String permissionList, + @ApiParam(value = ParamConstants.CATEGORY_DESCRIPTION) @QueryParam(ParamConstants.CATEGORY) String category) { + try { + return createOkResponse(catalogManager.getAdminManager().getEffectivePermissions(studyStr, getIdList(entryIdList), + getIdListOrEmpty(permissionList), category, token)); + } catch (Exception e) { + return createErrorResponse(e); + } + + } + @POST @Path("/users/{user}/groups/update") @Consumes(MediaType.APPLICATION_JSON) @ApiOperation(value = "Add or remove users from existing groups", response = Group.class) public Response updateGroups( + @ApiParam(value = ParamConstants.ORGANIZATION_DESCRIPTION) @QueryParam(ParamConstants.ORGANIZATION) String organizationId, @ApiParam(value = ParamConstants.USER_DESCRIPTION) @PathParam(ParamConstants.USER) String user, @ApiParam(value = "Action to be performed: ADD or REMOVE user to/from groups", allowableValues = "ADD,REMOVE", defaultValue = "ADD") @QueryParam("action") ParamUtils.AddRemoveAction action, @ApiParam(value = "JSON containing the parameters", required = true) UserUpdateGroup updateParams) { try { - return createOkResponse(catalogManager.getAdminManager().updateGroups(user, updateParams.getStudyIds(), updateParams.getGroupIds(), action, token)); + return createOkResponse(catalogManager.getAdminManager().updateGroups(organizationId, user, updateParams.getStudyIds(), updateParams.getGroupIds(), action, token)); } catch (Exception e) { return createErrorResponse(e); } @@ -176,7 +186,7 @@ public Response updateGroups( notes = "Mandatory fields: authOriginId, study
" + "
    " + "
  • authOriginId: Authentication origin id defined in the main Catalog configuration.
    • " - + "
  • study: Study [[user@]project:]study where the list of users will be associated to.
    • " + + "
  • study: Study [[organization@]project:]study where the list of users will be associated to.
    • " + "
  • from: Group defined in the authenticated origin to be synchronised.
    • " + "
  • to: Group in a study that will be synchronised.
    • " + "
  • syncAll: Flag indicating whether to synchronise all the groups present in the study with" @@ -187,13 +197,16 @@ public Response updateGroups( + "synchronised with any other group.
    • " + "
" ) - public Response externalSync(@ApiParam(value = "JSON containing the parameters", required = true) GroupSyncParams syncParams) { + public Response externalSync( + @ApiParam(value = ParamConstants.ORGANIZATION_DESCRIPTION) @QueryParam(ParamConstants.ORGANIZATION) String organizationId, + @ApiParam(value = "JSON containing the parameters", required = true) GroupSyncParams syncParams + ) { try { // TODO: These two methods should return an OpenCGAResult containing at least the number of changes if (syncParams.isSyncAll()) { - catalogManager.getUserManager().syncAllUsersOfExternalGroup(syncParams.getStudy(), syncParams.getAuthenticationOriginId(), token); + catalogManager.getUserManager().syncAllUsersOfExternalGroup(organizationId, syncParams.getStudy(), syncParams.getAuthenticationOriginId(), token); } else { - catalogManager.getUserManager().importRemoteGroupOfUsers(syncParams.getAuthenticationOriginId(), syncParams.getFrom(), + catalogManager.getUserManager().importRemoteGroupOfUsers(organizationId, syncParams.getAuthenticationOriginId(), syncParams.getFrom(), syncParams.getTo(), syncParams.getStudy(), true, token); } return createOkResponse(OpenCGAResult.empty(Group.class)); @@ -239,40 +252,6 @@ public Response groupBy( } - @POST - @Path("/catalog/indexStats") - @ApiOperation(value = "Sync Catalog into the Solr", response = Boolean.class) - public Response syncSolr(@ApiParam(value = "Collection to be indexed (file, sample, individual, family, cohort and/or job)." + - " If not provided, all of them will be indexed.") @QueryParam("collection") String collection) { - try { - boolean isEmpty = StringUtils.isEmpty(collection); - - ObjectMap params = new ObjectMap(); - List> results = new ArrayList<>(6); - if (isEmpty || collection.equalsIgnoreCase("file")) { - results.add(catalogManager.getJobManager().submit(ADMIN_STUDY_FQN, FileIndexTask.ID, Enums.Priority.MEDIUM, params, token)); - } - if (isEmpty || collection.equalsIgnoreCase("sample")) { - results.add(catalogManager.getJobManager().submit(ADMIN_STUDY_FQN, SampleIndexTask.ID, Enums.Priority.MEDIUM, params, token)); - } - if (isEmpty || collection.equalsIgnoreCase("individual")) { - results.add(catalogManager.getJobManager().submit(ADMIN_STUDY_FQN, IndividualIndexTask.ID, Enums.Priority.MEDIUM, params, token)); - } - if (isEmpty || collection.equalsIgnoreCase("family")) { - results.add(catalogManager.getJobManager().submit(ADMIN_STUDY_FQN, FamilyIndexTask.ID, Enums.Priority.MEDIUM, params, token)); - } - if (isEmpty || collection.equalsIgnoreCase("cohort")) { - results.add(catalogManager.getJobManager().submit(ADMIN_STUDY_FQN, CohortIndexTask.ID, Enums.Priority.MEDIUM, params, token)); - } - if (isEmpty || collection.equalsIgnoreCase("job")) { - results.add(catalogManager.getJobManager().submit(ADMIN_STUDY_FQN, JobIndexTask.ID, Enums.Priority.MEDIUM, params, token)); - } - return createOkResponse(OpenCGAResult.merge(results)); - } catch (Exception e) { - return createErrorResponse(e); - } - } - @POST @Path("/catalog/install") @ApiOperation(value = "Install OpenCGA database", notes = "Creates and initialises the OpenCGA database
" @@ -280,13 +259,11 @@ public Response syncSolr(@ApiParam(value = "Collection to be indexed (file, samp + "secretKey: Secret key needed to authenticate through OpenCGA (JWT)
" + "password: Password that will be set to perform future administrative operations over OpenCGA
" + "email: Administrator's email address.
" - + "organization: Administrator's organization.
" + "
    ") public Response install( @ApiParam(value = "JSON containing the mandatory parameters", required = true) InstallationParams installParams) { try { - catalogManager.installCatalogDB(installParams.getSecretKey(), installParams.getPassword(), installParams.getEmail(), - installParams.getOrganization(), false); + catalogManager.installCatalogDB("HS256", installParams.getSecretKey(), installParams.getPassword(), installParams.getEmail(), false); return createOkResponse(DataResult.empty()); } catch (Exception e) { return createErrorResponse(e); @@ -343,11 +320,14 @@ public Response install( @POST @Path("/catalog/jwt") @ApiOperation(value = "Change JWT secret key") - public Response jwt(@ApiParam(value = "JSON containing the parameters", required = true) JWTParams jwtParams) { + public Response jwt( + @ApiParam(value = ParamConstants.ORGANIZATION_DESCRIPTION) @QueryParam(ParamConstants.ORGANIZATION) String organizationId, + @ApiParam(value = "JSON containing the parameters", required = true) JWTParams jwtParams + ) { ObjectMap params = new ObjectMap(); params.putIfNotNull(MetaDBAdaptor.SECRET_KEY, jwtParams.getSecretKey()); try { - catalogManager.updateJWTParameters(params, token); + catalogManager.updateJWTParameters(organizationId, params, token); return createOkResponse(DataResult.empty()); } catch (Exception e) { return createErrorResponse(e); @@ -357,11 +337,14 @@ public Response jwt(@ApiParam(value = "JSON containing the parameters", required @POST @Path("/token") @ApiOperation(value = "Obtain a valid token for a user", hidden = true) - public Response token(@ApiParam(value = "Token parameters", required = true) TokenParams jwtParams) { + public Response token( + @ApiParam(value = ParamConstants.ORGANIZATION_DESCRIPTION) @QueryParam(ParamConstants.ORGANIZATION) String organizationId, + @ApiParam(value = "Token parameters", required = true) TokenParams jwtParams + ) { try { String newToken = jwtParams.getExpiration() != null - ? catalogManager.getUserManager().getToken(jwtParams.getUserId(), jwtParams.getAttributes(), jwtParams.getExpiration(), token) - : catalogManager.getUserManager().getNonExpiringToken(jwtParams.getUserId(), jwtParams.getAttributes(), token); + ? catalogManager.getUserManager().getToken(organizationId, jwtParams.getUserId(), jwtParams.getAttributes(), jwtParams.getExpiration(), token) + : catalogManager.getUserManager().getNonExpiringToken(organizationId, jwtParams.getUserId(), jwtParams.getAttributes(), token); AuthenticationResponse authResponse = new AuthenticationResponse(newToken); OpenCGAResult opencgaResponse = new OpenCGAResult<>(0, Collections.emptyList(), 1, Collections.singletonList(authResponse), 1); diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/AlignmentWebService.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/AlignmentWebService.java index 99fa4402fc9..277f289a6b6 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/AlignmentWebService.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/AlignmentWebService.java @@ -83,8 +83,11 @@ public Response indexRun( @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = AlignmentIndexParams.DESCRIPTION, required = true) AlignmentIndexParams params) { - return submitJob(AlignmentIndexOperation.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(AlignmentIndexOperation.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } //------------------------------------------------------------------------- @@ -177,9 +180,12 @@ public Response coverageRun( @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = CoverageIndexParams.DESCRIPTION, required = true) CoverageIndexParams params) { - return submitJob(AlignmentCoverageAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(AlignmentCoverageAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); // logger.debug("ObjectMap: {}", params); // @@ -386,9 +392,12 @@ public Response geneCoverageStatsRun( @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = AlignmentGeneCoverageStatsParams.DESCRIPTION, required = true) AlignmentGeneCoverageStatsParams params) { - return submitJob(AlignmentGeneCoverageStatsAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(AlignmentGeneCoverageStatsAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } //------------------------------------------------------------------------- @@ -404,9 +413,12 @@ public Response qcRun( @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = AlignmentQcParams.DESCRIPTION, required = true) AlignmentQcParams params) { - return submitJob(AlignmentQcAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(AlignmentQcAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } //------------------------------------------------------------------------- @@ -422,8 +434,11 @@ public Response bwaRun( @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = BwaWrapperParams.DESCRIPTION, required = true) BwaWrapperParams params) { - return submitJob(BwaWrapperAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(BwaWrapperAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -435,8 +450,11 @@ public Response samtoolsRun( @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = SamtoolsWrapperParams.DESCRIPTION, required = true) SamtoolsWrapperParams params) { - return submitJob(SamtoolsWrapperAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(SamtoolsWrapperAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -448,8 +466,11 @@ public Response deeptoolsRun( @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = DeeptoolsWrapperParams.DESCRIPTION, required = true) DeeptoolsWrapperParams params) { - return submitJob(DeeptoolsWrapperAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(DeeptoolsWrapperAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -461,8 +482,11 @@ public Response fastqcRun( @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = FastqcWrapperParams.DESCRIPTION, required = true) FastqcWrapperParams params) { - return submitJob(FastqcWrapperAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(FastqcWrapperAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -474,8 +498,11 @@ public Response picardRun( @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = PicardWrapperParams.DESCRIPTION, required = true) PicardWrapperParams params) { - return submitJob(PicardWrapperAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(PicardWrapperAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } //------------------------------------------------------------------------- diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/ClinicalWebService.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/ClinicalWebService.java index 30b06fb75e2..3dd6fa53aa4 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/ClinicalWebService.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/ClinicalWebService.java @@ -20,6 +20,7 @@ import org.apache.commons.lang3.StringUtils; import org.opencb.biodata.models.clinical.interpretation.ClinicalVariant; import org.opencb.commons.datastore.core.*; +import org.opencb.opencga.analysis.clinical.ClinicalAnalysisLoadTask; import org.opencb.opencga.analysis.clinical.ClinicalInterpretationManager; import org.opencb.opencga.analysis.clinical.ClinicalTsvAnnotationLoader; import org.opencb.opencga.analysis.clinical.exomiser.ExomiserInterpretationAnalysis; @@ -151,7 +152,7 @@ public Response create( @QueryParam(ParamConstants.CLINICAL_ANALYSIS_SKIP_CREATE_DEFAULT_INTERPRETATION_PARAM) boolean skipCreateInterpretation, @ApiParam(value = ParamConstants.INCLUDE_RESULT_DESCRIPTION, defaultValue = "false") @QueryParam(ParamConstants.INCLUDE_RESULT_PARAM) boolean includeResult, @ApiParam(name = "body", value = "JSON containing clinical analysis information", required = true) - ClinicalAnalysisCreateParams params) { + ClinicalAnalysisCreateParams params) { try { return createOkResponse(clinicalManager.create(studyStr, params.toClinicalAnalysis(), skipCreateInterpretation, queryOptions, token)); @@ -160,6 +161,27 @@ public Response create( } } + @POST + @Path("/load") + @ApiOperation(value = ClinicalAnalysisLoadTask.DESCRIPTION, response = Job.class) + public Response load( + @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, + @ApiParam(value = ParamConstants.JOB_ID_CREATION_DESCRIPTION) @QueryParam(ParamConstants.JOB_ID) String jobId, + @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, + @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, + @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, + @ApiParam(value = ClinicalAnalysisLoadParams.DESCRIPTION, required = true) ClinicalAnalysisLoadParams params) { + try { + // Execute load as a job + return submitJob(ClinicalAnalysisLoadTask.ID, study, params, jobId, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); + } catch (Exception e) { + return createErrorResponse("Load clinical analyses from file", e.getMessage()); + } + } + @POST @Path("/update") @Consumes(MediaType.APPLICATION_JSON) @@ -185,7 +207,7 @@ public Response update( @ApiParam(value = "Text attributes (Format: sex=male,age>20 ...)") @QueryParam("attributes") String attributes, @ApiParam(name = "body", value = "JSON containing clinical analysis information", required = true) - ClinicalAnalysisUpdateParams params) { + ClinicalAnalysisUpdateParams params) { try { query.remove(ParamConstants.STUDY_PARAM); return createOkResponse(clinicalManager.update(studyStr, query, params, true, queryOptions, token)); @@ -379,13 +401,14 @@ public Response delete( public Response info( @ApiParam(value = ParamConstants.CLINICAL_ANALYSES_DESCRIPTION) @PathParam(value = "clinicalAnalysis") String clinicalAnalysisStr, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, + @ApiParam(value = ParamConstants.CLINICAL_VERSION_DESCRIPTION) @QueryParam(ParamConstants.CLINICAL_VERSION_PARAM) String version, @ApiParam(value = ParamConstants.DELETED_DESCRIPTION, defaultValue = "false") @QueryParam(ParamConstants.DELETED_PARAM) boolean deleted) { try { query.remove(ParamConstants.STUDY_PARAM); query.remove("clinicalAnalysis"); List analysisList = getIdList(clinicalAnalysisStr); - DataResult analysisResult = clinicalManager.get(studyStr, analysisList, queryOptions, true, token); + DataResult analysisResult = clinicalManager.get(studyStr, analysisList, query, queryOptions, true, token); return createOkResponse(analysisResult); } catch (Exception e) { return createErrorResponse(e); @@ -428,6 +451,7 @@ public Response search( @ApiParam(value = ParamConstants.CLINICAL_DUE_DATE_DESCRIPTION) @QueryParam(ParamConstants.CLINICAL_DUE_DATE_PARAM) String dueDate, @ApiParam(value = ParamConstants.CLINICAL_QUALITY_CONTROL_SUMMARY_DESCRIPTION) @QueryParam(ParamConstants.CLINICAL_QUALITY_CONTROL_SUMMARY_PARAM) String qualityControl, @ApiParam(value = ParamConstants.CLINICAL_RELEASE_DESCRIPTION) @QueryParam(ParamConstants.CLINICAL_RELEASE_PARAM) String release, + @ApiParam(value = ParamConstants.SNAPSHOT_DESCRIPTION) @QueryParam(ParamConstants.SNAPSHOT_PARAM) int snapshot, @ApiParam(value = ParamConstants.CLINICAL_STATUS_DESCRIPTION) @QueryParam(ParamConstants.CLINICAL_STATUS_PARAM) String status, @ApiParam(value = ParamConstants.CLINICAL_INTERNAL_STATUS_DESCRIPTION) @QueryParam(ParamConstants.CLINICAL_INTERNAL_STATUS_PARAM) String internalStatus, @ApiParam(value = ParamConstants.ANNOTATION_DESCRIPTION) @QueryParam(Constants.ANNOTATION) String annotation, @@ -468,6 +492,7 @@ public Response distinct( @ApiParam(value = ParamConstants.CLINICAL_DUE_DATE_DESCRIPTION) @QueryParam(ParamConstants.CLINICAL_DUE_DATE_PARAM) String dueDate, @ApiParam(value = ParamConstants.CLINICAL_QUALITY_CONTROL_SUMMARY_DESCRIPTION) @QueryParam(ParamConstants.CLINICAL_QUALITY_CONTROL_SUMMARY_PARAM) String qualityControl, @ApiParam(value = ParamConstants.CLINICAL_RELEASE_DESCRIPTION) @QueryParam(ParamConstants.CLINICAL_RELEASE_PARAM) String release, + @ApiParam(value = ParamConstants.SNAPSHOT_DESCRIPTION) @QueryParam(ParamConstants.SNAPSHOT_PARAM) int snapshot, @ApiParam(value = ParamConstants.CLINICAL_STATUS_DESCRIPTION) @QueryParam(ParamConstants.CLINICAL_STATUS_PARAM) String status, @ApiParam(value = ParamConstants.CLINICAL_INTERNAL_STATUS_DESCRIPTION) @QueryParam(ParamConstants.CLINICAL_INTERNAL_STATUS_PARAM) String internalStatus, @ApiParam(value = ParamConstants.ANNOTATION_DESCRIPTION) @QueryParam(Constants.ANNOTATION) String annotation, @@ -553,12 +578,12 @@ public Response updateAcl( }) public Response create( @ApiParam(value = "Clinical analysis ID") @PathParam("clinicalAnalysis") String clinicalId, - @ApiParam(value = "[[user@]project:]study id") @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, + @ApiParam(value = "[[organization@]project:]study id") @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, @ApiParam(value = "Set interpretation as", allowableValues = "PRIMARY,SECONDARY", defaultValue = "SECONDARY") @QueryParam("setAs") ParamUtils.SaveInterpretationAs setAs, @ApiParam(value = ParamConstants.INCLUDE_RESULT_DESCRIPTION, defaultValue = "false") @QueryParam(ParamConstants.INCLUDE_RESULT_PARAM) boolean includeResult, @ApiParam(name = "body", value = "JSON containing clinical interpretation information", required = true) - InterpretationCreateParams params) { + InterpretationCreateParams params) { try { if (setAs == null) { setAs = ParamUtils.SaveInterpretationAs.SECONDARY; @@ -581,7 +606,7 @@ public Response create( dataType = "string", paramType = "query") }) public Response updateInterpretation( - @ApiParam(value = "[[user@]project:]study ID") @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, + @ApiParam(value = "[[organization@]project:]study ID") @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, @ApiParam(value = "Action to be performed if the array of primary findings is being updated.", allowableValues = "ADD,SET,REMOVE,REPLACE", defaultValue = "ADD") @QueryParam("primaryFindingsAction") ParamUtils.UpdateAction primaryFindingsAction, @@ -600,7 +625,7 @@ public Response updateInterpretation( @ApiParam(value = "Interpretation ID") @PathParam("interpretation") String interpretationId, @ApiParam(value = ParamConstants.INCLUDE_RESULT_DESCRIPTION, defaultValue = "false") @QueryParam(ParamConstants.INCLUDE_RESULT_PARAM) boolean includeResult, @ApiParam(name = "body", value = "JSON containing clinical interpretation information", required = true) - InterpretationUpdateParams params) { + InterpretationUpdateParams params) { try { if (primaryFindingsAction == null) { primaryFindingsAction = ParamUtils.UpdateAction.ADD; @@ -637,7 +662,7 @@ public Response updateInterpretation( @Consumes(MediaType.APPLICATION_JSON) @ApiOperation(value = "Revert to a previous interpretation version", response = Interpretation.class) public Response revertInterpretation( - @ApiParam(value = "[[user@]project:]study ID") @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, + @ApiParam(value = "[[organization@]project:]study ID") @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, @ApiParam(value = "Clinical analysis ID") @PathParam("clinicalAnalysis") String clinicalId, @ApiParam(value = "Interpretation ID") @PathParam("interpretation") String interpretationId, @ApiParam(value = "Version to revert to", required = true) @QueryParam("version") int version) { @@ -653,7 +678,7 @@ public Response revertInterpretation( @Consumes(MediaType.APPLICATION_JSON) @ApiOperation(value = "Delete interpretation", response = Interpretation.class) public Response deleteInterpretation( - @ApiParam(value = "[[user@]project:]study ID") @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, + @ApiParam(value = "[[organization@]project:]study ID") @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, @ApiParam(value = "Clinical analysis ID") @PathParam("clinicalAnalysis") String clinicalId, @ApiParam(value = "Interpretation IDs of the Clinical Analysis") @PathParam("interpretations") String interpretations, @ApiParam(value = "Interpretation id to set as primary from the list of secondaries in case of deleting the actual primary one") @@ -670,7 +695,7 @@ public Response deleteInterpretation( @Consumes(MediaType.APPLICATION_JSON) @ApiOperation(value = "Clear the fields of the main interpretation of the Clinical Analysis", response = Interpretation.class) public Response clearInterpretation( - @ApiParam(value = "[[user@]project:]study ID") @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, + @ApiParam(value = "[[organization@]project:]study ID") @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, @ApiParam(value = "Interpretation IDs of the Clinical Analysis") @PathParam("interpretations") String interpretations, @ApiParam(value = "Clinical analysis ID") @PathParam("clinicalAnalysis") String clinicalId) { try { @@ -686,7 +711,7 @@ public Response clearInterpretation( // @ApiOperation(value = "Update comments of an Interpretation", // response = org.opencb.biodata.models.clinical.interpretation.Interpretation.class, hidden = true) // public Response commentsUpdate( -// @ApiParam(value = "[[user@]project:]study ID") @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, +// @ApiParam(value = "[[organization@]project:]study ID") @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, // @ApiParam(value = "Clinical analysis ID") @PathParam("clinicalAnalysis") String clinicalId, // @ApiParam(value = "Interpretation ID") @PathParam("interpretation") String interpretationId, // @ApiParam(value = "Action to be performed.", defaultValue = "ADD") @QueryParam("action") ParamUtils.UpdateAction action, @@ -745,6 +770,7 @@ public Response interpretationSearch( @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, @ApiParam(value = ParamConstants.INTERPRETATION_ID_DESCRIPTION) @QueryParam(ParamConstants.INTERPRETATION_ID_PARAM) String id, @ApiParam(value = ParamConstants.INTERPRETATION_UUID_DESCRIPTION) @QueryParam(ParamConstants.INTERPRETATION_UUID_PARAM) String uuid, + @ApiParam(value = ParamConstants.INTERPRETATION_NAME_DESCRIPTION) @QueryParam(ParamConstants.INTERPRETATION_NAME_PARAM) String name, @ApiParam(value = ParamConstants.INTERPRETATION_CLINICAL_ANALYSIS_ID_DESCRIPTION) @QueryParam(ParamConstants.INTERPRETATION_CLINICAL_ANALYSIS_ID_PARAM) String clinicalAnalysisId, @ApiParam(value = ParamConstants.INTERPRETATION_ANALYST_ID_DESCRIPTION) @QueryParam(ParamConstants.INTERPRETATION_ANALYST_ID_PARAM) String analystId, @ApiParam(value = ParamConstants.INTERPRETATION_METHOD_NAME_DESCRIPTION) @QueryParam(ParamConstants.INTERPRETATION_METHOD_NAME_PARAM) String method, @@ -778,6 +804,7 @@ public Response interpretationDistinct( @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, @ApiParam(value = ParamConstants.INTERPRETATION_ID_DESCRIPTION) @QueryParam(ParamConstants.INTERPRETATION_ID_PARAM) String id, @ApiParam(value = ParamConstants.INTERPRETATION_UUID_DESCRIPTION) @QueryParam(ParamConstants.INTERPRETATION_UUID_PARAM) String uuid, + @ApiParam(value = ParamConstants.INTERPRETATION_NAME_DESCRIPTION) @QueryParam(ParamConstants.INTERPRETATION_NAME_PARAM) String name, @ApiParam(value = ParamConstants.INTERPRETATION_CLINICAL_ANALYSIS_ID_DESCRIPTION) @QueryParam(ParamConstants.INTERPRETATION_CLINICAL_ANALYSIS_ID_PARAM) String clinicalAnalysisId, @ApiParam(value = ParamConstants.INTERPRETATION_ANALYST_ID_DESCRIPTION) @QueryParam(ParamConstants.INTERPRETATION_ANALYST_ID_PARAM) String analystId, @ApiParam(value = ParamConstants.INTERPRETATION_METHOD_NAME_DESCRIPTION) @QueryParam(ParamConstants.INTERPRETATION_METHOD_NAME_PARAM) String methodsName, @@ -1142,6 +1169,9 @@ public Response rgaIndexRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.INDEX_AUXILIAR_COLLECTION_DESCRIPTION, defaultValue = "false") @QueryParam(ParamConstants.INDEX_AUXILIAR_COLLECTION) boolean indexAuxiliarColl, @ApiParam(value = RgaAnalysisParams.DESCRIPTION, required = true) RgaAnalysisParams params) { @@ -1150,9 +1180,9 @@ public Response rgaIndexRun( if (StringUtils.isNotEmpty(study)) { paramsMap.putIfAbsent(ParamConstants.STUDY_PARAM, study); } - return submitJob(AuxiliarRgaAnalysis.ID, null, study, paramsMap, jobName, jobDescription, dependsOn, jobTags); + return submitJob(AuxiliarRgaAnalysis.ID, null, study, paramsMap, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } else { - return submitJob(RgaAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(RgaAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } } @@ -1273,8 +1303,11 @@ public Response interpretationTieringRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = TieringInterpretationAnalysisParams.DESCRIPTION, required = true) TieringInterpretationAnalysisParams params) { - return submitJob(TieringInterpretationAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(TieringInterpretationAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @@ -1287,8 +1320,11 @@ public Response interpretationExomiserRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ExomiserInterpretationAnalysisParams.DESCRIPTION, required = true) ExomiserInterpretationAnalysisParams params) { - return submitJob(ExomiserInterpretationAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(ExomiserInterpretationAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -1300,8 +1336,11 @@ public Response interpretationTeamRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = TeamInterpretationAnalysisParams.DESCRIPTION, required = true) TeamInterpretationAnalysisParams params) { - return submitJob(TeamInterpretationAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(TeamInterpretationAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -1313,8 +1352,11 @@ public Response interpretationZettaRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ZettaInterpretationAnalysisParams.DESCRIPTION, required = true) ZettaInterpretationAnalysisParams params) { - return submitJob(ZettaInterpretationAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(ZettaInterpretationAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -1326,7 +1368,10 @@ public Response interpretationCancerTieringRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = CancerTieringInterpretationAnalysisParams.DESCRIPTION, required = true) CancerTieringInterpretationAnalysisParams params) { - return submitJob(CancerTieringInterpretationAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(CancerTieringInterpretationAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } } diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/VariantWebService.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/VariantWebService.java index 7885e38080b..70748fffb0c 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/VariantWebService.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/VariantWebService.java @@ -95,7 +95,6 @@ import java.util.*; import static org.opencb.opencga.analysis.variant.manager.VariantCatalogQueryUtils.SAVED_FILTER_DESCR; -import static org.opencb.opencga.analysis.variant.manager.VariantCatalogQueryUtils.geneRegionIntersect; import static org.opencb.opencga.core.api.ParamConstants.JOB_DEPENDS_ON; import static org.opencb.opencga.core.common.JacksonUtils.getUpdateObjectMapper; import static org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam.*; @@ -187,8 +186,11 @@ public Response variantFileIndex( @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = VariantIndexParams.DESCRIPTION, required = true) VariantIndexParams params) { - return submitJob(VariantIndexOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(VariantIndexOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @Deprecated @@ -200,11 +202,14 @@ public Response variantFileDelete( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = "Files to remove") @QueryParam("file") String file, @ApiParam(value = "Resume a previously failed indexation") @QueryParam("resume") boolean resume) throws WebServiceException { VariantFileDeleteParams params = new VariantFileDeleteParams(getIdList(file), resume); - return submitJob(VariantFileDeleteOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(VariantFileDeleteOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @GET @@ -399,8 +404,11 @@ public Response export( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = VariantExportParams.DESCRIPTION, required = true) VariantExportParams params) { - return submitJob(VariantExportTool.ID, project, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(VariantExportTool.ID, project, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @GET @@ -442,8 +450,11 @@ public Response statsRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = VariantStatsAnalysisParams.DESCRIPTION, required = true) VariantStatsAnalysisParams params) { - return submitJob(VariantStatsAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(VariantStatsAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -456,8 +467,11 @@ public Response statsExport( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = VariantStatsExportParams.DESCRIPTION, required = true) VariantStatsExportParams params) { - return submitJob("variant-stats-export", project, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob("variant-stats-export", project, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } // public static class StatsDeleteParams extends ToolParams { @@ -559,8 +573,11 @@ public Response sampleRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = SampleVariantFilterParams.DESCRIPTION, required = true) SampleVariantFilterParams params) { - return submitJob(SampleVariantFilterAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(SampleVariantFilterAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -572,8 +589,11 @@ public Response sampleEligibility( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = SampleEligibilityAnalysisParams.DESCRIPTION, required = true) SampleEligibilityAnalysisParams params) { - return submitJob(SampleEligibilityAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(SampleEligibilityAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -585,8 +605,11 @@ public Response exomiserRun( @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ExomiserWrapperParams.DESCRIPTION, required = true) ExomiserWrapperParams params) { - return submitJob(ExomiserWrapperAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(ExomiserWrapperAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @@ -680,8 +703,11 @@ public Response sampleStatsRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = SampleVariantStatsAnalysisParams.DESCRIPTION, required = true) SampleVariantStatsAnalysisParams params) { - return submitJob(SampleVariantStatsAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(SampleVariantStatsAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @GET @@ -729,8 +755,11 @@ public Response cohortStatsRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = CohortVariantStatsAnalysisParams.DESCRIPTION, required = true) CohortVariantStatsAnalysisParams params) { - return submitJob(CohortVariantStatsAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(CohortVariantStatsAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @GET @@ -902,8 +931,11 @@ public Response gwasRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = GwasAnalysisParams.DESCRIPTION, required = true) GwasAnalysisParams params) { - return submitJob(GwasAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(GwasAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } // @POST @@ -929,8 +961,11 @@ public Response mutationalSignatureRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = MutationalSignatureAnalysisParams.DESCRIPTION, required = true) MutationalSignatureAnalysisParams params) { - return submitJob(MutationalSignatureAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(MutationalSignatureAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @GET @@ -1000,7 +1035,7 @@ public Response mutationalSignatureQuery( MutationalSignatureAnalysis mutationalSignatureAnalysis = new MutationalSignatureAnalysis(); mutationalSignatureAnalysis.setUp(opencgaHome.toString(), catalogManager, storageEngineFactory, new ObjectMap(), outDir, null, - token); + false, token); mutationalSignatureAnalysis.setStudy(query.getString(STUDY.key())); mutationalSignatureAnalysis.setSignatureParams(params); @@ -1044,8 +1079,11 @@ public Response hrDetectRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = HRDetectAnalysisParams.DESCRIPTION, required = true) HRDetectAnalysisParams params) { - return submitJob(HRDetectAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(HRDetectAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -1057,8 +1095,11 @@ public Response mendelianErrorRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = MendelianErrorAnalysisParams.DESCRIPTION, required = true) MendelianErrorAnalysisParams params) { - return submitJob(MendelianErrorAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(MendelianErrorAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -1070,13 +1111,16 @@ public Response inferredSexRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = InferredSexAnalysisParams.DESCRIPTION, required = true) InferredSexAnalysisParams params) { return run(() -> { // Check before submitting the job InferredSexAnalysis.checkParameters(params.getIndividual(), params.getSample(), study, catalogManager, token); // Submit the inferred sex analysis - return submitJobRaw(InferredSexAnalysis.ID, null, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJobRaw(InferredSexAnalysis.ID, null, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); }); } @@ -1089,8 +1133,11 @@ public Response relatednessRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = RelatednessAnalysisParams.DESCRIPTION, required = true) RelatednessAnalysisParams params) { - return submitJob(RelatednessAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(RelatednessAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -1102,8 +1149,11 @@ public Response familyQcRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = FamilyQcAnalysisParams.DESCRIPTION, required = true) FamilyQcAnalysisParams params) { - return submitJob(FamilyQcAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(FamilyQcAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -1115,6 +1165,9 @@ public Response individualQcRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = IndividualQcAnalysisParams.DESCRIPTION, required = true) IndividualQcAnalysisParams params) { return run(() -> { // Check before submitting the job @@ -1122,7 +1175,7 @@ public Response individualQcRun( catalogManager, token); // Submit the individual QC analysis - return submitJobRaw(IndividualQcAnalysis.ID, null, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJobRaw(IndividualQcAnalysis.ID, null, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); }); } @@ -1135,9 +1188,12 @@ public Response sampleQcRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = SampleQcAnalysisParams.DESCRIPTION, required = true) SampleQcAnalysisParams params) { - return submitJob(SampleQcAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(SampleQcAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -1149,8 +1205,11 @@ public Response plinkRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = PlinkWrapperParams.DESCRIPTION, required = true) PlinkWrapperParams params) { - return submitJob(PlinkWrapperAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(PlinkWrapperAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -1162,8 +1221,11 @@ public Response rvtestsRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = RvtestsWrapperParams.DESCRIPTION, required = true) RvtestsWrapperParams params) { - return submitJob(RvtestsWrapperAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(RvtestsWrapperAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -1175,8 +1237,11 @@ public Response gatkRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = GatkWrapperParams.DESCRIPTION, required = true) GatkWrapperParams params) { - return submitJob(GatkWrapperAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(GatkWrapperAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -1188,8 +1253,11 @@ public Response knockoutRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = KnockoutAnalysisParams.DESCRIPTION, required = true) KnockoutAnalysisParams params) { - return submitJob(KnockoutAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(KnockoutAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @GET @@ -1229,10 +1297,13 @@ public Response genomePlotRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = GenomePlotAnalysisParams.DESCRIPTION, required = true) GenomePlotAnalysisParams params) { // To be sure: do not update quality control sample params.setSample(null); - return submitJob(GenomePlotAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(GenomePlotAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -1261,7 +1332,7 @@ public Response circos( ObjectMap executorParams = new ObjectMap(); executorParams.put("opencgaHome", opencgaHome); executorParams.put("token", token); - executor.setUp(null, executorParams, outDir.toPath()); + executor.setUp(null, executorParams, outDir.toPath(), null); // Run Circos executor StopWatch watch = StopWatch.createStarted(); @@ -1313,7 +1384,7 @@ public Response circos( // @Consumes(MediaType.APPLICATION_JSON) // @ApiOperation(value = "Validate a VCF file" + PENDING, response = QueryResponse.class) // public Response validate( -// @ApiParam(value = "Study [[user@]project:]study where study and project are the id") @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, +// @ApiParam(value = "Study [[organization@]project:]study where study and project are the id") @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, // @ApiParam(value = "VCF file id, name or path", required = true) @QueryParam("file") String file) { // return createPendingResponse(); // try { diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/operations/VariantOperationWebService.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/operations/VariantOperationWebService.java index 77574d73ad9..14fb051c85c 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/operations/VariantOperationWebService.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/operations/VariantOperationWebService.java @@ -29,6 +29,7 @@ import org.opencb.opencga.core.exceptions.VersionException; import org.opencb.opencga.core.models.job.Job; import org.opencb.opencga.core.models.operations.variant.*; +import org.opencb.opencga.core.models.study.VariantSetupResult; import org.opencb.opencga.core.models.variant.*; import org.opencb.opencga.core.tools.ToolParams; import org.opencb.opencga.core.tools.annotations.Api; @@ -101,6 +102,23 @@ public Response variantConfigure( }); } + @POST + @Path("/variant/setup") + @ApiOperation(value = "Execute Variant Setup to allow using the variant engine. This setup is necessary before starting any variant operation.", + response = VariantSetupResult.class) + public Response variantConfigure( + @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, + @ApiParam(value = "Variant setup params") VariantSetupParams params) { + return run(() -> { + StopWatch stopWatch = StopWatch.createStarted(); + VariantSetupResult result = variantManager.variantSetup(study, params, token); + return new DataResult<>() + .setResults(Collections.singletonList(result)) + .setNumResults(1) + .setTime(((int) stopWatch.getTime(TimeUnit.MILLISECONDS))); + }); + } + @POST @Path("/variant/index") @ApiOperation(value = VariantIndexOperationTool.DESCRIPTION, response = Job.class) @@ -109,9 +127,12 @@ public Response variantIndex( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = VariantIndexParams.DESCRIPTION) VariantIndexParams params) { - return submitOperation(VariantIndexOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitOperation(VariantIndexOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -122,9 +143,12 @@ public Response variantDelete( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = VariantFileDeleteParams.DESCRIPTION) VariantFileDeleteParams params) { - return submitOperation(VariantFileDeleteOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitOperation(VariantFileDeleteOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -135,9 +159,12 @@ public Response variantStudyDelete( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = VariantStudyDeleteParams.DESCRIPTION) VariantStudyDeleteParams params) { - return submitOperation(VariantStudyDeleteOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitOperation(VariantStudyDeleteOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -148,9 +175,12 @@ public Response variantIndexLauncher( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = VariantFileIndexJobLauncherParams.DESCRIPTION) VariantFileIndexJobLauncherParams params) { - return submitOperation(VariantFileIndexJobLauncherTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitOperation(VariantFileIndexJobLauncherTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -161,9 +191,12 @@ public Response variantMetadataSynchronize( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = VariantStorageMetadataSynchronizeParams.DESCRIPTION) VariantStorageMetadataSynchronizeParams params) { - return submitJobAdmin(VariantStorageMetadataSynchronizeOperationTool.ID, params, jobName, jobDescription, dependsOn, jobTags); + return submitJobAdmin(VariantStorageMetadataSynchronizeOperationTool.ID, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -174,8 +207,11 @@ public Response variantMetadataRepair( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = VariantStorageMetadataRepairToolParams.DESCRIPTION) VariantStorageMetadataRepairToolParams params) { - return submitJobAdmin(VariantStorageMetadataRepairTool.ID, params, jobName, jobDescription, dependsOn, jobTags); + return submitJobAdmin(VariantStorageMetadataRepairTool.ID, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -187,8 +223,11 @@ public Response statsIndex( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = VariantStatsAnalysisParams.DESCRIPTION, required = true) VariantStatsIndexParams params) { - return submitJob(VariantStatsIndexOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(VariantStatsIndexOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -200,8 +239,11 @@ public Response statsDelete( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = VariantStatsDeleteParams.DESCRIPTION, required = true) VariantStatsDeleteParams params) { - return submitJob(VariantStatsDeleteOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(VariantStatsDeleteOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @@ -214,10 +256,13 @@ public Response secondaryIndex( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = VariantCatalogQueryUtils.PROJECT_DESC) @QueryParam(ParamConstants.PROJECT_PARAM) String project, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = VariantSecondaryAnnotationIndexParams.DESCRIPTION) VariantSecondaryAnnotationIndexParams params) { - return variantSecondaryAnnotationIndex(jobName, jobDescription, dependsOn, jobTags, project, study, params); + return variantSecondaryAnnotationIndex(jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun, project, study, params); } @POST @@ -228,10 +273,13 @@ public Response variantSecondaryAnnotationIndex( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = VariantCatalogQueryUtils.PROJECT_DESC) @QueryParam(ParamConstants.PROJECT_PARAM) String project, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = VariantSecondaryAnnotationIndexParams.DESCRIPTION) VariantSecondaryAnnotationIndexParams params) { - return submitOperation(VariantSecondaryAnnotationIndexOperationTool.ID, project, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitOperation(VariantSecondaryAnnotationIndexOperationTool.ID, project, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @DELETE @@ -242,12 +290,15 @@ public Response secondaryIndex( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = "Samples to remove. Needs to provide all the samples in the secondary index.") @QueryParam("samples") String samples) { HashMap params = new HashMap<>(); params.put(ParamConstants.STUDY_PARAM, study); params.put("samples", samples); - return submitOperation(VariantSecondaryIndexSamplesDeleteOperationTool.ID, params, jobName, jobDescription, dependsOn, jobTags); + return submitOperation(VariantSecondaryIndexSamplesDeleteOperationTool.ID, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -258,10 +309,13 @@ public Response annotation( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.PROJECT_DESCRIPTION) @QueryParam(ParamConstants.PROJECT_PARAM) String project, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = VariantAnnotationIndexParams.DESCRIPTION) VariantAnnotationIndexParams params) { - return submitOperation(VariantAnnotationIndexOperationTool.ID, project, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitOperation(VariantAnnotationIndexOperationTool.ID, project, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @DELETE @@ -272,6 +326,9 @@ public Response annotationDelete( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = VariantCatalogQueryUtils.PROJECT_DESC) @QueryParam(ParamConstants.PROJECT_PARAM) String project, @ApiParam(value = "Annotation identifier") @QueryParam("annotationId") String annotationId ) { @@ -279,7 +336,7 @@ public Response annotationDelete( params.put(ParamConstants.PROJECT_PARAM, project); params.put("annotationId", annotationId); return submitOperationToProject(VariantAnnotationDeleteOperationTool.ID, project, params, jobName, jobDescription, dependsOn, - jobTags); + jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -290,10 +347,13 @@ public Response annotationSave( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = VariantCatalogQueryUtils.PROJECT_DESC) @QueryParam(ParamConstants.PROJECT_PARAM) String project, @ApiParam(value = VariantAnnotationSaveParams.DESCRIPTION) VariantAnnotationSaveParams params) { return submitOperationToProject(VariantAnnotationSaveOperationTool.ID, project, params, jobName, jobDescription, dependsOn, - jobTags); + jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -304,9 +364,12 @@ public Response scoreIndex( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = VariantScoreIndexParams.DESCRIPTION) VariantScoreIndexParams params) { - return submitOperation(VariantScoreIndexOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitOperation(VariantScoreIndexOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @DELETE @@ -317,6 +380,9 @@ public Response scoreDelete( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = "Unique name of the score within the study") @QueryParam("name") String name, @ApiParam(value = "Resume a previously failed remove") @QueryParam("resume") boolean resume, @@ -327,7 +393,7 @@ public Response scoreDelete( params.put("name", name); if (resume) params.put("resume", ""); if (force) params.put("force", ""); - return submitOperation(VariantScoreDeleteParams.ID, params, jobName, jobDescription, dependsOn, jobTags); + return submitOperation(VariantScoreDeleteParams.ID, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @Deprecated @@ -360,9 +426,12 @@ public Response sampleGenotypeIndex( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = VariantSecondarySampleIndexParams.DESCRIPTION) VariantSecondarySampleIndexParams params) { - return sampleIndex(jobName, jobDescription, dependsOn, jobTags, study, params); + return sampleIndex(jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun, study, params); } @Deprecated @@ -374,9 +443,12 @@ public Response sampleIndex( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = VariantSecondarySampleIndexParams.DESCRIPTION) VariantSecondarySampleIndexParams params) { - return variantSecondarySampleIndex(jobName, jobDescription, dependsOn, jobTags, study, params); + return variantSecondarySampleIndex(jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun, study, params); } @POST @@ -387,9 +459,12 @@ public Response variantSecondarySampleIndex( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = VariantSecondarySampleIndexParams.DESCRIPTION) VariantSecondarySampleIndexParams params) { - return submitOperation(VariantSecondarySampleIndexOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitOperation(VariantSecondarySampleIndexOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -400,9 +475,12 @@ public Response variantSampleDelete( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = VariantSampleDeleteParams.DESCRIPTION) VariantSampleDeleteParams params) { - return submitOperation(VariantSampleDeleteOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitOperation(VariantSampleDeleteOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -414,9 +492,12 @@ public Response familyGenotypeIndex( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = VariantFamilyIndexParams.DESCRIPTION) VariantFamilyIndexParams params) { - return familyIndex(jobName, jobDescription, dependsOn, jobTags, study, params); + return familyIndex(jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun, study, params); } @Deprecated @@ -428,9 +509,12 @@ public Response familyIndex( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = VariantFamilyIndexParams.DESCRIPTION) VariantFamilyIndexParams params) { - return submitOperation(VariantFamilyIndexOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitOperation(VariantFamilyIndexOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -441,9 +525,12 @@ public Response aggregateFamily( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = VariantAggregateFamilyParams.DESCRIPTION) VariantAggregateFamilyParams params) { - return submitOperation(VariantAggregateFamilyOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitOperation(VariantAggregateFamilyOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -454,9 +541,12 @@ public Response aggregate( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = VariantAggregateParams.DESCRIPTION) VariantAggregateParams params) { - return submitOperation(VariantAggregateOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitOperation(VariantAggregateOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -467,9 +557,12 @@ public Response julie( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.PROJECT_PARAM) @QueryParam(ParamConstants.PROJECT_PARAM) String project, @ApiParam(value = JulieParams.DESCRIPTION, required = true) JulieParams params) { - return submitOperationToProject(JulieTool.ID, project, params, jobName, jobDescription, dependsOn, jobTags); + return submitOperationToProject(JulieTool.ID, project, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -480,22 +573,25 @@ public Response variantPrune( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = VariantPruneParams.DESCRIPTION) VariantPruneParams params) { - return submitOperationToProject(VariantPruneOperationTool.ID, params.getProject(), params, jobName, jobDescription, dependsOn, jobTags); + return submitOperationToProject(VariantPruneOperationTool.ID, params.getProject(), params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } - public Response submitOperation(String toolId, String study, ToolParams params, - String jobName, String jobDescription, String jobDependsOn, String jobTags) { - return submitOperation(toolId, null, study, params, jobName, jobDescription, jobDependsOn, jobTags); + public Response submitOperation(String toolId, String study, ToolParams params, String jobName, String jobDescription, + String jobDependsOn, String jobTags, String jobScheduledStartTime, String jobPriority, Boolean dryRun) { + return submitOperation(toolId, null, study, params, jobName, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, dryRun); } public Response submitOperationToProject(String toolId, String project, ToolParams params, String jobName, String jobDescription, - String jobDependsOn, String jobTags) { - return submitOperation(toolId, project, null, params, jobName, jobDescription, jobDependsOn, jobTags); + String jobDependsOn, String jobTags, String jobScheduledStartTime, String jobPriority, Boolean dryRun) { + return submitOperation(toolId, project, null, params, jobName, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, dryRun); } public Response submitOperation(String toolId, String project, String study, ToolParams params, String jobName, String jobDescription, - String jobDependsOn, String jobTags) { + String jobDependsOn, String jobTags, String jobScheduledStartTime, String jobPriority, Boolean dryRun) { try { Map paramsMap = params.toParams(); if (StringUtils.isNotEmpty(study)) { @@ -504,26 +600,31 @@ public Response submitOperation(String toolId, String project, String study, Too if (StringUtils.isNotEmpty(project)) { paramsMap.put(ParamConstants.PROJECT_PARAM, project); } - return submitOperation(toolId, project, study, paramsMap, jobName, jobDescription, jobDependsOn, jobTags); + return submitOperation(toolId, project, study, paramsMap, jobName, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, + jobPriority, dryRun); } catch (Exception e) { return createErrorResponse(e); } } public Response submitOperationToProject(String toolId, String project, Map paramsMap, String jobName, - String jobDescription, String jobDependsOn, String jobTags) { - return submitOperation(toolId, project, null, paramsMap, jobName, jobDescription, jobDependsOn, jobTags); + String jobDescription, String jobDependsOn, String jobTags, String jobScheduledStartTime, + String jobPriority, Boolean dryRun) { + return submitOperation(toolId, project, null, paramsMap, jobName, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, + jobPriority, dryRun); } public Response submitOperation(String toolId, Map paramsMap, String jobName, String jobDescription, - String jobDependsOn, String jobTags) { + String jobDependsOn, String jobTags, String jobScheduledStartTime, String jobPriority, Boolean dryRun) { String project = (String) paramsMap.get(ParamConstants.PROJECT_PARAM); String study = (String) paramsMap.get(ParamConstants.STUDY_PARAM); - return submitOperation(toolId, project, study, paramsMap, jobName, jobDescription, jobDependsOn, jobTags); + return submitOperation(toolId, project, study, paramsMap, jobName, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, + jobPriority, dryRun); } public Response submitOperation(String toolId, String project, String study, Map paramsMap, String jobName, - String jobDescription, String jobDependsOne, String jobTags) { + String jobDescription, String jobDependsOne, String jobTags, String jobScheduledStartTime, + String jobPriority, Boolean dryRun) { Map dynamicParamsMap = new HashMap<>(); for (String key : this.params.keySet()) { String prefix = "dynamic_"; @@ -534,6 +635,6 @@ public Response submitOperation(String toolId, String project, String study, Map if (dynamicParamsMap.size() > 0) { paramsMap.put("dynamicParams", dynamicParamsMap); } - return submitJob(toolId, project, study, paramsMap, jobName, jobDescription, jobDependsOne, jobTags); + return submitJob(toolId, project, study, paramsMap, jobName, jobDescription, jobDependsOne, jobTags, jobScheduledStartTime, jobPriority, dryRun); } } diff --git a/opencga-server/src/main/resources/cli-config.yaml b/opencga-server/src/main/resources/cli-config.yaml index b7cead15b73..43a8143bac8 100644 --- a/opencga-server/src/main/resources/cli-config.yaml +++ b/opencga-server/src/main/resources/cli-config.yaml @@ -41,6 +41,9 @@ apiConfig: - name: name shortcut: n categoryConfigList: + - name: organizations + ignore: False + key: Organization - name: users ignore: False key: User diff --git a/opencga-server/src/test/java/org/opencb/opencga/server/generator/RestApiParserTest.java b/opencga-server/src/test/java/org/opencb/opencga/server/generator/RestApiParserTest.java index eaf2a87dcbf..e562973199e 100644 --- a/opencga-server/src/test/java/org/opencb/opencga/server/generator/RestApiParserTest.java +++ b/opencga-server/src/test/java/org/opencb/opencga/server/generator/RestApiParserTest.java @@ -65,6 +65,7 @@ public void testParse() { @Test public void parseToFile() throws IOException { List> classes = new LinkedList<>(); + classes.add(OrganizationWSServer.class); classes.add(UserWSServer.class); classes.add(ProjectWSServer.class); classes.add(StudyWSServer.class); diff --git a/opencga-server/src/test/java/org/opencb/opencga/server/rest/FileWSServerTest.java b/opencga-server/src/test/java/org/opencb/opencga/server/rest/FileWSServerTest.java index f061eae49d1..ee662ce1952 100644 --- a/opencga-server/src/test/java/org/opencb/opencga/server/rest/FileWSServerTest.java +++ b/opencga-server/src/test/java/org/opencb/opencga/server/rest/FileWSServerTest.java @@ -29,8 +29,8 @@ import org.opencb.commons.datastore.core.QueryResponse; import org.opencb.opencga.TestParamConstants; import org.opencb.opencga.catalog.db.api.FileDBAdaptor; +import org.opencb.opencga.catalog.db.mongodb.MongoBackupUtils; import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.managers.CatalogManagerTest; import org.opencb.opencga.core.common.IOUtils; import org.opencb.opencga.core.models.file.File; import org.opencb.opencga.core.models.job.Job; @@ -62,6 +62,7 @@ public class FileWSServerTest { private WebTarget webTarget; private static WSServerTestUtils serverTestUtils; private String sessionId; + private String organizationId = "test"; private String studyId = "user@1000G:phase1"; public static final Path ROOT_DIR = Paths.get("/tmp/opencga-server-FileWSServerTest-folder"); @@ -91,16 +92,16 @@ static public void shutdownServer() throws Exception { @Before public void init() throws Exception { webTarget = serverTestUtils.getWebTarget(); - sessionId = OpenCGAWSServer.catalogManager.getUserManager().login("user", TestParamConstants.PASSWORD).getToken(); + sessionId = OpenCGAWSServer.catalogManager.getUserManager().login(organizationId, "user", TestParamConstants.PASSWORD).getToken(); if (ROOT_DIR.toFile().exists()) { IOUtils.deleteDirectory(ROOT_DIR); } Files.createDirectory(ROOT_DIR); - CatalogManagerTest.createDebugFile(ROOT_DIR.resolve("file1.txt").toString()); - CatalogManagerTest.createDebugFile(ROOT_DIR.resolve("file2.txt").toString()); + MongoBackupUtils.createDebugFile(ROOT_DIR.resolve("file1.txt").toString()); + MongoBackupUtils.createDebugFile(ROOT_DIR.resolve("file2.txt").toString()); Files.createDirectory(ROOT_DIR.resolve("data")); - CatalogManagerTest.createDebugFile(ROOT_DIR.resolve("data").resolve("file2.txt").toString()); + MongoBackupUtils.createDebugFile(ROOT_DIR.resolve("data").resolve("file2.txt").toString()); String fileName = "variant-test-file.vcf.gz"; Files.copy(this.getClass().getClassLoader().getResourceAsStream(fileName), ROOT_DIR.resolve("data").resolve(fileName)); fileName = "HG00096.chrom20.small.bam"; @@ -182,7 +183,7 @@ public void updateFilePOST() throws Exception { .queryParam("sid", sessionId).request().post(Entity.json(params), String.class); QueryResponse response = WSServerTestUtils.parseResult(json, Object.class); - file = OpenCGAWSServer.catalogManager.getFileManager().get(file.getUid(), null, sessionId).first(); + file = OpenCGAWSServer.catalogManager.getFileManager().get(organizationId, file.getUid(), null, sessionId).first(); assertEquals(params.getString(FileDBAdaptor.QueryParams.DESCRIPTION.key()), file.getDescription()); } diff --git a/opencga-server/src/test/java/org/opencb/opencga/server/rest/IndividualWSServerTest.java b/opencga-server/src/test/java/org/opencb/opencga/server/rest/IndividualWSServerTest.java index 499d68b5e62..74ff8a9c417 100644 --- a/opencga-server/src/test/java/org/opencb/opencga/server/rest/IndividualWSServerTest.java +++ b/opencga-server/src/test/java/org/opencb/opencga/server/rest/IndividualWSServerTest.java @@ -61,6 +61,7 @@ public class IndividualWSServerTest { private static WSServerTestUtils serverTestUtils; private WebTarget webTarget; private static ObjectMapper jsonObjectMapper; + private String organizationId = "test"; private String sessionId; private String studyId = "user@1000G:phase1"; private String in1; @@ -93,7 +94,7 @@ static public void shutdownServer() throws Exception { @Before public void init() throws Exception { webTarget = serverTestUtils.getWebTarget(); - sessionId = OpenCGAWSServer.catalogManager.getUserManager().login("user", TestParamConstants.PASSWORD).getToken(); + sessionId = OpenCGAWSServer.catalogManager.getUserManager().login(organizationId, "user", TestParamConstants.PASSWORD).getToken(); in1 = OpenCGAWSServer.catalogManager.getIndividualManager().create(studyId, new Individual().setId("in1"), new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true), sessionId).first().getId(); in2 = OpenCGAWSServer.catalogManager.getIndividualManager().create(studyId, new Individual().setId("in2"), new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true), sessionId).first() diff --git a/opencga-server/src/test/java/org/opencb/opencga/server/rest/JobWSServerTest.java b/opencga-server/src/test/java/org/opencb/opencga/server/rest/JobWSServerTest.java index 30bf74c2ddf..0f855e040ee 100644 --- a/opencga-server/src/test/java/org/opencb/opencga/server/rest/JobWSServerTest.java +++ b/opencga-server/src/test/java/org/opencb/opencga/server/rest/JobWSServerTest.java @@ -32,6 +32,7 @@ public class JobWSServerTest { private static WSServerTestUtils serverTestUtils; private WebTarget webTarget; + private String organizationId = "test"; private String studyId = "user@1000G:phase1"; private String sessionId; @@ -54,7 +55,7 @@ static public void shutdownServer() throws Exception { public void init() throws Exception { // serverTestUtils.setUp(); webTarget = serverTestUtils.getWebTarget(); - sessionId = OpenCGAWSServer.catalogManager.getUserManager().login("user", TestParamConstants.PASSWORD).getToken(); + sessionId = OpenCGAWSServer.catalogManager.getUserManager().login(organizationId, "user", TestParamConstants.PASSWORD).getToken(); } @After diff --git a/opencga-server/src/test/java/org/opencb/opencga/server/rest/ProjectWSServerTest.java b/opencga-server/src/test/java/org/opencb/opencga/server/rest/ProjectWSServerTest.java index 518e867eaa5..04b5c77257d 100644 --- a/opencga-server/src/test/java/org/opencb/opencga/server/rest/ProjectWSServerTest.java +++ b/opencga-server/src/test/java/org/opencb/opencga/server/rest/ProjectWSServerTest.java @@ -146,7 +146,7 @@ public void modifyProject(long projectId, String sessionId) { System.out.println("\nJSON RESPONSE"); System.out.println(s); try { - org.codehaus.jackson.map.ObjectMapper objectMapper = new org.codehaus.jackson.map.ObjectMapper(); + ObjectMapper objectMapper = new ObjectMapper(); QueryResponse queryResponse = objectMapper.readValue(s, QueryResponse.class); //Map userData = objectMapper.readValue(s, Map.class); assertEquals("Expected [], actual [" + queryResponse.getError() + "]", "", queryResponse.getError()); diff --git a/opencga-server/src/test/java/org/opencb/opencga/server/rest/SampleWSServerTest.java b/opencga-server/src/test/java/org/opencb/opencga/server/rest/SampleWSServerTest.java index 1e70b03aff1..56e034fb005 100644 --- a/opencga-server/src/test/java/org/opencb/opencga/server/rest/SampleWSServerTest.java +++ b/opencga-server/src/test/java/org/opencb/opencga/server/rest/SampleWSServerTest.java @@ -24,8 +24,6 @@ import org.opencb.opencga.TestParamConstants; import org.opencb.opencga.catalog.db.api.SampleDBAdaptor; import org.opencb.opencga.core.api.ParamConstants; -import org.opencb.opencga.core.models.cohort.Cohort; -import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.individual.Individual; import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.testclassification.duration.MediumTests; @@ -38,13 +36,15 @@ /** * Created by jacobo on 25/06/15. */ +@Ignore @Category(MediumTests.class) public class SampleWSServerTest { private static WSServerTestUtils serverTestUtils; private WebTarget webTarget; private String sessionId; - private String studyId = "user@1000G:phase1"; + private String organizationId = "test"; + private String studyId = organizationId + "@1000G:phase1"; private long in1; private long s1, s2, s3, s4; @@ -67,7 +67,7 @@ static public void shutdownServer() throws Exception { public void init() throws Exception { // serverTestUtils.setUp(); webTarget = serverTestUtils.getWebTarget(); - sessionId = OpenCGAWSServer.catalogManager.getUserManager().login("user", TestParamConstants.PASSWORD).getToken(); + sessionId = OpenCGAWSServer.catalogManager.getUserManager().login(organizationId, "user", TestParamConstants.PASSWORD).getToken(); in1 = OpenCGAWSServer.catalogManager.getIndividualManager().create(studyId, new Individual().setId("in1"), new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true), sessionId).first().getUid(); s1 = OpenCGAWSServer.catalogManager.getSampleManager().create(studyId, new Sample().setId("s1"), new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true), sessionId).first().getUid(); diff --git a/opencga-server/src/test/java/org/opencb/opencga/server/rest/UserWSServerTest.java b/opencga-server/src/test/java/org/opencb/opencga/server/rest/UserWSServerTest.java index 2494c5cb1ad..c6a535628d7 100644 --- a/opencga-server/src/test/java/org/opencb/opencga/server/rest/UserWSServerTest.java +++ b/opencga-server/src/test/java/org/opencb/opencga/server/rest/UserWSServerTest.java @@ -16,7 +16,7 @@ package org.opencb.opencga.server.rest; -import org.codehaus.jackson.map.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.experimental.categories.Category; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.QueryResponse; diff --git a/opencga-server/src/test/java/org/opencb/opencga/server/rest/VariableSetWSServerTest.java b/opencga-server/src/test/java/org/opencb/opencga/server/rest/VariableSetWSServerTest.java index 177a529004a..82db337055a 100644 --- a/opencga-server/src/test/java/org/opencb/opencga/server/rest/VariableSetWSServerTest.java +++ b/opencga-server/src/test/java/org/opencb/opencga/server/rest/VariableSetWSServerTest.java @@ -16,7 +16,7 @@ package org.opencb.opencga.server.rest; -import org.codehaus.jackson.map.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.*; import org.junit.experimental.categories.Category; import org.junit.rules.ExpectedException; @@ -74,8 +74,6 @@ static public void shutdownServer() throws Exception { public void init() throws Exception { // serverTestUtils.setUp(); webTarget = serverTestUtils.getWebTarget(); - variableSetId = OpenCGAWSServer.catalogManager.getStudyManager().searchVariableSets(studyId, null,null, sessionId) - .first().getUid(); } diff --git a/opencga-storage/opencga-storage-app/pom.xml b/opencga-storage/opencga-storage-app/pom.xml index 2f760ec9164..df69dcc79dd 100644 --- a/opencga-storage/opencga-storage-app/pom.xml +++ b/opencga-storage/opencga-storage-app/pom.xml @@ -284,6 +284,11 @@ * + + + javax.ws.rs:jsr311-api + + @@ -297,9 +302,8 @@ storage-hadoop - storage-hadoop + !skipStorageHadoop - true @@ -309,9 +313,9 @@ org.opencb.opencga - ${opencga-storage-hadoop-deps.artifactId} + ${opencga-hadoop-lib.artifactId} ${project.parent.version} - shaded + true diff --git a/opencga-storage/opencga-storage-app/src/main/java/org/opencb/opencga/storage/app/cli/client/options/StorageVariantCommandOptions.java b/opencga-storage/opencga-storage-app/src/main/java/org/opencb/opencga/storage/app/cli/client/options/StorageVariantCommandOptions.java index 398b7f775cd..acd37799e0e 100644 --- a/opencga-storage/opencga-storage-app/src/main/java/org/opencb/opencga/storage/app/cli/client/options/StorageVariantCommandOptions.java +++ b/opencga-storage/opencga-storage-app/src/main/java/org/opencb/opencga/storage/app/cli/client/options/StorageVariantCommandOptions.java @@ -218,7 +218,10 @@ public static class GenericVariantDeleteOptions { splitter = CommaParameterSplitter.class, required = true) public List file = null; - @Parameter(names = {"--resume"}, description = "Resume a previously failed indexation") + @Parameter(names = {"--force"}, description = "Force delete operation. This would allow deleting partially loaded files.") + public boolean force; + + @Parameter(names = {"--resume"}, description = "Resume failed delete operation.") public boolean resume; } diff --git a/opencga-storage/opencga-storage-benchmark/pom.xml b/opencga-storage/opencga-storage-benchmark/pom.xml index 7b26545a09d..12400612512 100644 --- a/opencga-storage/opencga-storage-benchmark/pom.xml +++ b/opencga-storage/opencga-storage-benchmark/pom.xml @@ -128,18 +128,15 @@ org.opencb.opencga - ${opencga-storage-hadoop-deps.artifactId} + ${opencga-hadoop-lib.artifactId} ${project.parent.version} - shaded - test org.opencb.opencga - ${opencga-storage-hadoop-deps.artifactId} + ${opencga-hadoop-lib.artifactId} ${project.parent.version} - tests - test-jar test + test-jar junit diff --git a/opencga-storage/opencga-storage-core/pom.xml b/opencga-storage/opencga-storage-core/pom.xml index 0db07bf99ed..0a392fe9020 100644 --- a/opencga-storage/opencga-storage-core/pom.xml +++ b/opencga-storage/opencga-storage-core/pom.xml @@ -45,10 +45,12 @@ junit junit + test org.mockito mockito-core + test org.redisson @@ -105,11 +107,6 @@ commons-codec commons-codec - - commons-lang - commons-lang - test - org.apache.commons commons-compress diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/clinical/ClinicalVariantEngine.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/clinical/ClinicalVariantEngine.java deleted file mode 100644 index 4181d6608c0..00000000000 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/clinical/ClinicalVariantEngine.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Copyright 2015-2017 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.storage.core.clinical; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectReader; -import org.opencb.biodata.models.clinical.ClinicalComment; -import org.opencb.biodata.models.clinical.interpretation.ClinicalVariant; -import org.opencb.biodata.models.clinical.interpretation.Interpretation; -import org.opencb.commons.datastore.core.*; -import org.opencb.commons.utils.FileUtils; -import org.opencb.commons.utils.ListUtils; -import org.opencb.opencga.core.config.storage.StorageConfiguration; - -import java.io.IOException; -import java.nio.file.Path; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -import static org.opencb.commons.datastore.core.QueryParam.Type.*; - -public interface ClinicalVariantEngine { - - enum QueryParams implements QueryParam { - CLINICAL_ANALYSIS_ID("clinicalAnalysisId", INTEGER, ""), - FAMILY("family", TEXT_ARRAY, ""), - SUBJECT("subject", TEXT_ARRAY, ""), - SAMPLE("sample", TEXT_ARRAY, ""), - - STUDY_ID("studyId", INTEGER_ARRAY, ""), - STUDY("study", INTEGER_ARRAY, ""); // Alias to studyId in the database. Only for the webservices. - - private static Map map; - static { - map = new LinkedHashMap<>(); - for (QueryParams params : QueryParams.values()) { - map.put(params.key(), params); - } - } - - private final String key; - private Type type; - private String description; - - QueryParams(String key, Type type, String description) { - this.key = key; - this.type = type; - this.description = description; - } - - @Override - public String key() { - return key; - } - - @Override - public Type type() { - return type; - } - - @Override - public String description() { - return description; - } - - public static Map getMap() { - return map; - } - - public static QueryParams getParam(String key) { - return map.get(key); - } - } - - void create(String dbName) throws ClinicalVariantException; - - boolean isAlive(String collection) throws ClinicalVariantException; - - boolean exists(String dbName) throws ClinicalVariantException; - - void insert(Interpretation interpretation, String collection) throws IOException, ClinicalVariantException; - - /** - * Insert a list of Interpretation objects into Solr: previously each Interpretation object is - * converted to multiple ClinicalVariantSearchModel objects and they will be stored in Solr. - * - * @param interpretations List of Interpretation objects to insert - * @param collection Solr collection where to insert - * @throws IOException IOException - * @throws ClinicalVariantException ClinicalVariantException - */ - default void insert(List interpretations, String collection) throws IOException, ClinicalVariantException { - if (ListUtils.isNotEmpty(interpretations)) { - for (Interpretation interpretation: interpretations) { - insert(interpretation, collection); - } - } - } - - /** - * Load a JSON file containing Interpretation object into the Solr core/collection. - * - * @param interpretationJsonPath Path to the JSON file containing the Interpretation objects - * @param collection Solr collection where to insert - * @throws IOException IOException - * @throws ClinicalVariantException ClinicalVariantException - */ - default void insert(Path interpretationJsonPath, String collection) throws IOException, ClinicalVariantException { - FileUtils.checkFile(interpretationJsonPath); - - ObjectReader objectReader = new ObjectMapper().readerFor(Interpretation.class); - Interpretation interpretation = objectReader.readValue(interpretationJsonPath.toFile()); - insert(interpretation, collection); - } - - DataResult query(Query query, QueryOptions options, String collection) - throws IOException, ClinicalVariantException; - - DataResult interpretationQuery(Query query, QueryOptions options, String collection) - throws IOException, ClinicalVariantException; - - DataResult facet(Query query, QueryOptions queryOptions, String collection) - throws IOException, ClinicalVariantException; - - ClinicalVariantIterator iterator(Query query, QueryOptions options, String collection) - throws ClinicalVariantException, IOException; - - - void addInterpretationComment(long interpretationId, ClinicalComment comment, String collection) - throws IOException, ClinicalVariantException; - - void addClinicalVariantComment(long interpretationId, String variantId, ClinicalComment comment, String collection) - throws IOException, ClinicalVariantException; - - void setStorageConfiguration(StorageConfiguration storageConfiguration); -} diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/clinical/ClinicalVariantException.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/clinical/ClinicalVariantException.java deleted file mode 100644 index 093b5c214a6..00000000000 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/clinical/ClinicalVariantException.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2015-2017 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.storage.core.clinical; - -public class ClinicalVariantException extends Exception { - - public ClinicalVariantException(String message) { - super(message); - } - - public ClinicalVariantException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/clinical/ClinicalVariantIterator.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/clinical/ClinicalVariantIterator.java deleted file mode 100644 index 34491193106..00000000000 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/clinical/ClinicalVariantIterator.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright 2015-2017 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.storage.core.clinical; - - -import org.opencb.biodata.models.clinical.interpretation.ClinicalVariant; - -import java.util.Iterator; - -public interface ClinicalVariantIterator extends Iterator, AutoCloseable { -} diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/clinical/ClinicalVariantQueryParam.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/clinical/ClinicalVariantQueryParam.java deleted file mode 100644 index f3fb4255c58..00000000000 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/clinical/ClinicalVariantQueryParam.java +++ /dev/null @@ -1,248 +0,0 @@ -/* - * Copyright 2015-2017 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.storage.core.clinical; - -import org.opencb.commons.datastore.core.QueryParam; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import static org.opencb.commons.datastore.core.QueryParam.Type.STRING; -import static org.opencb.commons.datastore.core.QueryParam.Type.TEXT_ARRAY; -import static org.opencb.opencga.storage.core.variant.query.VariantQueryUtils.*; - -public final class ClinicalVariantQueryParam implements QueryParam { - - private final String key; - private final Type type; - private final String description; - - private static final List VALUES = new ArrayList<>(); - private static final String ACCEPTS_ALL_NONE = "Accepts '" + ALL + "' and '" + NONE + "'."; - private static final String ACCEPTS_AND_OR = "Accepts AND (" + AND + ") and OR (" + OR + ") operators."; - - // ---------- Catalog - - public static final String PROJECT_ID_DESCR = "List of project IDs"; - public static final ClinicalVariantQueryParam PROJECT_ID = new ClinicalVariantQueryParam("projectId", TEXT_ARRAY, PROJECT_ID_DESCR); - - public static final String ASSEMBLY_DESCR = "List of assemblies"; - public static final ClinicalVariantQueryParam ASSEMBLY = new ClinicalVariantQueryParam("assembly", TEXT_ARRAY, ASSEMBLY_DESCR); - - public static final String STUDY_ID_DESCR = "List of study IDs"; - public static final ClinicalVariantQueryParam STUDY_ID = new ClinicalVariantQueryParam("studyId", TEXT_ARRAY, - STUDY_ID_DESCR); - - // ---------- Clinical Analysis (aka CA) - - public static final String CA_ID_DESCR = "List of clinical analysis IDs"; - public static final ClinicalVariantQueryParam CA_ID = new ClinicalVariantQueryParam("caId", TEXT_ARRAY, - CA_ID_DESCR); - - public static final String CA_NAME_DESCR = "List of clinical analysis names"; - public static final ClinicalVariantQueryParam CA_NAME = new ClinicalVariantQueryParam("caName", TEXT_ARRAY, - CA_NAME_DESCR); - - public static final String CA_DESCRIPTION_DESCR = "Clinical analysis description"; - public static final ClinicalVariantQueryParam CA_DESCRIPTION = new ClinicalVariantQueryParam("caDescription", - TEXT_ARRAY, CA_DESCRIPTION_DESCR); - - public static final String CA_DISORDER_DESCR = "List of clinical analysis disorders"; - public static final ClinicalVariantQueryParam CA_DISORDER = new ClinicalVariantQueryParam("caDisorderId", - TEXT_ARRAY, CA_DISORDER_DESCR); - - public static final String CA_FILE_DESCR = "List of clinical analysis files"; - public static final ClinicalVariantQueryParam CA_FILE = new ClinicalVariantQueryParam("caFiles", TEXT_ARRAY, - CA_FILE_DESCR); - - public static final String CA_PROBAND_ID_DESCR = "List of proband IDs"; - public static final ClinicalVariantQueryParam CA_PROBAND_ID = new ClinicalVariantQueryParam("caProbandId", - TEXT_ARRAY, CA_PROBAND_ID_DESCR); - - public static final String CA_PROBAND_DISORDERS_DESCR = "List of proband disorders"; - public static final ClinicalVariantQueryParam CA_PROBAND_DISORDERS = new ClinicalVariantQueryParam("caProbandDisorders", - TEXT_ARRAY, CA_PROBAND_DISORDERS_DESCR); - - public static final String CA_PROBAND_PHENOTYPES_DESCR = "List of proband phenotypes"; - public static final ClinicalVariantQueryParam CA_PROBAND_PHENOTYPES = new ClinicalVariantQueryParam("caProbandPhenotypes", - TEXT_ARRAY, CA_PROBAND_PHENOTYPES_DESCR); - - public static final String CA_FAMILY_ID_DESCR = "List of family IDs"; - public static final ClinicalVariantQueryParam CA_FAMILY_ID = new ClinicalVariantQueryParam("caFamilyId", TEXT_ARRAY, - CA_FAMILY_ID_DESCR); - - public static final String CA_FAMILY_MEMBER_IDS_DESCR = "List of clinical analysis family member IDs"; - public static final ClinicalVariantQueryParam CA_FAMILY_MEMBER_IDS = new ClinicalVariantQueryParam("caFamilyMemberIds", TEXT_ARRAY, - CA_FAMILY_MEMBER_IDS_DESCR); - - public static final String CA_COMMENTS_DESCR = "List of clinical analysis comments"; - public static final ClinicalVariantQueryParam CA_COMMENTS = new ClinicalVariantQueryParam("caComments", TEXT_ARRAY, - CA_COMMENTS_DESCR); - - public static final String CA_INFO_DESCR = ""; - public static final ClinicalVariantQueryParam CA_INFO = new ClinicalVariantQueryParam("caInfo", TEXT_ARRAY, CA_INFO_DESCR); - - // ---------- Interpretation (aka INT) - - public static final String INT_ID_DESCR = "List of interpretation IDs"; - public static final ClinicalVariantQueryParam INT_ID = new ClinicalVariantQueryParam("intId", TEXT_ARRAY, INT_ID_DESCR); - - public static final String INT_SOFTWARE_NAME_DESCR = "List of interpretation software names"; - public static final ClinicalVariantQueryParam INT_SOFTWARE_NAME = new ClinicalVariantQueryParam("intSoftwareName", TEXT_ARRAY, - INT_SOFTWARE_NAME_DESCR); - - public static final String INT_SOFTWARE_VERSION_DESCR = "List of interpretation software versions"; - public static final ClinicalVariantQueryParam INT_SOFTWARE_VERSION = new ClinicalVariantQueryParam("intSoftwareVersion", TEXT_ARRAY, - INT_SOFTWARE_VERSION_DESCR); - - public static final String INT_ANALYST_NAME_DESCR = "List of interpretation analysist names"; - public static final ClinicalVariantQueryParam INT_ANALYST_NAME = new ClinicalVariantQueryParam("intAnalystName", TEXT_ARRAY, - INT_ANALYST_NAME_DESCR); - - public static final String INT_PANELS_DESCR = "List of interpretation panels"; - public static final ClinicalVariantQueryParam INT_PANELS = new ClinicalVariantQueryParam("intPanels", TEXT_ARRAY, - INT_PANELS_DESCR); - - public static final String INT_INFO_DESCR = ""; - public static final ClinicalVariantQueryParam INT_INFO = new ClinicalVariantQueryParam("intInfo", TEXT_ARRAY, INT_INFO_DESCR); - - public static final String INT_DESCRIPTION_DESCR = "Interpretation description"; - public static final ClinicalVariantQueryParam INT_DESCRIPTION = new ClinicalVariantQueryParam("intDescription", TEXT_ARRAY, - INT_DESCRIPTION_DESCR); - - public static final String INT_DEPENDENCY_DESCR = "List of interpretation dependency, format: name:version, e.g. cellbase:4.0"; - public static final ClinicalVariantQueryParam INT_DEPENDENCY = new ClinicalVariantQueryParam("intDependency", TEXT_ARRAY, - INT_DEPENDENCY_DESCR); - - public static final String INT_FILTERS_DESCR = "List of interpretation filters"; - public static final ClinicalVariantQueryParam INT_FILTERS = new ClinicalVariantQueryParam("intFilters", TEXT_ARRAY, - INT_FILTERS_DESCR); - - public static final String INT_COMMENTS_DESCR = "List of interpretation comments"; - public static final ClinicalVariantQueryParam INT_COMMENTS = new ClinicalVariantQueryParam("intComments", TEXT_ARRAY, - INT_COMMENTS_DESCR); - - public static final String INT_CREATION_DATE_DESCR = "Iinterpretation creation date (including date ranges)"; - public static final ClinicalVariantQueryParam INT_CREATION_DATE = new ClinicalVariantQueryParam("intCreationDate", STRING, - INT_CREATION_DATE_DESCR); - - // ---------- Reported variant (aka RV) - - public static final String RV_DE_NOVO_QUALITY_SCORE_DESCR = "List of reported variant de novo quality scores"; - public static final ClinicalVariantQueryParam RV_DE_NOVO_QUALITY_SCORE = new ClinicalVariantQueryParam("rvDeNovoQualityScore", - TEXT_ARRAY, RV_DE_NOVO_QUALITY_SCORE_DESCR); - - public static final String RV_COMMENTS_DESCR = "List of reported variant comments"; - public static final ClinicalVariantQueryParam RV_COMMENTS = new ClinicalVariantQueryParam("rvComments", TEXT_ARRAY, - RV_COMMENTS_DESCR); - - // ---------- Reported event (aka RE) - - public static final String RE_PHENOTYPE_NAMES_DESCR = "List of reported event phenotype names"; - public static final ClinicalVariantQueryParam RE_PHENOTYPE_NAMES = new ClinicalVariantQueryParam("rePhenotypeNames", TEXT_ARRAY, - RE_PHENOTYPE_NAMES_DESCR); - - public static final String RE_CONSEQUENCE_TYPE_IDS_DESCR = "List of reported event consequence type IDs"; - public static final ClinicalVariantQueryParam RE_CONSEQUENCE_TYPE_IDS = new ClinicalVariantQueryParam("reConsequenceTypeIds", - TEXT_ARRAY, RE_CONSEQUENCE_TYPE_IDS_DESCR); - - public static final String RE_GENE_NAMES_DESCR = "List of reported event gene names"; - public static final ClinicalVariantQueryParam RE_GENE_NAMES = new ClinicalVariantQueryParam("reGeneNames", TEXT_ARRAY, - RE_GENE_NAMES_DESCR); - - public static final String RE_XREFS_DESCR = "List of reported event phenotype xRefs"; - public static final ClinicalVariantQueryParam RE_XREFS = new ClinicalVariantQueryParam("reXrefs", TEXT_ARRAY, RE_XREFS_DESCR); - - public static final String RE_PANEL_IDS_DESCR = "List of reported event panel IDs"; - public static final ClinicalVariantQueryParam RE_PANEL_IDS = new ClinicalVariantQueryParam("rePanelNames", TEXT_ARRAY, - RE_PANEL_IDS_DESCR); - - public static final String RE_ACMG_DESCR = "List of reported event ACMG"; - public static final ClinicalVariantQueryParam RE_ACMG = new ClinicalVariantQueryParam("reAcmg", TEXT_ARRAY, RE_ACMG_DESCR); - - public static final String RE_CLINICAL_SIGNIFICANCE_DESCR = "List of reported event clinical significance"; - public static final ClinicalVariantQueryParam RE_CLINICAL_SIGNIFICANCE = new ClinicalVariantQueryParam("reClinicalSignificance", - TEXT_ARRAY, RE_CLINICAL_SIGNIFICANCE_DESCR); - - public static final String RE_DRUG_RESPONSE_DESCR = "List of reported event drug response"; - public static final ClinicalVariantQueryParam RE_DRUG_RESPONSE = new ClinicalVariantQueryParam("reDrugResponse", TEXT_ARRAY, - RE_DRUG_RESPONSE_DESCR); - - public static final String RE_TRAIT_ASSOCIATION_DESCR = "List of reported event trait association"; - public static final ClinicalVariantQueryParam RE_TRAIT_ASSOCIATION = new ClinicalVariantQueryParam("reTraitAssociation", TEXT_ARRAY, - RE_TRAIT_ASSOCIATION_DESCR); - - public static final String RE_FUNCTIONAL_EFFECT_DESCR = "List of reported event functional effect"; - public static final ClinicalVariantQueryParam RE_FUNCTIONAL_EFFECT = new ClinicalVariantQueryParam("reFunctionalEffect", TEXT_ARRAY, - RE_FUNCTIONAL_EFFECT_DESCR); - - public static final String RE_TUMORIGENESIS_DESCR = "List of reported event tumorigenesis"; - public static final ClinicalVariantQueryParam RE_TUMORIGENESIS = new ClinicalVariantQueryParam("reTumorigenesis", TEXT_ARRAY, - RE_TUMORIGENESIS_DESCR); - - public static final String RE_OTHER_CLASSIFICATION_DESCR = "List of reported event other classification"; - public static final ClinicalVariantQueryParam RE_OTHER_CLASSIFICATION = new ClinicalVariantQueryParam("reOtherClassification", - TEXT_ARRAY, RE_OTHER_CLASSIFICATION_DESCR); - - public static final String RE_ROLES_IN_CANCER_DESCR = "List of reported event roles in cancer"; - public static final ClinicalVariantQueryParam RE_ROLES_IN_CANCER = new ClinicalVariantQueryParam("reRolesInCancer", TEXT_ARRAY, - RE_ROLES_IN_CANCER_DESCR); - - public static final String RE_TIER_DESCR = "List of reported event tier"; - public static final ClinicalVariantQueryParam RE_TIER = new ClinicalVariantQueryParam("reTier", TEXT_ARRAY, RE_TIER_DESCR); - - public static final String RE_JUSTIFICATION_DESCR = "List of reported event justification"; - public static final ClinicalVariantQueryParam RE_JUSTIFICATION = new ClinicalVariantQueryParam("reJustification", TEXT_ARRAY, - RE_JUSTIFICATION_DESCR); - - public static final String RE_AUX_DESCR = ""; - public static final ClinicalVariantQueryParam RE_AUX = new ClinicalVariantQueryParam("reAux", TEXT_ARRAY, RE_AUX_DESCR); - - // Constructor - private ClinicalVariantQueryParam(String key, Type type, String description) { - this.key = key; - this.type = type; - this.description = description; - - VALUES.add(this); - } - - @Override - public String key() { - return key; - } - - @Override - public String description() { - return description; - } - - @Override - public Type type() { - return type; - } - - @Override - public String toString() { - return key() + " [" + type() + "] : " + description(); - } - - public static List values() { - return Collections.unmodifiableList(VALUES); - } -} diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManager.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManager.java index 6c539e4b077..1e8dcdc94dc 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManager.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManager.java @@ -573,11 +573,15 @@ public ProjectMetadata getProjectMetadata() { public ProjectMetadata getAndUpdateProjectMetadata(ObjectMap options) throws StorageEngineException { ProjectMetadata projectMetadata = getProjectMetadata(); + + checkSameSpeciesAndAssembly(options, projectMetadata); if (options != null && (projectMetadata == null || StringUtils.isEmpty(projectMetadata.getSpecies()) && options.containsKey(SPECIES.key()) || StringUtils.isEmpty(projectMetadata.getAssembly()) && options.containsKey(ASSEMBLY.key()))) { projectMetadata = updateProjectMetadata(pm -> { + // Check again, in case it was updated by another thread + checkSameSpeciesAndAssembly(options, pm); if (pm == null) { pm = new ProjectMetadata(); } @@ -598,6 +602,25 @@ public ProjectMetadata getAndUpdateProjectMetadata(ObjectMap options) throws Sto return projectMetadata; } + private static void checkSameSpeciesAndAssembly(ObjectMap options, ProjectMetadata projectMetadata) throws StorageEngineException { + if (options != null && projectMetadata != null) { + if (options.containsKey(ASSEMBLY.key())) { + if (StringUtils.isNotEmpty(projectMetadata.getAssembly()) && !projectMetadata.getAssembly() + .equalsIgnoreCase(options.getString(ASSEMBLY.key()))) { + throw new StorageEngineException("Incompatible assembly change from '" + projectMetadata.getAssembly() + "' to '" + + options.getString(ASSEMBLY.key()) + "'"); + } + } + if (options.containsKey(SPECIES.key())) { + if (StringUtils.isNotEmpty(projectMetadata.getSpecies()) && !projectMetadata.getSpecies() + .equalsIgnoreCase(toCellBaseSpeciesName(options.getString(SPECIES.key())))) { + throw new StorageEngineException("Incompatible species change from '" + projectMetadata.getSpecies() + "' to '" + + options.getString(SPECIES.key()) + "'"); + } + } + } + } + public DataResult getVariantFileMetadata(int studyId, int fileId, QueryOptions options) throws StorageEngineException { return fileDBAdaptor.getVariantFileMetadata(studyId, fileId, options); @@ -1671,8 +1694,6 @@ public int registerSecondaryIndexSamples(int studyId, List samples, bool CohortMetadata secondaryIndexCohort = getCohortMetadata(studyId, id); if (secondaryIndexCohort.getSamples().size() != sampleIds.size() || !secondaryIndexCohort.getSamples().containsAll(sampleIds)) { - System.out.println("secondaryIndexCohort = " + secondaryIndexCohort.getSamples()); - System.out.println("sampleIds = " + sampleIds); throw new StorageEngineException("Must provide all the samples from the secondary index: " + secondaryIndexCohort.getSamples() .stream() diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java index 0f51fbe39eb..77327d9d76a 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java @@ -24,13 +24,16 @@ import org.opencb.biodata.models.variant.avro.VariantAnnotation; import org.opencb.biodata.models.variant.metadata.SampleVariantStats; import org.opencb.biodata.models.variant.metadata.VariantMetadata; +import org.opencb.biodata.tools.variant.normalizer.extensions.VariantNormalizerExtensionFactory; import org.opencb.cellbase.client.config.ClientConfiguration; import org.opencb.cellbase.client.rest.CellBaseClient; import org.opencb.commons.datastore.core.*; +import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.storage.StorageConfiguration; import org.opencb.opencga.core.models.operations.variant.VariantAggregateFamilyParams; import org.opencb.opencga.core.models.operations.variant.VariantAggregateParams; +import org.opencb.opencga.core.models.variant.VariantSetupParams; import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.StorageEngine; import org.opencb.opencga.storage.core.StoragePipelineResult; @@ -153,7 +156,12 @@ public static SplitData from(ObjectMap options) { String loadSplitDataStr = options.getString(LOAD_SPLIT_DATA.key()); boolean multiFile = options.getBoolean(LOAD_MULTI_FILE_DATA.key()); if (StringUtils.isNotEmpty(loadSplitDataStr) && multiFile) { - throw new IllegalArgumentException("Unable to mix loadSplitFile and loadMultiFile"); + if (loadSplitDataStr.equalsIgnoreCase("multi")) { + return MULTI; + } else { + throw new IllegalArgumentException("Unable to mix " + LOAD_MULTI_FILE_DATA.key() + "=true and " + + LOAD_SPLIT_DATA.key() + "='" + loadSplitDataStr + "'"); + } } if (StringUtils.isEmpty(loadSplitDataStr) && !multiFile) { return null; @@ -1362,6 +1370,19 @@ public VariantQueryExecutor getVariantQueryExecutor(ParsedVariantQuery variantQu throw new VariantQueryException("No VariantQueryExecutor found to run the query!"); } + public final VariantQueryExecutor getVariantQueryExecutor(Class clazz) + throws StorageEngineException { + Optional first = getVariantQueryExecutors() + .stream() + .filter(e -> e instanceof SearchIndexVariantQueryExecutor) + .findFirst(); + if (first.isPresent()) { + return first.get(); + } else { + throw new StorageEngineException("VariantQueryExecutor " + clazz + " not found"); + } + } + public Query preProcessQuery(Query originalQuery, QueryOptions options) { try { return getVariantQueryParser().preProcessQuery(originalQuery, options); @@ -1485,6 +1506,45 @@ public VariantAggregationExecutor getVariantAggregationExecutor(Query query, Que throw new VariantQueryException("No VariantAggregationExecutor found to run the query. " + messages).setQuery(query); } + public ObjectMap inferConfigurationParams(VariantSetupParams params) { + ObjectMap options = new ObjectMap(); + + List normalizeExtensions = params.getNormalizeExtensions(); + if (normalizeExtensions != null && !normalizeExtensions.isEmpty()) { + if (!normalizeExtensions.equals(Collections.singletonList(ParamConstants.ALL))) { + List unsupportedExtensions = new ArrayList<>(); + for (String normalizeExtension : normalizeExtensions) { + if (!VariantNormalizerExtensionFactory.ALL_EXTENSIONS.contains(normalizeExtension)) { + unsupportedExtensions.add(normalizeExtension); + } + } + if (!unsupportedExtensions.isEmpty()) { + throw new IllegalArgumentException("Unsupported normalize extensions: " + unsupportedExtensions + ". Supported " + + "extensions are: " + VariantNormalizerExtensionFactory.ALL_EXTENSIONS); + } + } + options.put(NORMALIZATION_EXTENSIONS.key(), normalizeExtensions); + } + if (params.getDataDistribution() != null) { + switch (params.getDataDistribution()) { + case FILES_SPLIT_BY_CHROMOSOME: + options.put(LOAD_SPLIT_DATA.key(), SplitData.CHROMOSOME); + break; + case FILES_SPLIT_BY_REGION: + options.put(LOAD_SPLIT_DATA.key(), SplitData.REGION); + break; + case MULTIPLE_FILES_PER_SAMPLE: + options.put(LOAD_MULTI_FILE_DATA.key(), true); + options.put(LOAD_SPLIT_DATA.key(), SplitData.MULTI); + break; + default: + break; + } + } + + return options; + } + @Override public void close() throws IOException { cellBaseUtils = null; diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageOptions.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageOptions.java index bc1be055466..847ae860d70 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageOptions.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageOptions.java @@ -1,10 +1,11 @@ package org.opencb.opencga.storage.core.variant; import org.opencb.biodata.models.variant.metadata.Aggregation; -import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.YesNoAuto; import org.opencb.opencga.core.config.ConfigurationOption; +import java.util.Arrays; + public enum VariantStorageOptions implements ConfigurationOption { STUDY("study"), @@ -26,7 +27,7 @@ public enum VariantStorageOptions implements ConfigurationOption { TRANSFORM_ISOLATE("transform.isolate", false), // Do not store file in metadata NORMALIZATION_SKIP("normalization.skip", false), // Do not run normalization NORMALIZATION_REFERENCE_GENOME("normalization.referenceGenome"), - NORMALIZATION_EXTENSIONS("normalization.extensions", ParamConstants.NONE), + NORMALIZATION_EXTENSIONS("normalization.extensions", Arrays.asList("VAF", "SV", "CUSTOM")), DEDUPLICATION_POLICY("deduplication.policy", "maxQual"), DEDUPLICATION_BUFFER_SIZE("deduplication.bufferSize", 100), diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStoragePipeline.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStoragePipeline.java index 5b37d2512b1..34bbd5cfe5a 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStoragePipeline.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStoragePipeline.java @@ -60,10 +60,7 @@ import org.opencb.opencga.storage.core.io.plain.StringDataReader; import org.opencb.opencga.storage.core.io.plain.StringDataWriter; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; -import org.opencb.opencga.storage.core.metadata.models.CohortMetadata; -import org.opencb.opencga.storage.core.metadata.models.FileMetadata; -import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; -import org.opencb.opencga.storage.core.metadata.models.TaskMetadata; +import org.opencb.opencga.storage.core.metadata.models.*; import org.opencb.opencga.storage.core.variant.adaptors.GenotypeClass; import org.opencb.opencga.storage.core.variant.adaptors.VariantDBAdaptor; import org.opencb.opencga.storage.core.variant.io.VariantReaderUtils; @@ -78,6 +75,7 @@ import java.io.UncheckedIOException; import java.net.URI; import java.nio.ByteBuffer; +import java.nio.file.Paths; import java.util.*; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicInteger; @@ -424,15 +422,22 @@ protected Task initNormalizer(VariantFileMetadata metadata) th .then(new VariantSorterTask(100)) // Sort before generating reference blocks .then(new VariantReferenceBlockCreatorTask(metadata.getHeader())); } - if (CollectionUtils.isNotEmpty(enabledExtensions)) { + if (CollectionUtils.isEmpty(enabledExtensions)) { + enabledExtensions = NORMALIZATION_EXTENSIONS.defaultValue(); + } + if ((enabledExtensions.size() == 1 && enabledExtensions.contains(ParamConstants.NONE))) { + logger.info("Skip normalization extensions"); + } else { + logger.info("Enable normalization extensions: {}", enabledExtensions); VariantNormalizerExtensionFactory extensionFactory; if (enabledExtensions.size() == 1 && enabledExtensions.contains(ParamConstants.ALL)) { - extensionFactory = new VariantNormalizerExtensionFactory(); - } else { - extensionFactory = new VariantNormalizerExtensionFactory(new HashSet<>(enabledExtensions)); + enabledExtensions = NORMALIZATION_EXTENSIONS.defaultValue(); } + extensionFactory = new VariantNormalizerExtensionFactory(new HashSet<>(enabledExtensions)); Task extension = extensionFactory.buildExtensions(metadata); - if (extension != null) { + if (extension == null) { + logger.info("No normalization extensions can be used."); + } else { normalizer = normalizer.then(extension); } } @@ -561,17 +566,19 @@ public URI preLoad(URI input, URI output) throws StorageEngineException { return input; } - protected void preLoadRegisterAndValidateFile(int studyId, VariantFileMetadata fileMetadata) throws StorageEngineException { + protected void preLoadRegisterAndValidateFile(int studyId, VariantFileMetadata variantFileMetadata) throws StorageEngineException { final int fileId; String virtualFile = options.getString(LOAD_VIRTUAL_FILE.key()); + boolean loadSampleIndex = YesNoAuto.parse(options, LOAD_SAMPLE_INDEX.key()).orYes().booleanValue(); + VariantStorageEngine.SplitData splitData = VariantStorageEngine.SplitData.from(options); - if (VariantStorageEngine.SplitData.isPartialSplit(options)) { + if (VariantStorageEngine.SplitData.isPartialSplit(splitData)) { if (StringUtils.isEmpty(virtualFile)) { - fileId = getMetadataManager().registerFile(studyId, fileMetadata); + fileId = getMetadataManager().registerFile(studyId, variantFileMetadata); // throw new StorageEngineException("Unable to load file with 'split-data'. Missing virtual file belonging! " // + "Please, define " + LOAD_VIRTUAL_FILE.key()); } else { - fileId = getMetadataManager().registerPartialFile(studyId, virtualFile, fileMetadata); + fileId = getMetadataManager().registerPartialFile(studyId, virtualFile, variantFileMetadata); } } else { if (StringUtils.isNotEmpty(virtualFile)) { @@ -580,10 +587,85 @@ protected void preLoadRegisterAndValidateFile(int studyId, VariantFileMetadata f + " to " + VariantStorageEngine.SplitData.REGION + " or " + VariantStorageEngine.SplitData.CHROMOSOME); } else { - fileId = getMetadataManager().registerFile(studyId, fileMetadata); + fileId = getMetadataManager().registerFile(studyId, variantFileMetadata); } } setFileId(fileId); + FileMetadata fileMetadata = getMetadataManager().getFileMetadata(studyId, getFileId()); + + int version = getMetadataManager().getStudyMetadata(studyId).getSampleIndexConfigurationLatest().getVersion(); + Set alreadyIndexedSamples = new LinkedHashSet<>(); + Set processedSamples = new LinkedHashSet<>(); + Set samplesWithoutSplitData = new LinkedHashSet<>(); + for (String sample : variantFileMetadata.getSampleIds()) { + Integer sampleId = getMetadataManager().getSampleId(studyId, sample); + SampleMetadata sampleMetadata = getMetadataManager().getSampleMetadata(studyId, sampleId); + if (splitData != null && sampleMetadata.getSplitData() != null) { + if (splitData != sampleMetadata.getSplitData()) { + throw new StorageEngineException("Incompatible split data methods. " + + "Unable to mix requested " + splitData + + " with existing " + sampleMetadata.getSplitData()); + } + } + if (sampleMetadata.isIndexed()) { + if (sampleMetadata.getFiles().size() == 1 && sampleMetadata.getFiles().contains(fileMetadata.getId())) { + // It might happen that the sample is marked as INDEXED, but not the file. + // If the sample only belongs to this file (i.e. it's only file is this file), then ignore + // the overwrite the current sample metadata index status + sampleMetadata = getMetadataManager().updateSampleMetadata(studyId, sampleId, + sm -> sm.setIndexStatus(fileMetadata.getIndexStatus())); + } + } + if (sampleMetadata.isIndexed()) { + alreadyIndexedSamples.add(sample); + if (sampleMetadata.isAnnotated() + || !loadSampleIndex && sampleMetadata.getSampleIndexStatus(version) == TaskMetadata.Status.READY + || sampleMetadata.getSampleIndexAnnotationStatus(version) == TaskMetadata.Status.READY + || sampleMetadata.getFamilyIndexStatus(version) == TaskMetadata.Status.READY + || sampleMetadata.isFamilyIndexDefined()) { + processedSamples.add(sampleMetadata.getId()); + } + } + + if (splitData != null && splitData != sampleMetadata.getSplitData()) { + samplesWithoutSplitData.add(sampleId); + } + } + + if (!alreadyIndexedSamples.isEmpty()) { + if (splitData != null) { + logger.info("Loading split data"); + } else { + String fileName = Paths.get(variantFileMetadata.getPath()).getFileName().toString(); + throw StorageEngineException.alreadyLoadedSamples(fileName, new ArrayList<>(alreadyIndexedSamples)); + } + for (Integer sampleId : processedSamples) { + getMetadataManager().updateSampleMetadata(studyId, sampleId, sampleMetadata -> { + if (!loadSampleIndex) { + for (Integer v : sampleMetadata.getSampleIndexVersions()) { + sampleMetadata.setSampleIndexStatus(TaskMetadata.Status.NONE, v); + } + } + for (Integer v : sampleMetadata.getSampleIndexAnnotationVersions()) { + sampleMetadata.setSampleIndexAnnotationStatus(TaskMetadata.Status.NONE, v); + } + for (Integer v : sampleMetadata.getFamilyIndexVersions()) { + sampleMetadata.setFamilyIndexStatus(TaskMetadata.Status.NONE, v); + } + sampleMetadata.setAnnotationStatus(TaskMetadata.Status.NONE); + sampleMetadata.setMendelianErrorStatus(TaskMetadata.Status.NONE); + }); + } + } + + if (splitData != null) { + // Register loadSplitData + for (Integer sampleId : samplesWithoutSplitData) { + getMetadataManager().updateSampleMetadata(studyId, sampleId, sampleMetadata -> { + sampleMetadata.setSplitData(splitData); + }); + } + } } /** diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/GenotypeClass.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/GenotypeClass.java index d3f5e8dd1ef..db3375e5c68 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/GenotypeClass.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/GenotypeClass.java @@ -483,4 +483,14 @@ private static Genotype parseGenotype(String gt) { } return genotype; } + + public static Set classify(String gt) { + Set genotypeClasses = new HashSet<>(); + for (GenotypeClass value : values()) { + if (value.test(gt)) { + genotypeClasses.add(value); + } + } + return genotypeClasses; + } } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/VariantQueryParam.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/VariantQueryParam.java index c801f4448ff..1973d7f383d 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/VariantQueryParam.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/VariantQueryParam.java @@ -67,7 +67,8 @@ public final class VariantQueryParam implements QueryParam { public static final String STUDY_DESCR - = "Filter variants from the given studies, these can be either the numeric ID or the alias with the format user@project:study"; + = "Filter variants from the given studies, these can be either the numeric ID or the alias with the format " + + "organization@project:study"; public static final VariantQueryParam STUDY = new VariantQueryParam("study", TEXT_ARRAY, STUDY_DESCR); public static final String INCLUDE_STUDY_DESCR diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/annotation/annotators/AbstractCellBaseVariantAnnotator.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/annotation/annotators/AbstractCellBaseVariantAnnotator.java index da8d44b05b5..311e62d46ab 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/annotation/annotators/AbstractCellBaseVariantAnnotator.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/annotation/annotators/AbstractCellBaseVariantAnnotator.java @@ -213,6 +213,9 @@ protected List getVariantAnnotationList(List variant variantAnnotation.getAdditionalAttributes().put(GROUP_NAME.key(), additionalAttribute); } } + if (variantAnnotation.getConsequenceTypes() == null || variantAnnotation.getConsequenceTypes().isEmpty()) { + logger.warn("No consequence type found for variant " + variant); + } variantAnnotationList.add(variantAnnotation); } } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/io/json/VariantAnnotationJsonDataReader.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/io/json/VariantAnnotationJsonDataReader.java index 8d1cc1e4841..3dcb476f4e6 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/io/json/VariantAnnotationJsonDataReader.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/io/json/VariantAnnotationJsonDataReader.java @@ -79,7 +79,7 @@ public boolean open() { jsonObjectMapper.addMixIn(ConsequenceType.class, ConsequenceTypeMixin.class); jsonObjectMapper.configure(MapperFeature.REQUIRE_SETTERS_FOR_GETTERS, true); try { - parser = factory.createParser(inputStream); + parser = jsonObjectMapper.createParser(inputStream); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/accumulators/FacetFieldAccumulator.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/accumulators/FacetFieldAccumulator.java index a8141525691..bdf6d88437e 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/accumulators/FacetFieldAccumulator.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/accumulators/FacetFieldAccumulator.java @@ -70,11 +70,13 @@ public void evaluate(FacetField field) { * Accumulate T in the given field. * @param field Field * @param t element + * @return true if the count was increased, false otherwise */ - public final void accumulate(FacetField field, T t) { + public final boolean accumulate(FacetField field, T t) { List buckets = getBuckets(field, t); if (buckets == null || buckets.isEmpty()) { - return; + // Do not increase count if the element does not belong to any bucket + return false; } field.addCount(1); for (FacetField.Bucket bucket : buckets) { @@ -83,7 +85,17 @@ public final void accumulate(FacetField field, T t) { nestedFieldAccumulator.accumulate(bucket.getFacetFields().get(0), t); } } + return true; } protected abstract List getBuckets(FacetField field, T t); + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("FacetFieldAccumulator{name:'"); + sb.append(getName()).append('\''); + sb.append(", nestedFieldAccumulator:").append(nestedFieldAccumulator); + sb.append('}'); + return sb.toString(); + } } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SearchIndexVariantQueryExecutor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SearchIndexVariantQueryExecutor.java index bd7c79cb2e9..ccaae7dbe39 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SearchIndexVariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SearchIndexVariantQueryExecutor.java @@ -183,7 +183,9 @@ public VariantQueryResult approximateCount(ParsedVariantQuery variantQuery DataResult nativeResult = searchManager .nativeQuery(dbName, searchEngineQuery, queryOptions); - List variantIds = nativeResult.getResults().stream().map(VariantSearchModel::getId).collect(Collectors.toList()); + List variantIds = nativeResult.getResults().stream() + .map(VariantSearchModel::toVariantSimple) + .collect(Collectors.toList()); // Adjust numSamples if the results from SearchManager is smaller than numSamples // If this happens, the count is not approximated if (variantIds.size() < sampling) { @@ -287,12 +289,12 @@ public boolean doIntersectWithSearch(Query query, QueryOptions options) { return intersect; } - protected Iterator variantIdIteratorFromSearch(Query query) { + protected Iterator variantIdIteratorFromSearch(Query query) { return variantIdIteratorFromSearch(query, Integer.MAX_VALUE, 0, null); } - protected Iterator variantIdIteratorFromSearch(Query query, int limit, int skip, AtomicLong numTotalResults) { - Iterator variantsIterator; + protected Iterator variantIdIteratorFromSearch(Query query, int limit, int skip, AtomicLong numTotalResults) { + Iterator variantsIterator; QueryOptions queryOptions = new QueryOptions() .append(QueryOptions.LIMIT, limit) .append(QueryOptions.SKIP, skip) @@ -306,14 +308,14 @@ protected Iterator variantIdIteratorFromSearch(Query query, int limit, i } variantsIterator = nativeResult.getResults() .stream() - .map(VariantSearchModel::getId) + .map(VariantSearchModel::toVariantSimple) .iterator(); } else { SolrNativeIterator nativeIterator = searchManager.nativeIterator(dbName, query, queryOptions); if (numTotalResults != null) { numTotalResults.set(nativeIterator.getNumFound()); } - variantsIterator = Iterators.transform(nativeIterator, VariantSearchModel::getId); + variantsIterator = Iterators.transform(nativeIterator, VariantSearchModel::toVariantSimple); } } catch (VariantSearchException | IOException e) { throw new VariantQueryException("Error querying " + VariantSearchManager.SEARCH_ENGINE_ID, e); diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchModel.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchModel.java index 9b0bb69792c..835af18a0a0 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchModel.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchModel.java @@ -17,12 +17,15 @@ package org.opencb.opencga.storage.core.variant.search; import org.apache.solr.client.solrj.beans.Field; +import org.opencb.biodata.models.variant.Variant; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import static org.opencb.opencga.storage.core.variant.search.VariantSearchToVariantConverter.HASH_PREFIX; + /** * Created by wasim on 09/11/16. */ @@ -140,6 +143,9 @@ public class VariantSearchModel { @Field("fileInfo_*") private Map fileInfo; + @Field("attr_*") + private Map attr; + public static final double MISSING_VALUE = -100.0; @@ -171,6 +177,7 @@ public VariantSearchModel() { this.qual = new HashMap<>(); this.filter = new HashMap<>(); this.fileInfo = new HashMap<>(); + this.attr = new HashMap<>(); } public VariantSearchModel(VariantSearchModel init) { @@ -210,6 +217,7 @@ public VariantSearchModel(VariantSearchModel init) { this.qual = init.getQual(); this.filter = init.getFilter(); this.fileInfo = init.getFileInfo(); + this.attr = init.getAttr(); } @Override @@ -251,6 +259,7 @@ public String toString() { sb.append(", qual=").append(qual); sb.append(", filter=").append(filter); sb.append(", fileInfo=").append(fileInfo); + sb.append(", attr=").append(attr); sb.append('}'); return sb.toString(); } @@ -259,6 +268,17 @@ public String getId() { return id; } + public Variant toVariantSimple() { + String variantId = getId(); + if (variantId.startsWith(HASH_PREFIX)) { + Object o = getAttr().get("attr_id"); + variantId = o instanceof String ? (String) o : ((List) o).get(0); + } + Variant variant = new Variant(variantId); + variant.setId(variantId); + return variant; + } + public VariantSearchModel setId(String id) { this.id = id; return this; @@ -579,4 +599,12 @@ public VariantSearchModel setFileInfo(Map fileInfo) { return this; } + public Map getAttr() { + return attr; + } + + public VariantSearchModel setAttr(Map attr) { + this.attr = attr; + return this; + } } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchToVariantConverter.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchToVariantConverter.java index 62841c0a3b9..d9c119c2340 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchToVariantConverter.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchToVariantConverter.java @@ -56,6 +56,7 @@ public class VariantSearchToVariantConverter implements ComplexTypeConverter includeFields; @@ -79,10 +80,9 @@ public VariantSearchToVariantConverter(Set includeFields) { @Override public Variant convertToDataModelType(VariantSearchModel variantSearchModel) { // set chromosome, start, end, ref, alt from ID - Variant variant = new Variant(variantSearchModel.getId()); + Variant variant = variantSearchModel.toVariantSimple(); - // set ID, chromosome, start, end, ref, alt, type - variant.setId(variantSearchModel.getVariantId()); + // set chromosome, start, end, ref, alt, type // set variant type if (StringUtils.isNotEmpty(variantSearchModel.getType())) { @@ -662,8 +662,10 @@ public VariantSearchModel convertToStorageType(Variant variant) { List other = new ArrayList<>(); // Set general Variant attributes: id, dbSNP, chromosome, start, end, type - variantSearchModel.setId(variant.toString()); // Internal unique ID e.g. 3:1000:AT:- - variantSearchModel.setVariantId(variant.getId()); + String variantId = getVariantId(variant); + variantSearchModel.setId(variantId); // Internal unique ID e.g. 3:1000:AT:- + variantSearchModel.setVariantId(variantId); + variantSearchModel.getAttr().put("attr_id", variant.toString()); variantSearchModel.setChromosome(variant.getChromosome()); variantSearchModel.setStart(variant.getStart()); variantSearchModel.setEnd(variant.getEnd()); @@ -759,7 +761,12 @@ public VariantSearchModel convertToStorageType(Variant variant) { // Remove 'SO:' prefix to Store SO Accessions as integers and also store the gene - SO acc relation for (SequenceOntologyTerm sequenceOntologyTerm : conseqType.getSequenceOntologyTerms()) { - int soIdInt = Integer.parseInt(sequenceOntologyTerm.getAccession().substring(3)); + int soIdInt; + try { + soIdInt = Integer.parseInt(sequenceOntologyTerm.getAccession().substring(3)); + } catch (NumberFormatException e) { + soIdInt = ConsequenceTypeMappings.termToAccession.get(sequenceOntologyTerm.getName()); + } soAccessions.add(soIdInt); if (StringUtils.isNotEmpty(gene)) { @@ -1019,8 +1026,7 @@ public VariantSearchModel convertToStorageType(Variant variant) { // This field contains all possible IDs: id, dbSNP, names, genes, transcripts, protein, clinvar, hpo, ... // This will help when searching by variant id. This is added at the end of the method after collecting all IDs Set xrefs = variantAnnotationModelUtils.extractXRefs(variant.getAnnotation()); - xrefs.add(variantSearchModel.getId()); - xrefs.add(variantSearchModel.getVariantId()); + xrefs.add(variantId); if (variant.getNames() != null && !variant.getNames().isEmpty()) { variant.getNames().forEach(name -> { if (name != null) { @@ -1032,6 +1038,20 @@ public VariantSearchModel convertToStorageType(Variant variant) { return variantSearchModel; } + public static String getVariantId(Variant variant) { + String variantString = variant.toString(); + if (variantString.length() > 32766) { + // variantString.length() >= Short.MAX_VALUE + return hashVariantId(variant, variantString); + } else { + return variantString; + } + } + + public static String hashVariantId(Variant variant, String variantString) { + return HASH_PREFIX + variant.getChromosome() + ":" + variant.getStart() + ":" + Integer.toString(variantString.hashCode()); + } + private void convertStudies(Variant variant, VariantSearchModel variantSearchModel, List other) { // Sanity check if (CollectionUtils.isEmpty(variant.getStudies())) { diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/solr/SolrQueryParser.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/solr/SolrQueryParser.java index 0cf045ada4c..f12bb85a8b1 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/solr/SolrQueryParser.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/solr/SolrQueryParser.java @@ -25,7 +25,6 @@ import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.common.SolrException; import org.opencb.biodata.models.core.Region; -import org.opencb.biodata.models.variant.Variant; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.solr.FacetQueryParser; @@ -35,7 +34,10 @@ import org.opencb.opencga.storage.core.variant.adaptors.VariantField; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryException; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; -import org.opencb.opencga.storage.core.variant.query.*; +import org.opencb.opencga.storage.core.variant.query.KeyOpValue; +import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; +import org.opencb.opencga.storage.core.variant.query.Values; +import org.opencb.opencga.storage.core.variant.query.VariantQueryParser; import org.opencb.opencga.storage.core.variant.query.projection.VariantQueryProjectionParser; import org.opencb.opencga.storage.core.variant.search.VariantSearchToVariantConverter; import org.slf4j.Logger; @@ -79,7 +81,7 @@ public class SolrQueryParser { static { includeMap = new HashMap<>(); - includeMap.put("id", "id,variantId"); + includeMap.put("id", "id,variantId,attr_id"); includeMap.put("chromosome", "chromosome"); includeMap.put("start", "start"); includeMap.put("end", "end"); @@ -477,7 +479,9 @@ private String parseGenomicFilter(Query query) { genes.addAll(variantQueryXref.getGenes()); xrefs.addAll(variantQueryXref.getIds()); xrefs.addAll(variantQueryXref.getOtherXrefs()); - xrefs.addAll(variantQueryXref.getVariants().stream().map(Variant::toString).collect(Collectors.toList())); + xrefs.addAll(variantQueryXref.getVariants().stream() + .map(VariantSearchToVariantConverter::getVariantId) + .collect(Collectors.toList())); // Regions if (StringUtils.isNotEmpty(query.getString(REGION.key()))) { @@ -492,7 +496,7 @@ private String parseGenomicFilter(Query query) { // Consequence types (cts) String ctLogicalOperator = " OR "; if (StringUtils.isNotEmpty(query.getString(ANNOT_CONSEQUENCE_TYPE.key(), ""))) { - consequenceTypes = Arrays.asList(query.getString(ANNOT_CONSEQUENCE_TYPE.key()).split("[,;]")); + consequenceTypes = parseConsequenceTypes(Arrays.asList(query.getString(ANNOT_CONSEQUENCE_TYPE.key()).split("[,;]"))); if (query.getString(ANNOT_CONSEQUENCE_TYPE.key()).contains(";")) { ctLogicalOperator = " AND "; // TODO This must be removed as soon as we have the Query procesing in use @@ -1616,15 +1620,12 @@ private String[] includeFieldsWithMandatory(String[] includes) { return new String[0]; } - String[] mandatoryIncludeFields = new String[]{"id", "chromosome", "start", "end", "type"}; - String[] includeWithMandatory = new String[includes.length + mandatoryIncludeFields.length]; - for (int i = 0; i < includes.length; i++) { - includeWithMandatory[i] = includes[i]; - } - for (int i = 0; i < mandatoryIncludeFields.length; i++) { - includeWithMandatory[includes.length + i] = mandatoryIncludeFields[i]; - } - return includeWithMandatory; + Set mandatoryIncludeFields = new HashSet<>(Arrays.asList("id", "attr_id", "chromosome", "start", "end", "type")); + Set includeWithMandatory = new LinkedHashSet<>(includes.length + mandatoryIncludeFields.size()); + + includeWithMandatory.addAll(Arrays.asList(includes)); + includeWithMandatory.addAll(mandatoryIncludeFields); + return includeWithMandatory.toArray(new String[0]); } /** diff --git a/opencga-storage/opencga-storage-core/src/main/resources/storage-configuration.yml b/opencga-storage/opencga-storage-core/src/main/resources/storage-configuration.yml index a2f894900b3..b9970d18eaf 100644 --- a/opencga-storage/opencga-storage-core/src/main/resources/storage-configuration.yml +++ b/opencga-storage/opencga-storage-core/src/main/resources/storage-configuration.yml @@ -171,6 +171,7 @@ variant: #storage.hadoop.mr.executor.ssh.key: "~/.ssh/id_rsa" # Hadoop edge node ssh-key file storage.hadoop.mr.executor.ssh.password: "" # Hadoop edge node password. Only if ssh-key is not present. Requires sshpass to run storage.hadoop.mr.executor.ssh.remoteOpenCgaHome: # Remote opencga home location. Only if different than local location. + storage.hadoop.mr.executor.ssh.terminationGracePeriodSeconds: 120 # Termination grace period in seconds for the ssh executor. # Increase the ScannerTimeoutPeriod from 60000 (1min) to 300000 (5min) to avoid ScannerTimeoutExceptions # See opencb/opencga#352 for more info. diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/metadata/VariantMetadataConverterTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/metadata/VariantMetadataConverterTest.java index 2aacde2e3ce..30e4bba2e5f 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/metadata/VariantMetadataConverterTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/metadata/VariantMetadataConverterTest.java @@ -9,6 +9,7 @@ import org.junit.experimental.categories.Category; import org.opencb.biodata.models.variant.VariantFileMetadata; import org.opencb.biodata.models.variant.metadata.VariantMetadata; +import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.core.testclassification.duration.ShortTests; import org.opencb.opencga.storage.core.io.managers.IOConnectorProvider; import org.opencb.opencga.storage.core.io.managers.LocalIOConnector; @@ -45,6 +46,7 @@ public class VariantMetadataConverterTest { @Before public void setUp() throws Exception { metadataManager = new VariantStorageMetadataManager(new DummyVariantStorageMetadataDBAdaptorFactory()); + projectMetadata = metadataManager.getAndUpdateProjectMetadata(new ObjectMap()); URI uri = VariantStorageBaseTest.getResourceUri("platinum/1K.end.platinum-genomes-vcf-NA12877_S1.genome.vcf.gz"); variantReaderUtils = new VariantReaderUtils(new IOConnectorProvider(LocalIOConnector.class)); diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/utils/CellBaseUtilsTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/utils/CellBaseUtilsTest.java index 2fa0ae32221..ce6dad773e7 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/utils/CellBaseUtilsTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/utils/CellBaseUtilsTest.java @@ -1,6 +1,6 @@ package org.opencb.opencga.storage.core.utils; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.hamcrest.CoreMatchers; import org.junit.*; import org.junit.experimental.categories.Category; diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageBaseTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageBaseTest.java index fe5313bdc80..23ece394c79 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageBaseTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageBaseTest.java @@ -374,6 +374,11 @@ public static StoragePipelineResult runDefaultETL(URI inputUri, VariantStorageEn return storagePipelineResult; } + public static StoragePipelineResult runETL(VariantStorageEngine variantStorageManager, URI inputUri, URI outputUri, + ObjectMap params) throws StorageEngineException, FileFormatException, IOException { + return runETL(variantStorageManager, inputUri, outputUri, params, true, true, true); + } + public static StoragePipelineResult runETL(VariantStorageEngine variantStorageManager, URI inputUri, URI outputUri, ObjectMap params, boolean doExtract, diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineBNDTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineBNDTest.java index 32e1657fcb8..685291398ef 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineBNDTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineBNDTest.java @@ -48,6 +48,8 @@ public void before() throws Exception { variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL); variantStorageEngine.getConfiguration().getCellbase().setVersion(ParamConstants.CELLBASE_VERSION); variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE_GRCH38); + variantStorageEngine.getOptions().put(VariantStorageOptions.ASSEMBLY.key(), "grch38"); + if (!loaded) { clearDB(DB_NAME); loadFiles(); @@ -59,6 +61,8 @@ protected void loadFiles() throws Exception { variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL); variantStorageEngine.getConfiguration().getCellbase().setVersion(ParamConstants.CELLBASE_VERSION); variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE_GRCH38); + variantStorageEngine.getOptions().put(VariantStorageOptions.ASSEMBLY.key(), "grch38"); + studyMetadata = new StudyMetadata(1, "s1"); // variantStorageEngine.getOptions().append(VariantStorageOptions.ANNOTATOR_CELLBASE_EXCLUDE.key(), "expression,clinical"); input1 = getResourceUri("variant-test-bnd.vcf"); diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java index 8585b396ab4..af97483d71d 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java @@ -1,5 +1,6 @@ package org.opencb.opencga.storage.core.variant; +import org.junit.Assume; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; @@ -19,6 +20,10 @@ import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIterator; import org.opencb.opencga.storage.core.variant.io.VariantWriterFactory; +import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; +import org.opencb.opencga.storage.core.variant.query.executors.VariantQueryExecutor; +import org.opencb.opencga.storage.core.variant.search.SearchIndexVariantQueryExecutor; import java.net.URI; import java.nio.file.Paths; @@ -52,18 +57,21 @@ public abstract class VariantStorageEngineSVTest extends VariantStorageBaseTest public void before() throws Exception { if (!loaded) { clearDB(DB_NAME); - loadFiles(); - loaded = true; } - } - - protected void loadFiles() throws Exception { variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL); variantStorageEngine.getConfiguration().getCellbase().setVersion(ParamConstants.CELLBASE_VERSION); variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE_GRCH38); variantStorageEngine.getOptions().put(VariantStorageOptions.ASSEMBLY.key(), "grch38"); variantStorageEngine.reloadCellbaseConfiguration(); + if (!loaded) { + loadFiles(); + loaded = true; + } + } + + protected void loadFiles() throws Exception { + input1 = getResourceUri("variant-test-sv.vcf"); studyMetadata = new StudyMetadata(1, "s1"); variantStorageEngine.getOptions().append(VariantStorageOptions.ANNOTATOR_CELLBASE_EXCLUDE.key(), "expression,clinical"); @@ -105,6 +113,19 @@ public void checkCount() throws Exception { assertEquals(expected, count); } + @Test + public void checkSecondaryAnnotationIndex() throws Exception { + Assume.assumeTrue(variantStorageEngine.secondaryAnnotationIndexActiveAndAlive()); + VariantQueryExecutor variantQueryExecutor = variantStorageEngine.getVariantQueryExecutor(SearchIndexVariantQueryExecutor.class); + for (Variant variant : variantStorageEngine) { + ParsedVariantQuery query = variantStorageEngine + .parseQuery(new Query(VariantQueryParam.ID.key(), variant.toString()), new QueryOptions()); + VariantQueryResult result = variantQueryExecutor.get(query); + assertEquals(1, result.getNumResults()); + assertEquals(variant.toString(), result.first().toString()); + } + } + @Test public void checkCorrectnessFile1() throws Exception { checkCorrectness(VariantStorageEngineSVTest.input1); diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageSearchIntersectTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageSearchIntersectTest.java index 58bae56a733..28b7d0921ab 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageSearchIntersectTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageSearchIntersectTest.java @@ -65,7 +65,7 @@ @Ignore public abstract class VariantStorageSearchIntersectTest extends VariantStorageBaseTest { - @ClassRule + @ClassRule(order = 10) public static VariantSolrExternalResource solr = new VariantSolrExternalResource(); protected VariantDBAdaptor dbAdaptor; diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/GenotypeClassTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/GenotypeClassTest.java index f10ea441211..b504ac80753 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/GenotypeClassTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/GenotypeClassTest.java @@ -5,7 +5,9 @@ import org.opencb.opencga.core.testclassification.duration.ShortTests; import java.util.Arrays; +import java.util.HashSet; import java.util.List; +import java.util.Set; import static org.junit.Assert.assertEquals; @@ -28,7 +30,41 @@ public void testGenotypes() throws Exception { assertEquals(Arrays.asList("./1", "1/."), GenotypeClass.HET_MISS.filter(gts)); assertEquals(Arrays.asList("./.", "."), GenotypeClass.MISS.filter(gts)); assertEquals(Arrays.asList("0/2", "2/2", "2/3", "2/."), GenotypeClass.SEC_ALT.filter(gts)); + } + + @Test + public void classify() { + checkClassify("0/0", GenotypeClass.HOM_REF); + checkClassify("0|0", GenotypeClass.HOM_REF); + checkClassify("0", GenotypeClass.HOM_REF); + checkClassify("1/1", GenotypeClass.MAIN_ALT, GenotypeClass.HOM_ALT); + checkClassify("1/1/1", GenotypeClass.MAIN_ALT, GenotypeClass.HOM_ALT); + checkClassify("1", GenotypeClass.MAIN_ALT, GenotypeClass.HOM_ALT); + checkClassify("0/1", GenotypeClass.MAIN_ALT, GenotypeClass.HET_REF, GenotypeClass.HET); + checkClassify("1/0", GenotypeClass.MAIN_ALT, GenotypeClass.HET_REF, GenotypeClass.HET); + checkClassify("0|1", GenotypeClass.MAIN_ALT, GenotypeClass.HET_REF, GenotypeClass.HET); + checkClassify("1|0", GenotypeClass.MAIN_ALT, GenotypeClass.HET_REF, GenotypeClass.HET); + checkClassify("1/2", GenotypeClass.MAIN_ALT, GenotypeClass.SEC, GenotypeClass.HET, GenotypeClass.HET_ALT); + checkClassify("1/4", GenotypeClass.MAIN_ALT, GenotypeClass.SEC, GenotypeClass.HET, GenotypeClass.HET_ALT); + checkClassify("3/4", GenotypeClass.SEC_ALT, GenotypeClass.SEC); + checkClassify("3/4/5", GenotypeClass.SEC_ALT, GenotypeClass.SEC, GenotypeClass.HET); + checkClassify("561/941", GenotypeClass.SEC_ALT, GenotypeClass.SEC); + checkClassify("561/1", GenotypeClass.MAIN_ALT, GenotypeClass.SEC, GenotypeClass.HET, GenotypeClass.HET_ALT); + checkClassify("0/2", GenotypeClass.SEC_ALT, GenotypeClass.SEC); + checkClassify("0/3", GenotypeClass.SEC_ALT, GenotypeClass.SEC); + checkClassify("3/3", GenotypeClass.SEC_ALT, GenotypeClass.SEC); + checkClassify("1/.", GenotypeClass.MAIN_ALT, GenotypeClass.HET_MISS, GenotypeClass.HET); + checkClassify("0/."); + checkClassify("./.", GenotypeClass.MISS); + checkClassify(".", GenotypeClass.MISS); + checkClassify("NA", GenotypeClass.NA); + checkClassify("THIS_IS_NOT_A_GENOTYPE"); + } + private void checkClassify(String gt, GenotypeClass... expected) { + Set actual = GenotypeClass.classify(gt); +// System.out.println("GenotypeClass.classify(" + gt + ") = " + actual); + assertEquals(new HashSet<>(Arrays.asList(expected)), actual); } @Test diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantDBAdaptorMultiFileTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantDBAdaptorMultiFileTest.java index 903ba51f3a2..8071b86a14e 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantDBAdaptorMultiFileTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantDBAdaptorMultiFileTest.java @@ -1,6 +1,6 @@ package org.opencb.opencga.storage.core.variant.adaptors; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; @@ -145,10 +145,6 @@ public void testRelease() throws Exception { Integer minFileId = variant.getStudies().stream() .flatMap(s -> s.getFiles().stream()) .map(FileEntry::getFileId) - .map(s->{ - System.out.println("s = " + s); - return s; - }) .map(s -> StringUtils.removeStart(s, "1K.end.platinum-genomes-vcf-NA")) .map(s -> StringUtils.removeEnd(s, "_S1.genome.vcf.gz")) .map(Integer::valueOf) diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/annotation/annotators/VariantAnnotatorByApiKeyTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/annotation/annotators/VariantAnnotatorByApiKeyTest.java index 7365159a501..6ece2f05138 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/annotation/annotators/VariantAnnotatorByApiKeyTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/annotation/annotators/VariantAnnotatorByApiKeyTest.java @@ -1,16 +1,18 @@ package org.opencb.opencga.storage.core.variant.annotation.annotators; import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.junit.Assume; import org.junit.Before; import org.junit.Test; +import org.junit.experimental.categories.Category; import org.opencb.biodata.models.variant.Variant; import org.opencb.biodata.models.variant.avro.EvidenceEntry; import org.opencb.biodata.models.variant.avro.VariantAnnotation; import org.opencb.cellbase.client.rest.CellBaseClient; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.core.config.storage.StorageConfiguration; +import org.opencb.opencga.core.testclassification.duration.MediumTests; import org.opencb.opencga.storage.core.StorageEngine; import org.opencb.opencga.storage.core.metadata.models.ProjectMetadata; import org.opencb.opencga.storage.core.utils.CellBaseUtils; @@ -21,6 +23,7 @@ import static org.junit.Assert.assertEquals; import static org.opencb.opencga.storage.core.variant.VariantStorageOptions.ANNOTATOR_CELLBASE_INCLUDE; +@Category(MediumTests.class) public class VariantAnnotatorByApiKeyTest { private StorageConfiguration storageConfiguration; diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/annotation/annotators/VariantAnnotatorTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/annotation/annotators/VariantAnnotatorTest.java index 052a55cfcea..5f50c28828d 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/annotation/annotators/VariantAnnotatorTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/annotation/annotators/VariantAnnotatorTest.java @@ -1,6 +1,6 @@ package org.opencb.opencga.storage.core.variant.annotation.annotators; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.junit.Before; import org.junit.Ignore; import org.junit.Rule; diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyProjectMetadataAdaptor.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyProjectMetadataAdaptor.java index d223180d9d1..3ba92ed7f1c 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyProjectMetadataAdaptor.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyProjectMetadataAdaptor.java @@ -45,10 +45,9 @@ public void refresh() { @Override public synchronized DataResult getProjectMetadata() { final DataResult result = new DataResult<>(); - if (projectMetadata == null) { - projectMetadata = new ProjectMetadata("hsapiens", "grch37", 1); + if (projectMetadata != null) { + result.setResults(Collections.singletonList(projectMetadata.copy())); } - result.setResults(Collections.singletonList(projectMetadata.copy())); return result; } diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyVariantStorageEngine.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyVariantStorageEngine.java index 55866e24160..55903d221f2 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyVariantStorageEngine.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyVariantStorageEngine.java @@ -84,6 +84,10 @@ public static void configure(StorageEngineFactory factory, boolean clear) { } } + public static VariantStorageMetadataManager getVariantMetadataManager() { + return new VariantStorageMetadataManager(new DummyVariantStorageMetadataDBAdaptorFactory()); + } + @Override public String getStorageEngineId() { return STORAGE_ENGINE_ID; diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/io/VariantWriterFactoryTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/io/VariantWriterFactoryTest.java index 32ef120f888..38f995c823d 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/io/VariantWriterFactoryTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/io/VariantWriterFactoryTest.java @@ -23,14 +23,15 @@ import org.opencb.biodata.models.variant.Variant; import org.opencb.biodata.models.variant.metadata.VariantFileHeader; import org.opencb.biodata.models.variant.metadata.VariantFileHeaderComplexLine; +import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.io.DataWriter; import org.opencb.opencga.core.testclassification.duration.ShortTests; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; +import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; import org.opencb.opencga.storage.core.variant.dummy.DummyVariantDBAdaptor; -import org.opencb.opencga.storage.core.variant.dummy.DummyVariantStorageEngine; import org.opencb.opencga.storage.core.variant.dummy.DummyVariantStorageMetadataDBAdaptorFactory; import java.io.ByteArrayOutputStream; @@ -86,8 +87,10 @@ public void testContigLengthNull() throws IOException, StorageEngineException { new VariantFileHeaderComplexLine("contig", "chr3", null, null, null, Collections.singletonMap("length", ".")), new VariantFileHeaderComplexLine("contig", "chr4", null, null, null, Collections.singletonMap("length", "1234")) )); - StudyMetadata study = dbAdaptor.getMetadataManager().createStudy("study"); - dbAdaptor.getMetadataManager().unsecureUpdateStudyMetadata(study.setVariantHeader(header)); + VariantStorageMetadataManager metadataManager = dbAdaptor.getMetadataManager(); + metadataManager.getAndUpdateProjectMetadata(new ObjectMap()); + StudyMetadata study = metadataManager.createStudy("study"); + metadataManager.unsecureUpdateStudyMetadata(study.setVariantHeader(header)); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(10000); DataWriter writer = new VariantWriterFactory(dbAdaptor).newDataWriter( VariantWriterFactory.VariantOutputFormat.VCF, @@ -101,7 +104,6 @@ public void testContigLengthNull() throws IOException, StorageEngineException { writer.close(); String s = outputStream.toString(); - System.out.println("s = " + s); assertThat(s, containsString("##contig=")); assertThat(s, containsString("##contig=")); assertThat(s, containsString("##contig=")); diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutorTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutorTest.java index b2f6407b2c5..f5e4c2526b9 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutorTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutorTest.java @@ -48,15 +48,31 @@ public abstract class VariantQueryExecutorTest extends VariantStorageBaseTest { private DBAdaptorVariantQueryExecutor dbQueryExecutor; private List variantQueryExecutors; - @ClassRule - public static VariantSolrExternalResource solr = new VariantSolrExternalResource(); + public static VariantSolrExternalResource solr = null; + + + public void initSolr() throws Exception { + if (solr != null) { + solr = new VariantSolrExternalResource(); + solr.before(); + } + } + + @AfterClass + public static void afterClass() { + if (solr != null) { + solr.after(); + } + } @Before public void setUp() throws Exception { - + initSolr(); VariantDBAdaptor dbAdaptor = getVariantStorageEngine().getDBAdaptor(); VariantStorageMetadataManager metadataManager = dbAdaptor.getMetadataManager(); - solr.configure(variantStorageEngine); + if (solr != null) { + solr.configure(variantStorageEngine); + } if (!fileIndexed) { studyMetadata = newStudyMetadata(); // variantSource = new VariantSource(smallInputUri.getPath(), "testAlias", "testStudy", "Study for testing purposes"); @@ -100,9 +116,11 @@ public void setUp() throws Exception { variantStorageEngine.calculateStats(studyMetadata.getName(), new ArrayList<>(cohorts.keySet()), options); - solr.configure(variantStorageEngine); - variantStorageEngine.secondaryIndex(); - Assert.assertTrue(variantStorageEngine.secondaryAnnotationIndexActiveAndAlive()); + if (solr != null) { + solr.configure(variantStorageEngine); + variantStorageEngine.secondaryIndex(); + Assert.assertTrue(variantStorageEngine.secondaryAnnotationIndexActiveAndAlive()); + } variantQueryExecutors = variantStorageEngine.getVariantQueryExecutors(); dbQueryExecutor = null; diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/VariantSearchToVariantConverterTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/VariantSearchToVariantConverterTest.java index 5839db1745b..539ea8c22d6 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/VariantSearchToVariantConverterTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/VariantSearchToVariantConverterTest.java @@ -49,7 +49,7 @@ public void test() throws Exception { expectedVariant.addStudyEntry(aux.getStudy("2")); VariantSearchModel variantSearchModel = converter.convertToStorageType(expectedVariant); - assertNull(variantSearchModel.getVariantId()); + assertNotNull(variantSearchModel.getVariantId()); assertEquals(variantId, variantSearchModel.getId()); Variant actualVariant = converter.convertToDataModelType(variantSearchModel); diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/solr/SolrQueryParserTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/solr/SolrQueryParserTest.java index a74bcd8f8ed..2b1c3c3ba08 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/solr/SolrQueryParserTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/solr/SolrQueryParserTest.java @@ -43,8 +43,8 @@ public class SolrQueryParserTest { private String studyName = "platinum"; private String flBase = "fl=other,geneToSoAcc,traits,type,soAcc,score_*,sift,passStats_*,caddRaw,biotypes,polyphenDesc,studies,end,id,variantId," - + "popFreq_*,caddScaled,genes,chromosome,xrefs,start,gerp,polyphen,siftDesc," - + "phastCons,phylop,altStats_*,id,chromosome,start,end,type"; + + "popFreq_*,caddScaled,genes,chromosome,xrefs,start,gerp,polyphen,attr_id,siftDesc," + + "phastCons,phylop,altStats_*"; private String flDefault1 = flBase + ",fileInfo__*,qual__*,filter__*,sampleFormat__*"; private String flDefaultStudy = flBase + ",fileInfo__" + studyName + "__*,qual__" + studyName + "__*," + "filter__" + studyName + "__*,sampleFormat__" + studyName + "__*"; @@ -758,7 +758,7 @@ public void parseVariantScoreWithoutStudy() { @Test public void parseVariantScoreWithStudyFqn() { QueryOptions queryOptions = new QueryOptions(); - String studyFqn = "user@project:platinum"; + String studyFqn = "test@project:platinum"; Query query = new Query(); //query.put(STUDIES.key(), study); query.put(SCORE.key(), studyFqn + ":score2>=3.2;" + studyFqn + ":score3:pvalue<0.02"); diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/solr/VariantSolrExternalResource.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/solr/VariantSolrExternalResource.java index 03548c65102..dc5a019baa1 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/solr/VariantSolrExternalResource.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/solr/VariantSolrExternalResource.java @@ -60,9 +60,7 @@ public VariantSolrExternalResource(boolean embeded) { } @Override - protected void before() throws Throwable { - super.before(); - + public void before() throws Exception { Path rootDir = Paths.get("target/test-data", "junit-variant-solr-" + TimeUtils.getTimeMillis()); Files.createDirectories(rootDir); @@ -95,7 +93,7 @@ protected void before() throws Throwable { } @Override - protected void after() { + public void after() { super.after(); try { if (embeded) { diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/pom.xml new file mode 100644 index 00000000000..ca935dae3fa --- /dev/null +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/pom.xml @@ -0,0 +1,79 @@ + + + + 4.0.0 + + + org.opencb.opencga + opencga-storage-hadoop-compat + 3.2.1-SNAPSHOT + ../pom.xml + + + opencga-storage-hadoop-compat-api + jar + + + 2.0.2 + 2.7.7 + 5.0.0-HBase-2.0 + 19.0 + + + + + org.apache.hbase + hbase-client + ${hbase.version} + provided + + + org.apache.hbase + hbase-common + ${hbase.version} + provided + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + provided + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop.version} + provided + + + org.apache.phoenix + phoenix-core + ${phoenix.version} + provided + + + com.google.guava + guava + ${guava.version} + provided + + + + + \ No newline at end of file diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompatApi.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompatApi.java new file mode 100644 index 00000000000..48ab8846f65 --- /dev/null +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompatApi.java @@ -0,0 +1,39 @@ +package org.opencb.opencga.storage.hadoop; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Table; +import org.opencb.opencga.storage.hadoop.variant.annotation.phoenix.PhoenixCompatApi; + +import java.io.IOException; +import java.util.List; + +public abstract class HBaseCompatApi { + + // singleton + private static HBaseCompatApi instance; + public static HBaseCompatApi getInstance() { + if (instance == null) { + try { + instance = Class.forName("org.opencb.opencga.storage.hadoop.HBaseCompat") + .asSubclass(HBaseCompatApi.class) + .getDeclaredConstructor() + .newInstance(); + } catch (ReflectiveOperationException e) { + throw new IllegalStateException(e); + } + } + return instance; + } + + public abstract PhoenixCompatApi getPhoenixCompat(); + + public abstract void available(Configuration configuration) throws IOException; + + public abstract boolean isSolrTestingAvailable(); + + public abstract List getServerList(Admin admin) throws IOException; + + public abstract byte[][] getTableStartKeys(Admin admin, Table table) throws IOException; +} diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/src/main/java/org/opencb/opencga/storage/hadoop/variant/annotation/phoenix/PhoenixCompatApi.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/src/main/java/org/opencb/opencga/storage/hadoop/variant/annotation/phoenix/PhoenixCompatApi.java new file mode 100644 index 00000000000..caafcf0344b --- /dev/null +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/src/main/java/org/opencb/opencga/storage/hadoop/variant/annotation/phoenix/PhoenixCompatApi.java @@ -0,0 +1,21 @@ +package org.opencb.opencga.storage.hadoop.variant.annotation.phoenix; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapreduce.lib.db.DBWritable; +import org.apache.phoenix.compile.QueryPlan; +import org.apache.phoenix.mapreduce.PhoenixRecordReader; +import org.apache.phoenix.schema.PColumn; +import org.apache.phoenix.schema.PTable; + +import java.sql.SQLException; +import java.util.List; + +public interface PhoenixCompatApi { + + PTable makePTable(List columns) throws SQLException; + + PhoenixRecordReader newPhoenixRecordReader( + Class inputClass, Configuration configuration, QueryPlan queryPlan); + + boolean isDropColumnFromViewSupported(); +} diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/pom.xml new file mode 100644 index 00000000000..241ecd7a87b --- /dev/null +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/pom.xml @@ -0,0 +1,85 @@ + + + + 4.0.0 + + + org.opencb.opencga + opencga-storage-hadoop-compat + 3.2.1-SNAPSHOT + ../pom.xml + + + + opencga-storage-hadoop-compat-hbase2.0 + jar + + + 2.0.2 + 2.7.7 + 5.0.0-HBase-2.0 + 19.0 + + + + + org.opencb.opencga + opencga-storage-hadoop-compat-api + ${project.version} + + + org.apache.hbase + hbase-client + ${hbase.version} + provided + + + org.apache.hbase + hbase-common + ${hbase.version} + provided + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + provided + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop.version} + provided + + + org.apache.phoenix + phoenix-core + ${phoenix.version} + provided + + + com.google.guava + guava + ${guava.version} + provided + + + + + \ No newline at end of file diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java new file mode 100644 index 00000000000..bb74dabf4c6 --- /dev/null +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java @@ -0,0 +1,48 @@ +package org.opencb.opencga.storage.hadoop; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.util.Bytes; +import org.opencb.opencga.storage.hadoop.variant.annotation.phoenix.PhoenixCompat; +import org.opencb.opencga.storage.hadoop.variant.annotation.phoenix.PhoenixCompatApi; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class HBaseCompat extends HBaseCompatApi { + + @Override + public void available(Configuration configuration) throws IOException { + HBaseAdmin.available(configuration); + } + + @Override + public boolean isSolrTestingAvailable() { + return false; + } + + @Override + public PhoenixCompatApi getPhoenixCompat() { + return new PhoenixCompat(); + } + + @Override + public List getServerList(Admin admin) throws IOException { + return new ArrayList<>(admin.getClusterStatus().getServers()); + } + + public byte[][] getTableStartKeys(Admin admin, Table table) throws IOException { + List regions = admin.getRegions(table.getName()); + regions.sort((o1, o2) -> Bytes.compareTo(o1.getStartKey(), o2.getStartKey())); + byte[][] startKeys = new byte[regions.size()][]; + for (int i = 0; i < regions.size(); i++) { + startKeys[i] = regions.get(i).getStartKey(); + } + return startKeys; + } +} diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/src/main/java/org/opencb/opencga/storage/hadoop/variant/annotation/phoenix/OpenCGAUpsertExecutor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/src/main/java/org/opencb/opencga/storage/hadoop/variant/annotation/phoenix/OpenCGAUpsertExecutor.java new file mode 100644 index 00000000000..8566d571268 --- /dev/null +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/src/main/java/org/opencb/opencga/storage/hadoop/variant/annotation/phoenix/OpenCGAUpsertExecutor.java @@ -0,0 +1,31 @@ +package org.opencb.opencga.storage.hadoop.variant.annotation.phoenix; + +import org.apache.phoenix.schema.types.PDataType; +import org.apache.phoenix.util.ColumnInfo; +import org.apache.phoenix.util.UpsertExecutor; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.util.List; +import com.google.common.base.Function; + +public abstract class OpenCGAUpsertExecutor extends UpsertExecutor { + + public OpenCGAUpsertExecutor(Connection conn, String tableName, List columnInfoList, + UpsertListener upsertListener) { + super(conn, tableName, columnInfoList, upsertListener); + } + + protected OpenCGAUpsertExecutor(Connection conn, List columnInfoList, PreparedStatement preparedStatement, + UpsertListener upsertListener) { + super(conn, columnInfoList, preparedStatement, upsertListener); + } + + @Override + protected Function createConversionFunction(PDataType dataType) { + java.util.function.Function f = createJavaConversionFunction(dataType); + return f::apply; + } + + protected abstract java.util.function.Function createJavaConversionFunction(PDataType dataType); +} diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/src/main/java/org/opencb/opencga/storage/hadoop/variant/annotation/phoenix/PhoenixCompat.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/src/main/java/org/opencb/opencga/storage/hadoop/variant/annotation/phoenix/PhoenixCompat.java new file mode 100644 index 00000000000..f6881c972f5 --- /dev/null +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/src/main/java/org/opencb/opencga/storage/hadoop/variant/annotation/phoenix/PhoenixCompat.java @@ -0,0 +1,31 @@ +package org.opencb.opencga.storage.hadoop.variant.annotation.phoenix; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapreduce.lib.db.DBWritable; +import org.apache.phoenix.compile.QueryPlan; +import org.apache.phoenix.mapreduce.PhoenixRecordReader; +import org.apache.phoenix.schema.PColumn; +import org.apache.phoenix.schema.PTable; +import org.apache.phoenix.schema.PTableImpl; + +import java.sql.SQLException; +import java.util.List; + +public class PhoenixCompat implements PhoenixCompatApi { + + @Override + public PTable makePTable(List columns) throws SQLException { + return PTableImpl.makePTable(new PTableImpl(), columns); + } + + @Override + public PhoenixRecordReader newPhoenixRecordReader(Class inputClass, Configuration configuration, + QueryPlan queryPlan) { + return new PhoenixRecordReader<>(inputClass, configuration, queryPlan); + } + + @Override + public boolean isDropColumnFromViewSupported() { + return true; + } +} diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/pom.xml new file mode 100644 index 00000000000..3fdd2a30431 --- /dev/null +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/pom.xml @@ -0,0 +1,85 @@ + + + + 4.0.0 + + + org.opencb.opencga + opencga-storage-hadoop-compat + 3.2.1-SNAPSHOT + ../pom.xml + + + + opencga-storage-hadoop-compat-hbase2.2 + jar + + + 2.2.5 + 2.8.5 + 5.0.0-HBase-2.0 + 19.0 + + + + + org.opencb.opencga + opencga-storage-hadoop-compat-api + ${project.version} + + + org.apache.hbase + hbase-client + ${hbase.version} + provided + + + org.apache.hbase + hbase-common + ${hbase.version} + provided + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + provided + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop.version} + provided + + + org.apache.phoenix + phoenix-core + ${phoenix.version} + provided + + + com.google.guava + guava + ${guava.version} + provided + + + + + \ No newline at end of file diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java new file mode 100644 index 00000000000..f7cf534508a --- /dev/null +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java @@ -0,0 +1,48 @@ +package org.opencb.opencga.storage.hadoop; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.util.Bytes; +import org.opencb.opencga.storage.hadoop.variant.annotation.phoenix.PhoenixCompat; +import org.opencb.opencga.storage.hadoop.variant.annotation.phoenix.PhoenixCompatApi; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class HBaseCompat extends HBaseCompatApi { + + public void available(Configuration configuration) throws IOException { + HBaseAdmin.available(configuration); + } + + @Override + public boolean isSolrTestingAvailable() { + return false; + } + + @Override + public PhoenixCompatApi getPhoenixCompat() { + return new PhoenixCompat(); + } + + @Override + public List getServerList(Admin admin) throws IOException { + return new ArrayList<>(admin.getClusterMetrics().getServersName()); + } + + @Override + public byte[][] getTableStartKeys(Admin admin, Table table) throws IOException { + List regions = admin.getRegions(table.getName()); + regions.sort((o1, o2) -> Bytes.compareTo(o1.getStartKey(), o2.getStartKey())); + byte[][] startKeys = new byte[regions.size()][]; + for (int i = 0; i < regions.size(); i++) { + startKeys[i] = regions.get(i).getStartKey(); + } + return startKeys; + } +} diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/src/main/java/org/opencb/opencga/storage/hadoop/variant/annotation/phoenix/OpenCGAUpsertExecutor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/src/main/java/org/opencb/opencga/storage/hadoop/variant/annotation/phoenix/OpenCGAUpsertExecutor.java new file mode 100644 index 00000000000..dc1c49f2a26 --- /dev/null +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/src/main/java/org/opencb/opencga/storage/hadoop/variant/annotation/phoenix/OpenCGAUpsertExecutor.java @@ -0,0 +1,31 @@ +package org.opencb.opencga.storage.hadoop.variant.annotation.phoenix; + +import com.google.common.base.Function; +import org.apache.phoenix.schema.types.PDataType; +import org.apache.phoenix.util.ColumnInfo; +import org.apache.phoenix.util.UpsertExecutor; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.util.List; + +public abstract class OpenCGAUpsertExecutor extends UpsertExecutor { + + public OpenCGAUpsertExecutor(Connection conn, String tableName, List columnInfoList, + UpsertListener upsertListener) { + super(conn, tableName, columnInfoList, upsertListener); + } + + protected OpenCGAUpsertExecutor(Connection conn, List columnInfoList, PreparedStatement preparedStatement, + UpsertListener upsertListener) { + super(conn, columnInfoList, preparedStatement, upsertListener); + } + + @Override + protected Function createConversionFunction(PDataType dataType) { + java.util.function.Function f = createJavaConversionFunction(dataType); + return f::apply; + } + + protected abstract java.util.function.Function createJavaConversionFunction(PDataType dataType); +} diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/src/main/java/org/opencb/opencga/storage/hadoop/variant/annotation/phoenix/PhoenixCompat.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/src/main/java/org/opencb/opencga/storage/hadoop/variant/annotation/phoenix/PhoenixCompat.java new file mode 100644 index 00000000000..e8f5aa08857 --- /dev/null +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/src/main/java/org/opencb/opencga/storage/hadoop/variant/annotation/phoenix/PhoenixCompat.java @@ -0,0 +1,30 @@ +package org.opencb.opencga.storage.hadoop.variant.annotation.phoenix; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapreduce.lib.db.DBWritable; +import org.apache.phoenix.compile.QueryPlan; +import org.apache.phoenix.mapreduce.PhoenixRecordReader; +import org.apache.phoenix.schema.PColumn; +import org.apache.phoenix.schema.PTable; +import org.apache.phoenix.schema.PTableImpl; + +import java.sql.SQLException; +import java.util.List; + +public class PhoenixCompat implements PhoenixCompatApi { + + public PTable makePTable(List columns) throws SQLException { + return PTableImpl.makePTable(new PTableImpl(), columns); + } + + @Override + public PhoenixRecordReader newPhoenixRecordReader(Class inputClass, Configuration configuration, + QueryPlan queryPlan) { + return new PhoenixRecordReader<>(inputClass, configuration, queryPlan); + } + + @Override + public boolean isDropColumnFromViewSupported() { + return true; + } +} diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/pom.xml new file mode 100644 index 00000000000..112516d4f72 --- /dev/null +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/pom.xml @@ -0,0 +1,85 @@ + + + + 4.0.0 + + + org.opencb.opencga + opencga-storage-hadoop-compat + 3.2.1-SNAPSHOT + ../pom.xml + + + + opencga-storage-hadoop-compat-hbase2.4 + jar + + + 2.4.17 + 2.10.0 + 5.1.3 + 1.1.0 + + + + + org.opencb.opencga + opencga-storage-hadoop-compat-api + ${project.version} + + + org.apache.hbase + hbase-client + ${hbase.version} + provided + + + org.apache.hbase + hbase-common + ${hbase.version} + provided + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + provided + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop.version} + provided + + + org.apache.phoenix + phoenix-core + ${phoenix.version} + provided + + + org.apache.phoenix.thirdparty + phoenix-shaded-guava + ${phoenix-thirdparty.version} + provided + + + + + \ No newline at end of file diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java new file mode 100644 index 00000000000..194b47b7794 --- /dev/null +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java @@ -0,0 +1,41 @@ +package org.opencb.opencga.storage.hadoop; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.Table; +import org.opencb.opencga.storage.hadoop.variant.annotation.phoenix.PhoenixCompat; +import org.opencb.opencga.storage.hadoop.variant.annotation.phoenix.PhoenixCompatApi; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class HBaseCompat extends HBaseCompatApi { + + @Override + public void available(Configuration configuration) throws IOException { + HBaseAdmin.available(configuration); + } + + @Override + public boolean isSolrTestingAvailable() { + return true; + } + + @Override + public PhoenixCompatApi getPhoenixCompat() { + return new PhoenixCompat(); + } + + @Override + public List getServerList(Admin admin) throws IOException { + return new ArrayList<>(admin.getClusterMetrics().getServersName()); + } + + @Override + public byte[][] getTableStartKeys(Admin admin, Table table) throws IOException { + return table.getRegionLocator().getStartKeys(); + } +} diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/src/main/java/org/opencb/opencga/storage/hadoop/variant/annotation/phoenix/OpenCGAUpsertExecutor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/src/main/java/org/opencb/opencga/storage/hadoop/variant/annotation/phoenix/OpenCGAUpsertExecutor.java new file mode 100644 index 00000000000..c60e0f7a5b5 --- /dev/null +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/src/main/java/org/opencb/opencga/storage/hadoop/variant/annotation/phoenix/OpenCGAUpsertExecutor.java @@ -0,0 +1,30 @@ +package org.opencb.opencga.storage.hadoop.variant.annotation.phoenix; + +import org.apache.phoenix.schema.types.PDataType; +import org.apache.phoenix.util.ColumnInfo; +import org.apache.phoenix.util.UpsertExecutor; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.util.List; + +public abstract class OpenCGAUpsertExecutor extends UpsertExecutor { + + public OpenCGAUpsertExecutor(Connection conn, String tableName, List columnInfoList, + UpsertListener upsertListener) { + super(conn, tableName, columnInfoList, upsertListener); + } + + protected OpenCGAUpsertExecutor(Connection conn, List columnInfoList, PreparedStatement preparedStatement, + UpsertListener upsertListener) { + super(conn, columnInfoList, preparedStatement, upsertListener); + } + + @Override + protected org.apache.phoenix.thirdparty.com.google.common.base.Function createConversionFunction(PDataType dataType) { + java.util.function.Function f = createJavaConversionFunction(dataType); + return f::apply; + } + + protected abstract java.util.function.Function createJavaConversionFunction(PDataType dataType); +} diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/src/main/java/org/opencb/opencga/storage/hadoop/variant/annotation/phoenix/PhoenixCompat.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/src/main/java/org/opencb/opencga/storage/hadoop/variant/annotation/phoenix/PhoenixCompat.java new file mode 100644 index 00000000000..4d6bc040d82 --- /dev/null +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/src/main/java/org/opencb/opencga/storage/hadoop/variant/annotation/phoenix/PhoenixCompat.java @@ -0,0 +1,51 @@ +package org.opencb.opencga.storage.hadoop.variant.annotation.phoenix; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapreduce.lib.db.DBWritable; +import org.apache.phoenix.compile.QueryPlan; +import org.apache.phoenix.iterate.MapReduceParallelScanGrouper; +import org.apache.phoenix.iterate.ParallelScanGrouper; +import org.apache.phoenix.mapreduce.PhoenixRecordReader; +import org.apache.phoenix.schema.PColumn; +import org.apache.phoenix.schema.PName; +import org.apache.phoenix.schema.PTable; +import org.apache.phoenix.schema.PTableImpl; + +import java.lang.reflect.Constructor; +import java.sql.SQLException; +import java.util.Collections; +import java.util.List; + +public class PhoenixCompat implements PhoenixCompatApi { + + @Override + public PTable makePTable(List columns) throws SQLException { + return new PTableImpl.Builder() + .setName(PName.EMPTY_NAME) + .setColumns(columns) + .setIndexes(Collections.emptyList()) + .setPhysicalNames(Collections.emptyList()) + .build(); + } + + @Override + public PhoenixRecordReader newPhoenixRecordReader(Class inputClass, Configuration configuration, + QueryPlan queryPlan) { + try { + Constructor constructor = PhoenixRecordReader.class + .getDeclaredConstructor(Class.class, Configuration.class, QueryPlan.class, ParallelScanGrouper.class); + constructor.setAccessible(true); + return constructor.newInstance(inputClass, configuration, queryPlan, MapReduceParallelScanGrouper.getInstance()); + } catch (ReflectiveOperationException e) { + throw new IllegalStateException(e); + } + } + + @Override + public boolean isDropColumnFromViewSupported() { + // Phoenix 5.1.x does not support drop column from view. + // Might be related with https://issues.apache.org/jira/browse/PHOENIX-6025 + return false; + } + +} diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/pom.xml new file mode 100644 index 00000000000..1b1393dfceb --- /dev/null +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/pom.xml @@ -0,0 +1,63 @@ + + + + + 4.0.0 + + + org.opencb.opencga + opencga-storage-hadoop + 3.2.1-SNAPSHOT + ../pom.xml + + + opencga-storage-hadoop-compat + pom + + + opencga-storage-hadoop-compat-api + opencga-storage-hadoop-compat-hbase2.0 + opencga-storage-hadoop-compat-hbase2.2 + opencga-storage-hadoop-compat-hbase2.4 + + + + + + org.apache.hbase + hbase-client + ${hbase.version} + provided + + + org.apache.hbase + hbase-common + ${hbase.version} + provided + + + org.apache.phoenix + phoenix-core + ${phoenix.version} + provided + + + + + \ No newline at end of file diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml index 6d7a6649ad7..876fb2c1054 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml @@ -30,6 +30,105 @@ opencga-storage-hadoop-core jar + + + + + org.eclipse.jetty + jetty-server + ${jetty-for-hadoop-test.version} + test + + + org.eclipse.jetty + jetty-webapp + ${jetty-for-hadoop-test.version} + test + + + org.eclipse.jetty + jetty-servlet + ${jetty-for-hadoop-test.version} + test + + + org.eclipse.jetty + jetty-http + ${jetty-for-hadoop-test.version} + test + + + org.eclipse.jetty.http2 + http2-hpack + ${jetty-for-hadoop-test.version} + test + + + org.eclipse.jetty + jetty-io + ${jetty-for-hadoop-test.version} + test + + + org.eclipse.jetty + jetty-util + ${jetty-for-hadoop-test.version} + test + + + org.eclipse.jetty + jetty-util-ajax + ${jetty-for-hadoop-test.version} + test + + + org.eclipse.jetty + jetty-xml + ${jetty-for-hadoop-test.version} + test + + + org.eclipse.jetty + jetty-rewrite + ${jetty-for-hadoop-test.version} + test + + + org.eclipse.jetty + jetty-security + ${jetty-for-hadoop-test.version} + test + + + org.eclipse.jetty + jetty-client + ${jetty-for-hadoop-test.version} + test + + + org.eclipse.jetty + jetty-project + ${jetty-for-hadoop-test.version} + test + + + org.eclipse.jetty + jetty-alpn-client + ${jetty-for-hadoop-test.version} + test + + + org.eclipse.jetty + jetty-alpn-java-client + ${jetty-for-hadoop-test.version} + test + + + + org.opencb.opencga @@ -50,144 +149,124 @@ - com.google.guava - guava + org.opencb.opencga.hadoop.thirdparty + ${opencga-hadoop-shaded.artifactId} + ${opencga.hadoop.thirdparty.version} + true + + + org.yaml + snakeyaml + + org.opencb.opencga - ${opencga-storage-hadoop-deps.artifactId} + opencga-storage-hadoop-compat-api ${project.parent.version} - shaded - org.apache.parquet - parquet-avro + org.opencb.opencga + opencga-storage-hadoop-compat-${opencga-storage-hadoop-compat.id} + ${project.parent.version} + true - com.google.code.findbugs - annotations + org.opencb.commons + commons-lib - org.glassfish.jersey.inject - jersey-hk2 + org.opencb.commons + commons-datastore-core - - - org.opencb.opencga - opencga-storage-core - ${project.parent.version} - test-jar - test + org.opencb.commons + commons-datastore-solr - junit - junit - test + org.opencb.biodata + biodata-tools - org.mockito - mockito-core + org.opencb.biodata + biodata-formats - org.opencb.opencga - ${opencga-storage-hadoop-deps.artifactId} - ${project.parent.version} - tests - test-jar - test + org.opencb.biodata + biodata-models - org.glassfish.jersey.containers - jersey-container-servlet + org.opencb.oskar + oskar-analysis - org.apache.solr - solr-solrj + org.opencb.ga4gh + ga4gh - org.apache.solr - solr-test-framework + org.opencb.cellbase + cellbase-client - - org.opencb.ga4gh - ga4gh - - - commons-io - commons-io + com.google.guava + guava - org.slf4j - slf4j-api + org.apache.parquet + parquet-avro - org.slf4j - jcl-over-slf4j + com.google.code.findbugs + annotations - com.google.protobuf - protobuf-java + org.glassfish.jersey.inject + jersey-hk2 - org.opencb.biodata - biodata-formats + org.glassfish.jersey.containers + jersey-container-servlet - com.fasterxml.jackson.core - jackson-annotations + org.apache.solr + solr-solrj + - org.opencb.cellbase - cellbase-client + com.github.samtools + htsjdk - org.apache.logging.log4j - log4j-core + org.slf4j + slf4j-api + + - org.apache.parquet - parquet-hadoop + commons-io + commons-io - commons-lang - commons-lang + org.slf4j + jcl-over-slf4j - org.apache.ant - ant + com.google.protobuf + protobuf-java - org.apache.zookeeper - zookeeper + org.apache.parquet + parquet-hadoop - org.opencb.commons - commons-lib + org.apache.ant + ant org.apache.commons commons-lang3 - - org.opencb.commons - commons-datastore-core - - - org.opencb.oskar - oskar-analysis - - - com.fasterxml.jackson.core - jackson-core - - - org.hamcrest - hamcrest-core - test - org.apache.commons commons-collections4 @@ -200,7 +279,6 @@ org.apache.avro avro-mapred hadoop2 - commons-logging @@ -211,24 +289,102 @@ parquet-common - org.opencb.biodata - biodata-tools + org.apache.logging.log4j + log4j-api - org.opencb.biodata - biodata-models + com.fasterxml.jackson.core + jackson-core - org.apache.logging.log4j - log4j-api + com.fasterxml.jackson.core + jackson-databind com.fasterxml.jackson.core - jackson-databind + jackson-annotations + + - com.github.samtools - htsjdk + org.opencb.opencga + opencga-storage-core + ${project.parent.version} + test-jar + test + + + org.opencb.opencga.hadoop.thirdparty + ${opencga-hadoop-shaded.artifactId} + ${opencga.hadoop.thirdparty.version} + test-jar + test + true + + + org.yaml + snakeyaml + + + + + junit + junit + test + + + org.mockito + mockito-core + test + + + org.apache.solr + solr-test-framework + test + + + org.apache.hadoop + hadoop-auth + + + org.apache.hadoop + hadoop-common + + + org.apache.hadoop + hadoop-hdfs + + + org.apache.hadoop + hadoop-hdfs-client + + + org.apache.htrace + htrace-core + + + org.apache.httpcomponents + httpcore + + + org.apache.httpcomponents + httpmime + + + org.apache.zookeeper + zookeeper + + + + + org.hamcrest + hamcrest-core + test + + + org.apache.logging.log4j + log4j-core + test @@ -246,69 +402,50 @@ - - - - - - storage-hadoop - - - storage-hadoop - - true - - - - - - maven-assembly-plugin - 3.2.0 + + org.apache.maven.plugins + maven-dependency-plugin + 3.6.0 + + + analyze + + analyze-only + - - src/assembly/src.xml - + true + + + org.opencb.cellbase:cellbase-client + + + org.apache.zookeeper:zookeeper + + + org.apache.logging.log4j:log4j-api + + + commons-lang:commons-lang + + + org.opencb.opencga:opencga-storage-hadoop-compat-api + + + + + com.google.code.findbugs:jsr305 + + + org.apache.zookeeper:zookeeper + + + commons-lang:commons-lang + + - - - make-assembly - package - - single - - - - - - org.apache.maven.plugins - maven-dependency-plugin - 3.6.0 - - - analyze - - analyze-only - - - true - - - * - - - - org.apache.logging.log4j:log4j-api - org.apache.logging.log4j:log4j-core - org.opencb.cellbase:cellbase-client - org.apache.zookeeper:zookeeper - - - - - - - - - - + + + + + diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/assembly/src.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/assembly/src.xml deleted file mode 100644 index 5432844fa24..00000000000 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/assembly/src.xml +++ /dev/null @@ -1,24 +0,0 @@ - - jar-with-dependencies - - jar - - false - - - / - true - true - runtime - - org.eclipse.jetty:* - org.apache.logging.log4j:log4j-slf4j-impl - org.slf4j:slf4j-simple - org.slf4j:slf4j-log4j12 - - - - - diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/app/HBaseMain.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/app/HBaseMain.java index 4daa7789327..7e80e787dea 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/app/HBaseMain.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/app/HBaseMain.java @@ -1,20 +1,21 @@ package org.opencb.opencga.storage.hadoop.app; import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang.ArrayUtils; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.StopWatch; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.*; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.SnapshotDescription; -import org.apache.hadoop.hbase.client.TableState; +import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.exceptions.IllegalArgumentIOException; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; import org.apache.tools.ant.types.Commandline; import org.opencb.commons.ProgressLogger; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.storage.StorageConfiguration; +import org.opencb.opencga.storage.hadoop.HBaseCompat; import org.opencb.opencga.storage.hadoop.utils.HBaseManager; import org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageEngine; import org.opencb.opencga.storage.hadoop.variant.executors.MRExecutor; @@ -78,9 +79,15 @@ public void run(String[] args) throws Exception { case MOVE_TABLE_REGIONS: moveTableRegions(args); break; - case BALANCE_TABLE_REGIONS: - balanceTableRegions(getArg(args, 1), getArgsMap(args, 2, "maxMoves")); + case BALANCE_TABLE_REGIONS: { + ObjectMap argsMap = getArgsMap(args, 2, "maxMoves", "dryRun", "ignoreExceptions", "maxRetries"); + balanceTableRegions(getArg(args, 1), + argsMap.getInt("maxMoves", 50000), + argsMap.getBoolean("dryRun", false), + argsMap.getBoolean("ignoreExceptions", false), + argsMap.getInt("maxRetries", 1)); break; + } case "tables": case LIST_TABLES: print(listTables(getArg(args, 1, "")).stream().map(TableName::getNameWithNamespaceInclAsString).iterator()); @@ -197,8 +204,10 @@ public void run(String[] args) throws Exception { System.out.println(" (see " + CHECK_TABLES_WITH_REGIONS_ON_DEAD_SERVERS + ") by creating a temporary snapshot"); System.out.println(" " + MOVE_TABLE_REGIONS + " "); System.out.println(" Move all regions from selected tables to new random nodes."); -// System.out.println(" " + BALANCE_TABLE_REGIONS + " [--maxMoves N]"); // FIXME -// System.out.println(" " + REGIONS_PER_TABLE + " "); // FIXME + System.out.println(" " + BALANCE_TABLE_REGIONS + " [--maxMoves N]" + + " [--maxRetries 1] [--ignoreExceptions]" + + " [--dryRun]"); + System.out.println(" " + REGIONS_PER_TABLE + " "); // FIXME System.out.println(" " + CLONE_TABLES + " " + "[--keepSnapshots] [--dryRun] [--snapshotSuffix ]"); System.out.println(" Clone all selected tables by creating an intermediate snapshot."); @@ -207,8 +216,8 @@ public void run(String[] args) throws Exception { + "[--skipTablesWithSnapshot]"); System.out.println(" " + SNAPSHOT_TABLE + " [--dryRun] [--snapshotSuffix ] " + "[--skipTablesWithSnapshot]"); - System.out.println(" " + DELETE_SNAPSHOTS + " [--dryRun] [--skipMissing]"); System.out.println(" Create a snapshot for all selected tables."); + System.out.println(" " + DELETE_SNAPSHOTS + " [--dryRun] [--skipMissing]"); System.out.println(" " + CLONE_SNAPSHOTS + " [--dryRun] " + "[--tablePrefixChange :] " + "[--onExistingTables [fail|skip|drop] ]"); @@ -258,9 +267,9 @@ private void exec(String tool, List args) throws Exception { engine.setConfiguration(storageConfiguration, HadoopVariantStorageEngine.STORAGE_ENGINE_ID, ""); MRExecutor mrExecutor = engine.getMRExecutor(); - int exitError = mrExecutor.run(tool, args.toArray(new String[0])); - if (exitError != 0) { - throw new Exception("Exec failed with exit number '" + exitError + "'"); + MRExecutor.Result result = mrExecutor.run(tool, args.toArray(new String[0])); + if (result.getExitValue() != 0) { + throw new Exception("Exec failed with exit number '" + result.getExitValue() + "'"); } } @@ -318,7 +327,7 @@ private void exportSnapshot(String storageConfigurationPath, String snapshot, St engine.setConfiguration(storageConfiguration, HadoopVariantStorageEngine.STORAGE_ENGINE_ID, ""); MRExecutor mrExecutor = engine.getMRExecutor(); - int exitError = mrExecutor.run("hbase", args.toArray(new String[0])); + int exitError = mrExecutor.run("hbase", args.toArray(new String[0])).getExitValue(); if (exitError != 0) { throw new Exception("ExportSnapshot failed with exit number '" + exitError + "'"); } @@ -326,47 +335,38 @@ private void exportSnapshot(String storageConfigurationPath, String snapshot, St } private void regionsPerTable(String tableNameStr) throws Exception { -// TableName tableName = getTable(tableNameStr); -// hBaseManager.act(tableName.getNameAsString(), (table, admin) -> { -// List servers = new ArrayList<>(admin.getClusterStatus().getServers()); -// Map regionsPerServer = new HashMap<>(); -// -// List> tableRegionsAndLocations = getTableRegionsAndLocations(tableName, admin); -// -// System.out.println("#REGION\tSERVER\tSTART_KEY\tEND_KEY"); -// for (Pair pair : tableRegionsAndLocations) { -// RegionInfo region = pair.getFirst(); -// ServerName server = pair.getSecond(); -// regionsPerServer.merge(server.getServerName(), 1, Integer::sum); -// -// System.out.println(region.getEncodedName() -// + "\t" + server.getServerName() -// + "\t" + Bytes.toStringBinary(region.getStartKey()) -// + "\t" + Bytes.toStringBinary(region.getEndKey())); -// } -// -// System.out.println(""); -// System.out.println("#SERVER\tREGIONS"); -// for (ServerName server : servers) { -// System.out.println(server.getServerName() + "\t" + regionsPerServer.getOrDefault(server.getServerName(), 0)); -// } -// -// -// -// return null; -// }); - } - -// private List> getTableRegionsAndLocations(TableName tableName, Admin admin) throws IOException { -// List> tableRegionsAndLocations; -//// try (ZooKeeperWatcher zkw = new ZooKeeperWatcher(admin.getConfiguration(), "hbase-main", null)) { -//// tableRegionsAndLocations = MetaTableAccessor -//// .getTableRegionsAndLocations(zkw, admin.getConnection(), tableName); -//// } -// tableRegionsAndLocations = MetaTableAccessor -// .getTableRegionsAndLocations(admin.getConnection(), tableName); -// return tableRegionsAndLocations; -// } + TableName tableName = getTable(tableNameStr); + hBaseManager.act(tableName.getNameAsString(), (table, admin) -> { + List servers = new ArrayList<>(admin.getClusterStatus().getServers()); + Map regionsPerServer = new HashMap<>(); + + List> tableRegionsAndLocations = getTableRegionsAndLocations(tableName, admin); + + System.out.println("#REGION\tSERVER\tSTART_KEY\tEND_KEY"); + for (Pair pair : tableRegionsAndLocations) { + RegionInfo region = pair.getFirst(); + ServerName server = pair.getSecond(); + regionsPerServer.merge(server.getServerName(), 1, Integer::sum); + + System.out.println(region.getEncodedName() + + "\t" + server.getServerName() + + "\t" + Bytes.toStringBinary(region.getStartKey()) + + "\t" + Bytes.toStringBinary(region.getEndKey())); + } + + System.out.println(""); + System.out.println("#SERVER\tREGIONS"); + for (ServerName server : servers) { + System.out.println(server.getServerName() + "\t" + regionsPerServer.getOrDefault(server.getServerName(), 0)); + } + + return null; + }); + } + + private List> getTableRegionsAndLocations(TableName tableName, Admin admin) throws IOException { + return MetaTableAccessor.getTableRegionsAndLocations(admin.getConnection(), tableName); + } private void reassignTablesWithRegionsOnDeadServers(String[] args) throws Exception { String tableNameFilter = getArg(args, 1); @@ -415,47 +415,81 @@ private void moveTableRegions(String[] args) throws Exception { }); } - private void balanceTableRegions(String tableNameStr, ObjectMap options) throws Exception { -// TableName tableName = getTable(tableNameStr); -// -// int regionCount = hBaseManager.act(tableName.getNameAsString(), (table, admin) -> { -// int maxMoves = options.getInt("maxMoves", 50000); -// List servers = new ArrayList<>(admin.getClusterStatus().getServers()); -// List> tableRegionsAndLocations = getTableRegionsAndLocations(tableName, admin); -// int expectedRegionsPerServer = (tableRegionsAndLocations.size() / servers.size()) + 1; -// Map regionsPerServer = new HashMap<>(); -// servers.forEach(s -> regionsPerServer.put(s.getServerName(), 0)); -// for (Pair pair : tableRegionsAndLocations) { -// regionsPerServer.merge(pair.getSecond().getServerName(), 1, Integer::sum); -// } -// -// for (Pair pair : tableRegionsAndLocations) { -// if (maxMoves < 0) { -// System.out.println("Reached max moves!"); -// break; -// } -// -// String sourceHost = pair.getSecond().getServerName(); -// if (regionsPerServer.get(sourceHost) > expectedRegionsPerServer) { -// Collections.shuffle(servers); -// Optional targetOptional = servers.stream() -// .filter(s -> regionsPerServer.get(s.getServerName()) < expectedRegionsPerServer).findAny(); -// if (!targetOptional.isPresent()) { -// break; -// } -// String testHost = targetOptional.get().getServerName(); -// regionsPerServer.merge(sourceHost, -1, Integer::sum); -// regionsPerServer.merge(testHost, 1, Integer::sum); -// System.out.println("Move region '" + pair.getFirst().getEncodedName() + "' from " + sourceHost + " to " + testHost); -// StopWatch stopWatch = StopWatch.createStarted(); -// admin.move(pair.getFirst().getEncodedNameAsBytes(), Bytes.toBytes(testHost)); -// System.out.println("Moved in "+TimeUtils.durationToString(stopWatch)); -// -// maxMoves--; -// } -// } -// return tableRegionsAndLocations.size(); -// }); + private void balanceTableRegions(String tableNameStr, int maxMoves, boolean dryRun, boolean ignoreExceptions, int maxRetries) + throws Exception { + TableName tableName = getTable(tableNameStr); + + LOGGER.info("Balancing table " + tableName.getNameAsString() + " with maxMoves=" + maxMoves + ", dryRun=" + dryRun + + ", ignoreExceptions=" + ignoreExceptions + ", maxRetries=" + maxRetries); + int regionCount = hBaseManager.act(tableName.getNameAsString(), (table, admin) -> { + List servers = HBaseCompat.getInstance().getServerList(admin); + List> tableRegionsAndLocations = getTableRegionsAndLocations(tableName, admin); + int expectedRegionsPerServer = (tableRegionsAndLocations.size() / servers.size()) + 1; + + Map regionsPerServer = new HashMap<>(); + servers.forEach(s -> regionsPerServer.put(s.getServerName(), 0)); + for (Pair pair : tableRegionsAndLocations) { + regionsPerServer.merge(pair.getSecond().getServerName(), 1, Integer::sum); + } + + // Shuffle the regions to avoid hotspots + Collections.shuffle(tableRegionsAndLocations); + + int moves = 0; + for (Pair pair : tableRegionsAndLocations) { + if (moves > maxMoves) { + LOGGER.info("Reached max moves!"); + break; + } + + ServerName sourceHost = pair.getSecond(); + // If the source host has more regions than expected, move one to another server + if (regionsPerServer.get(sourceHost.getServerName()) > expectedRegionsPerServer) { + // Iterate over the servers in a random order + Collections.shuffle(servers); + Optional targetOptional = servers.stream() + .filter(s -> !s.equals(sourceHost)) + .filter(s -> regionsPerServer.get(s.getServerName()) < expectedRegionsPerServer) + .findAny(); + if (!targetOptional.isPresent()) { + break; + } + ServerName targetHost = targetOptional.get(); + LOGGER.info("Move region '" + pair.getFirst().getEncodedName() + "' from " + sourceHost + " to " + targetHost); + StopWatch stopWatch = StopWatch.createStarted(); + int attempts = 0; + while (attempts < maxRetries) { + try { + if (dryRun) { + LOGGER.info("[DRY-RUN]: admin.move('" + pair.getFirst().getEncodedName() + "," + targetHost + ")"); + } else { + admin.move(pair.getFirst().getEncodedNameAsBytes(), Bytes.toBytes(targetHost.getServerName())); + } + break; + } catch (Exception e) { + LOGGER.info("Error moving region: " + e.getMessage()); + attempts++; + if (attempts < maxRetries) { + LOGGER.info("Retrying... " + attempts + "/" + maxRetries); + } else if (!ignoreExceptions) { + throw e; + } else { + LOGGER.info("Ignoring exception. Unable to move region after " + attempts + " attempts."); + break; + } + } + } + LOGGER.info("Moved in " + TimeUtils.durationToString(stopWatch)); + + regionsPerServer.merge(sourceHost.getServerName(), -1, Integer::sum); + regionsPerServer.merge(targetHost.getServerName(), 1, Integer::sum); + + moves++; + } + } + return tableRegionsAndLocations.size(); + }); + System.out.println("#Balanced regions for table '" + tableNameStr + "' . Total regions: " + regionCount); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/app/PhoenixMain.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/app/PhoenixMain.java index b663d8728e7..1b513b5eed8 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/app/PhoenixMain.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/app/PhoenixMain.java @@ -153,7 +153,7 @@ private void dropView(VariantHadoopDBAdaptor dbAdaptor) throws Exception { throw new IllegalStateException("Variants table '" + dbAdaptor.getVariantTable() + "' doesn't exist"); } - VariantPhoenixSchemaManager.dropTable(dbAdaptor.getHBaseManager(), dbAdaptor.getVariantTable(), true); + VariantPhoenixSchemaManager.dropView(dbAdaptor.getHBaseManager(), dbAdaptor.getVariantTable(), true); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/app/VariantEngineUtilsMain.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/app/VariantEngineUtilsMain.java index ebba09ba9f6..b4ff45f08df 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/app/VariantEngineUtilsMain.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/app/VariantEngineUtilsMain.java @@ -1,6 +1,6 @@ package org.opencb.opencga.storage.hadoop.app; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; @@ -171,7 +171,7 @@ private void delete(String[] args) throws IOException, SQLException, ClassNotFou LOGGER.info("[DRY-RUN] drop phoenix view '{}'", table); } else { LOGGER.info("Drop phoenix view '{}'", table); - VariantPhoenixSchemaManager.dropTable(hBaseManager, table, true); + VariantPhoenixSchemaManager.dropView(hBaseManager, table, true); } } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/app/VariantMetadataMain.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/app/VariantMetadataMain.java index 20cc6d58073..202ab39f0b4 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/app/VariantMetadataMain.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/app/VariantMetadataMain.java @@ -1,7 +1,7 @@ package org.opencb.opencga.storage.hadoop.app; import com.google.common.collect.Iterators; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.TableName; @@ -28,12 +28,6 @@ /** * Created on 12/03/18. * - * java -classpath opencga-storage-hadoop-core-1.4.0-rc-dev-jar-with-dependencies.jar:conf/hadoop/ - * org.opencb.opencga.storage.hadoop.app.AdminMain tables list - * - * java -classpath opencga-storage-hadoop-core-1.4.0-rc-dev-jar-with-dependencies.jar:conf/hadoop/ - * org.opencb.opencga.storage.hadoop.app.AdminMain studies list - * * @author Jacobo Coll <jacobo167@gmail.com> */ public class VariantMetadataMain extends AbstractMain { diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/AbstractHBaseDriver.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/AbstractHBaseDriver.java index 9bcc56abb56..49fdbf02237 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/AbstractHBaseDriver.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/AbstractHBaseDriver.java @@ -11,16 +11,17 @@ import org.apache.hadoop.hbase.mapreduce.TableInputFormat; import org.apache.hadoop.hbase.mapreduce.TableOutputFormat; import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.MRJobConfig; +import org.apache.hadoop.mapreduce.*; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; +import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.parquet.hadoop.ParquetFileWriter; import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil; import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.opencga.core.common.ExceptionUtils; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.hadoop.io.HDFSIOConnector; @@ -52,6 +53,8 @@ public abstract class AbstractHBaseDriver extends Configured implements Tool { public static final String COUNTER_GROUP_NAME = "OPENCGA.HBASE"; public static final String COLUMNS_TO_COUNT = "columns_to_count"; + public static final String MR_APPLICATION_ID = "MR_APPLICATION_ID"; + public static final String ERROR_MESSAGE = "ERROR_MESSAGE"; private static final Logger LOGGER = LoggerFactory.getLogger(AbstractHBaseDriver.class); protected String table; @@ -187,11 +190,19 @@ public final int run(String[] args) throws Exception { boolean succeed = executeJob(job); if (!succeed) { LOGGER.error("error with job!"); + if (!"NA".equals(job.getStatus().getFailureInfo())) { + LOGGER.error("Failure info: " + job.getStatus().getFailureInfo()); + printKeyValue(ERROR_MESSAGE, job.getStatus().getFailureInfo()); + } + } LOGGER.info("================================================="); LOGGER.info("Finish job " + getJobName()); LOGGER.info("Total time : " + TimeUtils.durationToString(stopWatch)); LOGGER.info("================================================="); + for (Counter counter : job.getCounters().getGroup(COUNTER_GROUP_NAME)) { + printKeyValue(counter.getName(), counter.getValue()); + } postExecution(job); close(); @@ -228,23 +239,50 @@ protected void parseFixedParams(String[] args) { private boolean executeJob(Job job) throws IOException, InterruptedException, ClassNotFoundException { Thread hook = new Thread(() -> { + LOGGER.info("Shutdown hook called!"); + LOGGER.info("Gracefully stopping the job '" + job.getJobID() + "' ..."); try { - if (!job.isComplete()) { - job.killJob(); + if (job.getJobState() == JobStatus.State.RUNNING) { + if (!job.isComplete()) { + job.killJob(); + } + LOGGER.info("Job '" + job.getJobID() + "' stopped!"); + } else { + LOGGER.info("Job '" + job.getJobID() + "' is not running. Nothing to do."); } // onError(); - } catch (IOException e) { + } catch (Exception e) { LOGGER.error("Error", e); } }); try { Runtime.getRuntime().addShutdownHook(hook); - return job.waitForCompletion(true); - } finally { + job.submit(); + JobID jobID = job.getJobID(); + String applicationId = jobID.appendTo(new StringBuilder(ApplicationId.appIdStrPrefix)).toString(); + printKeyValue(MR_APPLICATION_ID, applicationId); + boolean completion = job.waitForCompletion(true); Runtime.getRuntime().removeShutdownHook(hook); + return completion; + } catch (Exception e) { + // Do not use a finally block to remove shutdownHook, as finally blocks will be executed even if the JVM is killed, + // and this would throw IllegalStateException("Shutdown in progress"); + try { + Runtime.getRuntime().removeShutdownHook(hook); + } catch (Exception e1) { + e.addSuppressed(e1); + } + throw e; } } + protected static void printKeyValue(String key, Object value) { + // Print key value using System.err directly so it can be read by the MRExecutor + // Do not use logger, as it may be redirected to a file or stdout + // Do not use stdout, as it won't be read by the MRExecutor + System.err.println(key + "=" + value); + } + protected boolean isLocal(Path path) { return HDFSIOConnector.isLocal(path.toUri(), getConf()); } @@ -326,6 +364,7 @@ protected void deleteTemporaryFile(Path outdir) throws IOException { FileSystem fileSystem = outdir.getFileSystem(getConf()); fileSystem.delete(outdir, true); fileSystem.cancelDeleteOnExit(outdir); + LOGGER.info("Temporary file deleted!"); } public class MapReduceOutputFile { @@ -502,6 +541,7 @@ protected static void main(String[] args, Class aClass) { System.exit(code); } catch (Exception e) { LoggerFactory.getLogger(aClass).error("Error executing " + aClass, e); + printKeyValue(ERROR_MESSAGE, ExceptionUtils.prettyExceptionMessage(e, false, true)); System.exit(1); } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/DeleteHBaseColumnDriver.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/DeleteHBaseColumnDriver.java index 52879f01eed..53473a977b2 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/DeleteHBaseColumnDriver.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/DeleteHBaseColumnDriver.java @@ -81,10 +81,15 @@ public void setupJob(Job job, String table) throws IOException { List scans; if (!regions.isEmpty()) { - scans = new ArrayList<>(regions.size() / 2); + LOGGER.info("Delete rows from {} table ranges (start - end)", regions.size()); + scans = new ArrayList<>(regions.size()); for (Pair region : regions) { Scan scan = new Scan(templateScan); scans.add(scan); + LOGGER.info(" - [ '" + + Bytes.toStringBinary(region.getFirst()) + "' , '" + + Bytes.toStringBinary(region.getSecond()) + + "' )"); if (region.getFirst() != null && region.getFirst().length != 0) { scan.setStartRow(region.getFirst()); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/HBaseManager.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/HBaseManager.java index f9bfa4d0a3f..1f6cd77efd8 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/HBaseManager.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/HBaseManager.java @@ -17,24 +17,28 @@ package org.opencb.opencga.storage.hadoop.utils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.StopWatch; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.*; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.util.Bytes; import org.opencb.opencga.core.common.ExceptionUtils; +import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.storage.hadoop.HBaseCompat; import org.opencb.opencga.storage.hadoop.auth.HBaseCredentials; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; +import java.nio.ByteBuffer; +import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; +import java.util.stream.Collectors; import static org.opencb.opencga.storage.hadoop.utils.PersistentResultScanner.isValid; @@ -388,6 +392,139 @@ public boolean createTableIfNeeded(String tableName, byte[] columnFamily, return createTableIfNeeded(getConnection(), tableName, columnFamily, preSplits, compressionType); } + public boolean splitAndMove(Admin admin, TableName name, byte[] expectedSplit) throws IOException { + return splitAndMove(admin, name, expectedSplit, 3, true); + } + + public boolean splitAndMove(Admin admin, TableName tableName, byte[] expectedSplit, int retries, boolean ignoreExceptions) + throws IOException { + StopWatch stopWatch = StopWatch.createStarted(); + int count = 0; + while (count < retries) { + count++; + try { + // Check if split point exists + RegionInfo regionInfo = getRegionInfo(admin, tableName, expectedSplit); + + if (regionInfo == null) { + LOGGER.info("Splitting table '{}' at '{}'", tableName, Bytes.toStringBinary(expectedSplit)); + admin.split(tableName, expectedSplit); + regionInfo = getRegionInfo(admin, tableName, expectedSplit); + int getRegionInfoAttempts = 10; + while (regionInfo == null) { + try { + Thread.sleep(200); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + regionInfo = getRegionInfo(admin, tableName, expectedSplit); + getRegionInfoAttempts--; + if (getRegionInfoAttempts == 0) { + throw new DoNotRetryRegionException( + "Split point " + Bytes.toStringBinary(expectedSplit) + " not found after creation"); + } + } + } else if (count == 1) { + // First try and split point exists. Skip moving region + LOGGER.info("Split point {} already exists. Nothing to do.", Bytes.toStringBinary(expectedSplit)); + return false; + } + LOGGER.info("Moving region '{}' to another region server", regionInfo.getRegionNameAsString()); + admin.move(regionInfo.getEncodedNameAsBytes(), (byte[]) null); + LOGGER.info("New region created '{}' in {}", regionInfo.getRegionNameAsString(), TimeUtils.durationToString(stopWatch)); + return true; + } catch (IOException | RuntimeException e) { + if (ignoreExceptions) { + if (e instanceof DoNotRetryRegionException) { + LOGGER.warn("Error splitting table {} at {}. Retry {}/{} : {}", tableName, + Bytes.toStringBinary(expectedSplit), count, retries, e.getMessage()); + } else { + LOGGER.warn("Error splitting table {} at {}. Retry {}/{}", tableName, + Bytes.toStringBinary(expectedSplit), count, retries, e); + } + } else { + throw e; + } + } + try { + // Wait before retry + LOGGER.info("Waiting before retrying split table '{}' at '{}'", tableName, Bytes.toStringBinary(expectedSplit)); + Thread.sleep(1000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + LOGGER.warn("Unable to split table '{}' at '{}'", tableName, Bytes.toStringBinary(expectedSplit)); + return false; + } + + public static RegionInfo getRegionInfo(Admin admin, TableName name, byte[] expectedSplit) throws IOException { + return admin.getRegions(name) + .stream() + .filter(region -> Bytes.equals(region.getStartKey(), expectedSplit)) + .findFirst() + .orElse(null); + } + + public int expandTableIfNeeded(String tableName, int batch, + Function> batchSplitsGenerator, + int extraBatches, Function batchPlaceholderSplitGenerator) throws IOException { + return expandTableIfNeeded(tableName, Collections.singletonList(batch), batchSplitsGenerator, extraBatches, + batchPlaceholderSplitGenerator); + } + public int expandTableIfNeeded(String tableName, Collection batches, + Function> batchSplitsGenerator, + int extraBatches, Function batchPlaceholderSplitGenerator) throws IOException { + if (batches.isEmpty()) { + throw new IllegalArgumentException("No batches provided"); + } + // Get the expected splits for these batches + Collection expectedSplits = new LinkedHashSet<>(); + for (Integer batch : batches) { + expectedSplits.addAll(batchSplitsGenerator.apply(batch)); + } + int lastBatch = batches.stream().max(Integer::compareTo).get(); + + // Add some split placeholders for the extra batches + for (int i = lastBatch + 1; i < lastBatch + 1 + extraBatches; i++) { + expectedSplits.add(batchPlaceholderSplitGenerator.apply(i)); + } + // Shuffle the splits + List shuffledSplits = new ArrayList<>(expectedSplits); + Collections.shuffle(shuffledSplits); + + // Ensure that the table is split at least until the next expected split + return act(tableName, (table, admin) -> { + int newSplits = 0; + Set existingSplits = Arrays.stream(HBaseCompat.getInstance().getTableStartKeys(admin, table)) + .map(ByteBuffer::wrap) + .collect(Collectors.toSet()); + + int expectedNewSplits = 0; + for (byte[] expectedSplit : expectedSplits) { + if (!existingSplits.contains(ByteBuffer.wrap(expectedSplit))) { + expectedNewSplits++; + LOGGER.info("Missing split point '{}' at '{}'", tableName, Bytes.toStringBinary(expectedSplit)); + } + } + if (expectedNewSplits == 0) { + return 0; + } else { + LOGGER.info("Found {} missing split points. Splitting table '{}'", expectedNewSplits, tableName); + } + + for (byte[] expectedSplit : expectedSplits) { + if (!existingSplits.contains(ByteBuffer.wrap(expectedSplit))) { + if (splitAndMove(admin, table.getName(), expectedSplit)) { + newSplits++; + } + } + } + return newSplits; + }); + } + + /** * Create default HBase table layout with one column family. * diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/AbstractVariantsTableDriver.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/AbstractVariantsTableDriver.java index 0619d56fd30..35305dd4fdc 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/AbstractVariantsTableDriver.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/AbstractVariantsTableDriver.java @@ -25,7 +25,6 @@ import org.apache.hadoop.hbase.filter.*; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.util.Tool; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; @@ -34,7 +33,6 @@ import org.opencb.opencga.storage.core.variant.VariantStorageOptions; import org.opencb.opencga.storage.hadoop.utils.AbstractHBaseDriver; import org.opencb.opencga.storage.hadoop.utils.HBaseManager; -import org.opencb.opencga.storage.hadoop.variant.archive.ArchiveDriver; import org.opencb.opencga.storage.hadoop.variant.archive.ArchiveTableHelper; import org.opencb.opencga.storage.hadoop.variant.gaps.FillMissingFromArchiveTask; import org.opencb.opencga.storage.hadoop.variant.metadata.HBaseVariantStorageMetadataDBAdaptorFactory; @@ -54,7 +52,7 @@ /** * Created by mh719 on 21/11/2016. */ -public abstract class AbstractVariantsTableDriver extends AbstractHBaseDriver implements Tool { +public abstract class AbstractVariantsTableDriver extends AbstractHBaseDriver { public static final String CONFIG_VARIANT_TABLE_NAME = "opencga.variant.table.name"; public static final String TIMESTAMP = "opencga.variant.table.timestamp"; @@ -266,7 +264,7 @@ protected String getVariantsTable() { } protected String getArchiveTable() { - return getConf().get(ArchiveDriver.CONFIG_ARCHIVE_TABLE_NAME, StringUtils.EMPTY); + return getConf().get(ArchiveTableHelper.CONFIG_ARCHIVE_TABLE_NAME, StringUtils.EMPTY); } protected HBaseVariantTableNameGenerator getTableNameGenerator() { @@ -343,7 +341,7 @@ public static void setNoneTimestamp(Job job) { public static String[] buildArgs(String archiveTable, String variantsTable, int studyId, Collection fileIds, ObjectMap other) { - other.put(ArchiveDriver.CONFIG_ARCHIVE_TABLE_NAME, archiveTable); + other.put(ArchiveTableHelper.CONFIG_ARCHIVE_TABLE_NAME, archiveTable); other.put(AbstractVariantsTableDriver.CONFIG_VARIANT_TABLE_NAME, variantsTable); other.put(STUDY_ID, studyId); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopLocalLoadVariantStoragePipeline.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopLocalLoadVariantStoragePipeline.java index 9f93c3935dd..b1a21badba6 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopLocalLoadVariantStoragePipeline.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopLocalLoadVariantStoragePipeline.java @@ -37,7 +37,6 @@ import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.io.managers.IOConnectorProvider; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; -import org.opencb.opencga.storage.core.metadata.models.FileMetadata; import org.opencb.opencga.storage.core.metadata.models.SampleMetadata; import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; @@ -96,83 +95,6 @@ public HadoopLocalLoadVariantStoragePipeline(StorageConfiguration configuration, @Override protected void preLoadRegisterAndValidateFile(int studyId, VariantFileMetadata variantFileMetadata) throws StorageEngineException { super.preLoadRegisterAndValidateFile(studyId, variantFileMetadata); - boolean loadSampleIndex = YesNoAuto.parse(getOptions(), LOAD_SAMPLE_INDEX.key()).orYes().booleanValue(); - FileMetadata fileMetadata = getMetadataManager().getFileMetadata(studyId, getFileId()); - - int version = getMetadataManager().getStudyMetadata(studyId).getSampleIndexConfigurationLatest().getVersion(); - Set alreadyIndexedSamples = new LinkedHashSet<>(); - Set processedSamples = new LinkedHashSet<>(); - Set samplesWithoutSplitData = new LinkedHashSet<>(); - VariantStorageEngine.SplitData splitData = VariantStorageEngine.SplitData.from(options); - for (String sample : variantFileMetadata.getSampleIds()) { - Integer sampleId = getMetadataManager().getSampleId(studyId, sample); - SampleMetadata sampleMetadata = getMetadataManager().getSampleMetadata(studyId, sampleId); - if (splitData != null && sampleMetadata.getSplitData() != null) { - if (!splitData.equals(sampleMetadata.getSplitData())) { - throw new StorageEngineException("Incompatible split data methods. " - + "Unable to mix requested " + splitData - + " with existing " + sampleMetadata.getSplitData()); - } - } - if (sampleMetadata.isIndexed()) { - if (sampleMetadata.getFiles().size() == 1 && sampleMetadata.getFiles().contains(fileMetadata.getId())) { - // It might happen that the sample is marked as INDEXED, but not the file. - // If the sample only belongs to this file (i.e. it's only file is this file), then ignore - // the overwrite the current sample metadata index status - sampleMetadata = getMetadataManager().updateSampleMetadata(studyId, sampleId, - sm -> sm.setIndexStatus(fileMetadata.getIndexStatus())); - } - } - if (sampleMetadata.isIndexed()) { - alreadyIndexedSamples.add(sample); - if (sampleMetadata.isAnnotated() - || !loadSampleIndex && sampleMetadata.getSampleIndexStatus(version) == Status.READY - || sampleMetadata.getSampleIndexAnnotationStatus(version) == Status.READY - || sampleMetadata.getFamilyIndexStatus(version) == Status.READY - || sampleMetadata.isFamilyIndexDefined()) { - processedSamples.add(sampleMetadata.getId()); - } - } - - if (splitData != null && splitData != sampleMetadata.getSplitData()) { - samplesWithoutSplitData.add(sampleId); - } - } - - if (!alreadyIndexedSamples.isEmpty()) { - if (splitData != null) { - logger.info("Loading split data"); - } else { - String fileName = Paths.get(variantFileMetadata.getPath()).getFileName().toString(); - throw StorageEngineException.alreadyLoadedSamples(fileName, new ArrayList<>(alreadyIndexedSamples)); - } - for (Integer sampleId : processedSamples) { - getMetadataManager().updateSampleMetadata(studyId, sampleId, sampleMetadata -> { - if (!loadSampleIndex) { - for (Integer v : sampleMetadata.getSampleIndexVersions()) { - sampleMetadata.setSampleIndexStatus(Status.NONE, v); - } - } - for (Integer v : sampleMetadata.getSampleIndexAnnotationVersions()) { - sampleMetadata.setSampleIndexAnnotationStatus(Status.NONE, v); - } - for (Integer v : sampleMetadata.getFamilyIndexVersions()) { - sampleMetadata.setFamilyIndexStatus(Status.NONE, v); - } - sampleMetadata.setAnnotationStatus(Status.NONE); - sampleMetadata.setMendelianErrorStatus(Status.NONE); - }); - } - } - - if (splitData != null) { - // Register loadSplitData - for (Integer sampleId : samplesWithoutSplitData) { - getMetadataManager().updateSampleMetadata(studyId, sampleId, sampleMetadata -> { - sampleMetadata.setSplitData(splitData); - }); - } - } } @Override @@ -346,8 +268,8 @@ protected void loadFromProto(URI input, URI outdir, ArchiveTableHelper helper, P // Update list of loaded genotypes this.loadedGenotypes = sampleIndexDBLoader.getLoadedGenotypes(); this.sampleIndexVersion = sampleIndexDBLoader.getSampleIndexVersion(); - this.largestVariantLength = largestVariantTask.getMaxLength(); } + this.largestVariantLength = largestVariantTask.getMaxLength(); } protected void loadFromAvro(URI input, URI outdir, ArchiveTableHelper helper, ProgressLogger progressLogger) @@ -409,8 +331,8 @@ protected void loadFromAvroWithArchive(URI input, URI outdir, ArchiveTableHelper // Update list of loaded genotypes this.loadedGenotypes = sampleIndexDBLoader.getLoadedGenotypes(); this.sampleIndexVersion = sampleIndexDBLoader.getSampleIndexVersion(); - this.largestVariantLength = largestVariantTask.getMaxLength(); } + this.largestVariantLength = largestVariantTask.getMaxLength(); logLoadResults(variantReader.getVariantFileMetadata(), resolver, hadoopDBWriter); } @@ -457,8 +379,8 @@ protected void loadFromAvroWithoutArchive(URI input, URI outdir, ArchiveTableHel // Update list of loaded genotypes this.loadedGenotypes = sampleIndexDBLoader.getLoadedGenotypes(); this.sampleIndexVersion = sampleIndexDBLoader.getSampleIndexVersion(); - this.largestVariantLength = largestVariantTask.getMaxLength(); } + this.largestVariantLength = largestVariantTask.getMaxLength(); logLoadResults(variantReader.getVariantFileMetadata(), resolver, hadoopDBWriter); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java index d75eed0ecec..84f00b042e9 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java @@ -31,6 +31,7 @@ import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.core.common.IOUtils; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.common.UriUtils; import org.opencb.opencga.core.config.DatabaseCredentials; @@ -38,6 +39,7 @@ import org.opencb.opencga.core.config.storage.StorageEngineConfiguration; import org.opencb.opencga.core.models.operations.variant.VariantAggregateFamilyParams; import org.opencb.opencga.core.models.operations.variant.VariantAggregateParams; +import org.opencb.opencga.core.models.variant.VariantSetupParams; import org.opencb.opencga.storage.core.StoragePipelineResult; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.exceptions.StoragePipelineException; @@ -77,6 +79,7 @@ import org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.VariantPhoenixSchemaManager; import org.opencb.opencga.storage.hadoop.variant.adaptors.sample.HBaseVariantSampleDataManager; import org.opencb.opencga.storage.hadoop.variant.annotation.HadoopDefaultVariantAnnotationManager; +import org.opencb.opencga.storage.hadoop.variant.archive.ArchiveDeleteHBaseColumnTask; import org.opencb.opencga.storage.hadoop.variant.archive.ArchiveTableHelper; import org.opencb.opencga.storage.hadoop.variant.executors.MRExecutor; import org.opencb.opencga.storage.hadoop.variant.executors.MRExecutorFactory; @@ -826,7 +829,10 @@ private void remove(String study, List files, List samples, URI archiveColumns.add(family + ':' + ArchiveTableHelper.getRefColumnName(fileId)); archiveColumns.add(family + ':' + ArchiveTableHelper.getNonRefColumnName(fileId)); } - String[] deleteFromArchiveArgs = DeleteHBaseColumnDriver.buildArgs(archiveTable, archiveColumns, options); + ObjectMap thisOptions = new ObjectMap(options); + ArchiveDeleteHBaseColumnTask.configureTask(thisOptions, fileIds); + + String[] deleteFromArchiveArgs = DeleteHBaseColumnDriver.buildArgs(archiveTable, archiveColumns, thisOptions); getMRExecutor().run(DeleteHBaseColumnDriver.class, deleteFromArchiveArgs, "Delete from archive table"); return stopWatch.now(TimeUnit.MILLISECONDS); }); @@ -1082,6 +1088,49 @@ protected List initVariantAggregationExecutors() { return executors; } + @Override + public ObjectMap inferConfigurationParams(VariantSetupParams params) { + ObjectMap options = super.inferConfigurationParams(params); + ObjectMap configuredOptions = getOptions(); + + long expectedHBaseRegionSize = IOUtils.fromHumanReadableToByte("7.5GiB"); + + options.put(EXPECTED_SAMPLES_NUMBER.key(), params.getExpectedSamples()); + options.put(EXPECTED_FILES_NUMBER.key(), params.getExpectedFiles()); + + // Variant pre-split + int defaultVariantPreSplit = configuredOptions + .getInt(VARIANT_TABLE_PRESPLIT_SIZE.key(), VARIANT_TABLE_PRESPLIT_SIZE.defaultValue()); + float variantsFileToHBaseMultiplier = 1.3f; + Long averageFileSize = IOUtils.fromHumanReadableToByte(params.getAverageFileSize(), true); + float variantsTableSize = params.getExpectedFiles() * averageFileSize * variantsFileToHBaseMultiplier; + int variantPreSplit = (int) (variantsTableSize / expectedHBaseRegionSize); + options.put(VARIANT_TABLE_PRESPLIT_SIZE.key(), Math.max(defaultVariantPreSplit, variantPreSplit)); + + // Archive pre-split + int filesPerBatch = configuredOptions + .getInt(ARCHIVE_FILE_BATCH_SIZE.key(), ARCHIVE_FILE_BATCH_SIZE.defaultValue()); + float archiveFileToHBaseMultiplier = 1.2f; + float archiveTableSize = filesPerBatch * averageFileSize.floatValue() * archiveFileToHBaseMultiplier; + int archiveTablePreSplit = (int) (archiveTableSize / expectedHBaseRegionSize); + options.put(ARCHIVE_TABLE_PRESPLIT_SIZE.key(), Math.max(1, archiveTablePreSplit)); + + // SampleIndex pre-split + long averageSizePerVariant; + if (params.getVariantsPerSample() > 3500000) { + // With this many variants per sample, most of them won't have much data + averageSizePerVariant = IOUtils.fromHumanReadableToByte("13B"); + } else { + // With a small number of variants per sample, most of them will have a lot of data + averageSizePerVariant = IOUtils.fromHumanReadableToByte("25B"); + } + long sampleIndexSize = params.getVariantsPerSample() * averageSizePerVariant; + int samplesPerSplit = (int) (expectedHBaseRegionSize / sampleIndexSize); + options.put(SAMPLE_INDEX_TABLE_PRESPLIT_SIZE.key(), samplesPerSplit); + + return options; + } + @Override public void close() throws IOException { super.close(); @@ -1259,15 +1308,7 @@ private HBaseVariantTableNameGenerator getTableNameGenerator() { public void testConnection() throws StorageEngineException { try { Configuration conf = getHadoopConfiguration(); - try { - // HBase 2.x -// HBaseAdmin.available(conf); - HBaseAdmin.class.getMethod("available", Configuration.class).invoke(null, conf); - } catch (NoSuchMethodException e) { - // HBase 1.x -// HBaseAdmin.checkHBaseAvailable(conf); - HBaseAdmin.class.getMethod("checkHBaseAvailable", Configuration.class).invoke(null, conf); - } + HBaseAdmin.available(conf); // new PhoenixHelper(conf).newJdbcConnection().getMetaData().getTables(null, null, null, null); } catch (Exception e) { logger.error("Connection to database '" + dbName + "' failed", e); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageOptions.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageOptions.java index ab6dfaf87ef..817605be87c 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageOptions.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageOptions.java @@ -8,7 +8,7 @@ public enum HadoopVariantStorageOptions implements ConfigurationOption { HADOOP_LOAD_FILES_IN_PARALLEL("storage.hadoop.load.filesInParallel", 1), HBASE_NAMESPACE("storage.hadoop.hbase.namespace"), - EXPECTED_FILES_NUMBER("expected_files_number", 5000), + EXPECTED_FILES_NUMBER("expected_files_number", 50), EXPECTED_SAMPLES_NUMBER("expected_samples_number"), DBADAPTOR_PHOENIX_FETCH_SIZE("storage.hadoop.phoenix.fetchSize", -1), DBADAPTOR_PHOENIX_QUERY_COMPLEXITY_THRESHOLD("storage.hadoop.phoenix.queryComplexityThreshold", 250), @@ -58,12 +58,13 @@ public enum HadoopVariantStorageOptions implements ConfigurationOption { MR_EXECUTOR_SSH_REMOTE_OPENCGA_HOME("storage.hadoop.mr.executor.ssh.remoteOpenCgaHome"), MR_EXECUTOR_SSH_HADOOP_SSH_BIN("storage.hadoop.mr.executor.ssh.hadoop-ssh.bin", "misc/scripts/hadoop-ssh.sh"), MR_EXECUTOR_SSH_HADOOP_SCP_BIN("storage.hadoop.mr.executor.ssh.hadoop-scp.bin", "misc/scripts/hadoop-scp.sh"), + MR_EXECUTOR_SSH_HADOOP_TERMINATION_GRACE_PERIOD_SECONDS("storage.hadoop.mr.executor.ssh.terminationGracePeriodSeconds", 120), ///////////////////////// // Variant table configuration ///////////////////////// VARIANT_TABLE_COMPRESSION("storage.hadoop.variant.table.compression", Compression.Algorithm.SNAPPY.getName()), - VARIANT_TABLE_PRESPLIT_SIZE("storage.hadoop.variant.table.preSplit.numSplits", 500), + VARIANT_TABLE_PRESPLIT_SIZE("storage.hadoop.variant.table.preSplit.numSplits", 50), // Do not create phoenix indexes. Testing purposes only VARIANT_TABLE_INDEXES_SKIP("storage.hadoop.variant.table.indexes.skip"), VARIANT_TABLE_LOAD_REFERENCE("storage.hadoop.variant.table.load.reference", false), @@ -77,7 +78,8 @@ public enum HadoopVariantStorageOptions implements ConfigurationOption { // Archive table configuration ///////////////////////// ARCHIVE_TABLE_COMPRESSION("storage.hadoop.archive.table.compression", Compression.Algorithm.GZ.getName()), - ARCHIVE_TABLE_PRESPLIT_SIZE("storage.hadoop.archive.table.preSplit.splitsPerBatch", 500), + ARCHIVE_TABLE_PRESPLIT_SIZE("storage.hadoop.archive.table.preSplit.splitsPerBatch", 10), + ARCHIVE_TABLE_PRESPLIT_EXTRA_SPLITS("storage.hadoop.archive.table.preSplit.extraSplits", 3), ARCHIVE_CHUNK_SIZE("storage.hadoop.archive.table.chunkSize", 1000), ARCHIVE_FILE_BATCH_SIZE("storage.hadoop.archive.table.fileBatchSize", 1000), @@ -91,7 +93,8 @@ public enum HadoopVariantStorageOptions implements ConfigurationOption { // Sample index table configuration ///////////////////////// SAMPLE_INDEX_TABLE_COMPRESSION("storage.hadoop.sampleIndex.table.compression", Compression.Algorithm.SNAPPY.getName()), - SAMPLE_INDEX_TABLE_PRESPLIT_SIZE("storage.hadoop.sampleIndex.table.preSplit.samplesPerSplit", 15), + SAMPLE_INDEX_TABLE_PRESPLIT_SIZE("storage.hadoop.sampleIndex.table.preSplit.samplesPerSplit", 200), + SAMPLE_INDEX_TABLE_PRESPLIT_EXTRA_SPLITS("storage.hadoop.sampleIndex.table.preSplit.extraSplits", 5), SAMPLE_INDEX_BUILD_MAX_SAMPLES_PER_MR("storage.hadoop.sampleIndex.build.maxSamplesPerMR", 2000), SAMPLE_INDEX_ANNOTATION_MAX_SAMPLES_PER_MR("storage.hadoop.sampleIndex.annotation.maxSamplesPerMR", 2000), SAMPLE_INDEX_FAMILY_MAX_TRIOS_PER_MR("storage.hadoop.sampleIndex.family.maxTriosPerMR", 1000), diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStoragePipeline.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStoragePipeline.java index 9b86cba3f6a..c3f5ebf550e 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStoragePipeline.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStoragePipeline.java @@ -170,6 +170,7 @@ public URI preLoad(URI input, URI output) throws StorageEngineException { try { if (YesNoAuto.parse(getOptions(), LOAD_ARCHIVE.key()).orYes().booleanValue()) { ArchiveTableHelper.createArchiveTableIfNeeded(getOptions(), getArchiveTable(), dbAdaptor.getConnection()); + ArchiveTableHelper.expandTableIfNeeded(getOptions(), getArchiveTable(), getFileId(), dbAdaptor.getHBaseManager()); } else { logger.info("Skip archive table"); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/PhoenixHelper.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/PhoenixHelper.java index 1de8cc422e8..a7a87cad4e7 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/PhoenixHelper.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/PhoenixHelper.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; @@ -30,6 +31,7 @@ import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; import org.apache.phoenix.jdbc.PhoenixDriver; import org.apache.phoenix.monitoring.GlobalClientMetrics; +import org.apache.phoenix.monitoring.GlobalMetric; import org.apache.phoenix.schema.ConcurrentTableMutationException; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.PTableType; @@ -42,6 +44,7 @@ import org.opencb.opencga.core.common.BatchUtils; import org.opencb.opencga.core.common.ExceptionUtils; import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.storage.hadoop.HBaseCompat; import org.opencb.opencga.storage.hadoop.utils.HBaseManager; import org.opencb.opencga.storage.hadoop.variant.GenomeHelper; import org.slf4j.Logger; @@ -124,8 +127,15 @@ private boolean execute(Connection con, String sql, int retry) throws SQLExcepti Thread.sleep(millis); } catch (InterruptedException interruption) { Thread.currentThread().interrupt(); + // Sleep interrupted. Stop retrying + throw e; + } + try { + return execute(con, sql, retry - 1); + } catch (Exception e1) { + e.addSuppressed(e1); + throw e; } - return execute(con, sql, retry - 1); } catch (SQLException | RuntimeException e) { logger.error("Error executing '{}'", sql); throw e; @@ -201,8 +211,31 @@ public String buildDropTable(String tableName, PTableType tableType, boolean ifE return sb.toString(); } - public void dropTable(Connection con, String tableName, PTableType tableType, boolean ifExists, boolean cascade) throws SQLException { - execute(con, buildDropTable(tableName, tableType, ifExists, cascade)); + public void dropTable(org.apache.hadoop.hbase.client.Connection hbaseCon, Connection con, String tableName, PTableType tableType, + boolean ifExists, boolean cascade) throws SQLException, IOException { + String sql = buildDropTable(tableName, tableType, ifExists, cascade); + logger.info("Dropping phoenix {}: {}", tableType, tableName); + logger.info(sql); + execute(con, sql); + + try (Admin admin = hbaseCon.getAdmin()) { + // Flush the SYSTEM.CATALOG table to avoid "unexpected errors" when creating a new table with the same name + // This was first observed when running tests in with Phoenix 5.1 + TableName systemCatalog; + if (PhoenixHelper.isNamespaceMappingEnabled(PTableType.SYSTEM, conf)) { + systemCatalog = TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME, + PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE); + } else { + systemCatalog = TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME + + "." + PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE); + } + if (admin.tableExists(systemCatalog)) { + logger.info("Flushing phoenix system catalog table '" + systemCatalog + "'"); + admin.flush(systemCatalog); + } else { + logger.info("System catalog table '" + systemCatalog + "' does not exist, unable to flush it."); + } + } } public void addMissingColumns(Connection con, String tableName, Collection newColumns, PTableType tableType) @@ -254,6 +287,12 @@ public String buildAlterDropColumns(String tableName, Collection c public void dropColumns(Connection con, String tableName, Collection columns, PTableType tableType) throws SQLException { + if (!HBaseCompat.getInstance().getPhoenixCompat().isDropColumnFromViewSupported()) { + logger.info("Dropping columns is not supported for Phoenix version {}.{} . Skipping drop columns.", + PhoenixDriver.INSTANCE.getMajorVersion(), PhoenixDriver.INSTANCE.getMinorVersion()); + return; + } + Set existingColumns = getColumns(con, tableName, tableType) .stream() .map(Column::column) @@ -292,8 +331,9 @@ public Connection openJdbcConnection() throws SQLException, ClassNotFoundExcepti // logger.info("Opening connection to PhoenixDriver"); Connection connection = QueryUtil.getConnection(conf); List stackTrace = ExceptionUtils.getOpencbStackTrace(); + GlobalMetric metric = GlobalClientMetrics.GLOBAL_OPEN_PHOENIX_CONNECTIONS.getMetric(); logger.info("Open Phoenix DB connection #{} {} called from {}", - GlobalClientMetrics.GLOBAL_OPEN_PHOENIX_CONNECTIONS.getMetric().getTotalSum(), + metric.getValue(), connection, stackTrace); return connection; } @@ -303,7 +343,7 @@ private void closeJdbcConnection(Connection connection) throws SQLException { logger.info("Close Phoenix connection {} called from {}", connection, ExceptionUtils.getOpencbStackTrace()); connection.close(); logger.info("Global Phoenix Connections opened: #{}", - GlobalClientMetrics.GLOBAL_OPEN_PHOENIX_CONNECTIONS.getMetric().getTotalSum()); + GlobalClientMetrics.GLOBAL_OPEN_PHOENIX_CONNECTIONS.getMetric().getValue()); } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixSchemaManager.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixSchemaManager.java index 6f917dbabdd..2cb6bbe5403 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixSchemaManager.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixSchemaManager.java @@ -327,7 +327,14 @@ private void createTableIfNeeded() throws SQLException { try { phoenixHelper.execute(con, sql); } catch (Exception e) { - if (!phoenixHelper.tableExists(con, variantsTableName)) { + boolean tableExists; + try { + tableExists = phoenixHelper.tableExists(con, variantsTableName); + } catch (Exception e1) { + e.addSuppressed(e1); + tableExists = false; + } + if (!tableExists) { throw e; } else { logger.info(DEFAULT_TABLE_TYPE + " {} already exists", variantsTableName); @@ -339,16 +346,13 @@ private void createTableIfNeeded() throws SQLException { } } - public void dropTable(boolean ifExists) throws SQLException { - phoenixHelper.dropTable(con, variantsTableName, VariantPhoenixSchema.DEFAULT_TABLE_TYPE, ifExists, true); - } - - public static void dropTable(HBaseManager hBaseManager, String variantsTableName, boolean ifExists) - throws SQLException, ClassNotFoundException { + public static void dropView(HBaseManager hBaseManager, String variantsTableName, boolean ifExists) + throws SQLException, ClassNotFoundException, IOException { // VariantStorageMetadataManager not needed for dropping table try (VariantPhoenixSchemaManager manager = new VariantPhoenixSchemaManager(hBaseManager.getConf(), variantsTableName, null, hBaseManager)) { - manager.dropTable(ifExists); + manager.phoenixHelper.dropTable(hBaseManager.getConnection(), manager.con, manager.variantsTableName, + PTableType.VIEW, ifExists, true); } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/analysis/gwas/GwasHBaseMapReduceAnalysisExecutor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/analysis/gwas/GwasHBaseMapReduceAnalysisExecutor.java index ce29e3f18fe..102638aab8c 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/analysis/gwas/GwasHBaseMapReduceAnalysisExecutor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/analysis/gwas/GwasHBaseMapReduceAnalysisExecutor.java @@ -25,7 +25,7 @@ public void run() throws ToolException { List samples1 = getSampleList1(); List samples2 = getSampleList2(); - if (getConfiguration().getMethod().equals(GwasConfiguration.Method.CHI_SQUARE_TEST)) { + if (getGwasConfiguration().getMethod().equals(GwasConfiguration.Method.CHI_SQUARE_TEST)) { addWarning("Unable to calculate chi-square test."); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/annotation/phoenix/VariantAnnotationUpsertExecutor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/annotation/phoenix/VariantAnnotationUpsertExecutor.java index 43b17b4ba90..651864a76dd 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/annotation/phoenix/VariantAnnotationUpsertExecutor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/annotation/phoenix/VariantAnnotationUpsertExecutor.java @@ -16,10 +16,8 @@ package org.opencb.opencga.storage.hadoop.variant.annotation.phoenix; -import com.google.common.base.Function; import org.apache.phoenix.schema.types.PDataType; import org.apache.phoenix.schema.types.PhoenixArray; -import org.apache.phoenix.util.UpsertExecutor; import org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.PhoenixHelper.Column; import org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.VariantPhoenixSchema; import org.slf4j.Logger; @@ -29,7 +27,11 @@ import java.sql.Array; import java.sql.Connection; import java.sql.SQLException; -import java.util.*; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.function.Function; import java.util.stream.Collectors; import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_MUTATION_SQL_COUNTER; @@ -39,7 +41,7 @@ * * @author Jacobo Coll <jacobo167@gmail.com> */ -public class VariantAnnotationUpsertExecutor extends UpsertExecutor, Object> { +public class VariantAnnotationUpsertExecutor extends OpenCGAUpsertExecutor, Object> { private static final Logger LOG = LoggerFactory.getLogger(VariantAnnotationUpsertExecutor.class); private final List columnList; @@ -122,7 +124,7 @@ private Array toArray(PDataType elementDataType, Collection input) { @Override - protected Function createConversionFunction(PDataType dataType) { + protected Function createJavaConversionFunction(PDataType dataType) { // return input -> dataType.toObject(input, dataType); return input -> input; } @@ -135,7 +137,7 @@ public void close() throws IOException { } catch (SQLException e) { throw new IOException(e); } - LOG.debug("GLOBAL_MUTATION_SQL_COUNTER = " + GLOBAL_MUTATION_SQL_COUNTER.getMetric().getTotalSum()); + LOG.debug("GLOBAL_MUTATION_SQL_COUNTER = " + GLOBAL_MUTATION_SQL_COUNTER.getMetric().getValue()); } // void putDynamicColumns(Map map) { diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveDeleteHBaseColumnTask.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveDeleteHBaseColumnTask.java new file mode 100644 index 00000000000..0fb9fb0adbf --- /dev/null +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveDeleteHBaseColumnTask.java @@ -0,0 +1,42 @@ +package org.opencb.opencga.storage.hadoop.variant.archive; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.opencga.storage.hadoop.utils.DeleteHBaseColumnDriver; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class ArchiveDeleteHBaseColumnTask extends DeleteHBaseColumnDriver.DeleteHBaseColumnTask { + + public static final String FILE_BATCHES_WITH_FILES_TO_DELETE_FROM_ARCHIVE_INDEX = "fileBatchesWithFilesToDeleteFromArchiveIndex"; + + public static void configureTask(ObjectMap options, List fileIds) { + ArchiveRowKeyFactory keyFactory = new ArchiveRowKeyFactory(options); + Set fileBatchesSet = new HashSet<>(); + for (Integer fileId : fileIds) { + fileBatchesSet.add(keyFactory.getFileBatch(fileId)); + } + List fileBatches = new ArrayList<>(fileBatchesSet); + fileBatches.sort(Integer::compareTo); + options.put(DeleteHBaseColumnDriver.DELETE_HBASE_COLUMN_TASK_CLASS, ArchiveDeleteHBaseColumnTask.class.getName()); + options.put(ArchiveDeleteHBaseColumnTask.FILE_BATCHES_WITH_FILES_TO_DELETE_FROM_ARCHIVE_INDEX, fileBatches); + } + + @Override + public List> getRegionsToDelete(Configuration configuration) { + int[] fileBatches = configuration.getInts(FILE_BATCHES_WITH_FILES_TO_DELETE_FROM_ARCHIVE_INDEX); + List> regions = new ArrayList<>(); + ArchiveRowKeyFactory archiveRowKeyFactory = new ArchiveRowKeyFactory(configuration); + for (int fileBatch : fileBatches) { + regions.add(new Pair<>( + Bytes.toBytes(archiveRowKeyFactory.generateBlockIdFromBatch(fileBatch)), + Bytes.toBytes(archiveRowKeyFactory.generateBlockIdFromBatch(fileBatch + 1)))); + } + return regions; + } +} diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveDriver.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveDriver.java index 86104fbe0d7..c1de4b9f1b1 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveDriver.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveDriver.java @@ -65,7 +65,6 @@ public class ArchiveDriver extends Configured implements Tool { public static final String CONFIG_ARCHIVE_INPUT_FILE_VCF = "opencga.archive.input.file.vcf"; public static final String CONFIG_ARCHIVE_INPUT_FILE_VCF_META = "opencga.archive.input.file.vcf.meta"; - public static final String CONFIG_ARCHIVE_TABLE_NAME = "opencga.archive.table.name"; private static final Logger LOGGER = LoggerFactory.getLogger(ArchiveDriver.class); @@ -82,7 +81,7 @@ public int run(String[] args) throws Exception { URI inputFile = URI.create(conf.get(CONFIG_ARCHIVE_INPUT_FILE_VCF)); URI inputMetaFile = URI.create(conf.get(CONFIG_ARCHIVE_INPUT_FILE_VCF_META)); - String tableName = conf.get(CONFIG_ARCHIVE_TABLE_NAME); + String tableName = conf.get(ArchiveTableHelper.CONFIG_ARCHIVE_TABLE_NAME); int studyId = conf.getInt(HadoopVariantStorageEngine.STUDY_ID, -1); int fileId = conf.getInt(HadoopVariantStorageEngine.FILE_ID, -1); @@ -223,7 +222,7 @@ public static int privateMain(String[] args, Configuration conf) throws Exceptio conf.set(CONFIG_ARCHIVE_INPUT_FILE_VCF, toolArgs[0]); conf.set(CONFIG_ARCHIVE_INPUT_FILE_VCF_META, toolArgs[1]); conf = HBaseManager.addHBaseSettings(conf, toolArgs[2]); - conf.set(CONFIG_ARCHIVE_TABLE_NAME, toolArgs[3]); + conf.set(ArchiveTableHelper.CONFIG_ARCHIVE_TABLE_NAME, toolArgs[3]); conf.set(HadoopVariantStorageEngine.STUDY_ID, toolArgs[4]); conf.set(HadoopVariantStorageEngine.FILE_ID, toolArgs[5]); //set the configuration back, so that Tool can configure itself diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveRowKeyFactory.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveRowKeyFactory.java index 43e9cf97cf7..d20e5eee924 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveRowKeyFactory.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveRowKeyFactory.java @@ -143,6 +143,27 @@ public String generateBlockId(int fileId) { return sb.toString(); } + /** + * Generates a Row key based on Chromosome and position adjusted for the + * Chunk size.
    + *
      + *
    • Using {@link Region#normalizeChromosome(String)} to get standard chromosome + * name + *
    • Using {@link #getSliceId(long)} to return slice position + *
    + * e.g. using chunk size 100, separator _ with chr2 and 1234 would result in + * 2_12 + * + * @param fileBatch File batch + * @return {@link String} Row key string + */ + public String generateBlockIdFromBatch(int fileBatch) { + StringBuilder sb = new StringBuilder(FILE_BATCH_PAD + 1); + sb.append(StringUtils.leftPad(String.valueOf(fileBatch), FILE_BATCH_PAD, '0')); + sb.append(getSeparator()); + return sb.toString(); + } + public String generateBlockIdFromSlice(int fileId, String chrom, long slice) { return generateBlockIdFromSliceAndBatch(getFileBatch(fileId), chrom, slice); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveTableHelper.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveTableHelper.java index 7c831e3cc81..348d4d33e00 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveTableHelper.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveTableHelper.java @@ -54,8 +54,9 @@ public class ArchiveTableHelper extends GenomeHelper { public static final byte[] NON_REF_COLUMN_SUFIX_BYTES = Bytes.toBytes(NON_REF_COLUMN_SUFIX); public static final String REF_COLUMN_SUFIX = "_R"; public static final byte[] REF_COLUMN_SUFIX_BYTES = Bytes.toBytes(REF_COLUMN_SUFIX); + public static final String CONFIG_ARCHIVE_TABLE_NAME = "opencga.archive.table.name"; - private final Logger logger = LoggerFactory.getLogger(ArchiveTableHelper.class); + private static Logger logger = LoggerFactory.getLogger(ArchiveTableHelper.class); private final AtomicReference meta = new AtomicReference<>(); private final ArchiveRowKeyFactory keyFactory; private final byte[] nonRefColumn; @@ -187,17 +188,13 @@ public static String getRefColumnName(int fileId) { public static boolean createArchiveTableIfNeeded(Configuration conf, String tableName) throws IOException { try (Connection con = ConnectionFactory.createConnection(conf)) { - return createArchiveTableIfNeeded(conf, tableName, con); + Compression.Algorithm compression = Compression.getCompressionAlgorithmByName( + conf.get(ARCHIVE_TABLE_COMPRESSION.key(), ARCHIVE_TABLE_COMPRESSION.defaultValue())); + final List preSplits = generateArchiveTableBootPreSplitHuman(conf); + return HBaseManager.createTableIfNeeded(con, tableName, COLUMN_FAMILY_BYTES, preSplits, compression); } } - public static boolean createArchiveTableIfNeeded(Configuration conf, String tableName, Connection con) throws IOException { - Compression.Algorithm compression = Compression.getCompressionAlgorithmByName( - conf.get(ARCHIVE_TABLE_COMPRESSION.key(), ARCHIVE_TABLE_COMPRESSION.defaultValue())); - final List preSplits = generateArchiveTableBootPreSplitHuman(conf); - return HBaseManager.createTableIfNeeded(con, tableName, COLUMN_FAMILY_BYTES, preSplits, compression); - } - public static boolean createArchiveTableIfNeeded(ObjectMap conf, String tableName, Connection con) throws IOException { Compression.Algorithm compression = Compression.getCompressionAlgorithmByName( conf.getString(ARCHIVE_TABLE_COMPRESSION.key(), ARCHIVE_TABLE_COMPRESSION.defaultValue())); @@ -205,6 +202,20 @@ public static boolean createArchiveTableIfNeeded(ObjectMap conf, String tableNam return HBaseManager.createTableIfNeeded(con, tableName, COLUMN_FAMILY_BYTES, preSplits, compression); } + public static void expandTableIfNeeded(ObjectMap options, String archiveTable, int fileId, HBaseManager hBaseManager) + throws IOException { + int splitsPerBatch = options.getInt(ARCHIVE_TABLE_PRESPLIT_SIZE.key(), ARCHIVE_TABLE_PRESPLIT_SIZE.defaultValue()); + int extraBatches = options.getInt(ARCHIVE_TABLE_PRESPLIT_EXTRA_SPLITS.key(), ARCHIVE_TABLE_PRESPLIT_EXTRA_SPLITS.defaultValue()); + ArchiveRowKeyFactory rowKeyFactory = new ArchiveRowKeyFactory(options); + int thisBatch = rowKeyFactory.getFileBatch(fileId); + int newRegions = hBaseManager.expandTableIfNeeded(archiveTable, thisBatch, + batch -> generateBatchSplitsHuman(rowKeyFactory, splitsPerBatch, batch), + extraBatches, batch -> Bytes.toBytes(rowKeyFactory.generateBlockIdFromBatch(batch))); + if (newRegions > 0) { + logger.info("Archive table '" + archiveTable + "' expanded with " + newRegions + " new regions for batch " + thisBatch); + } + } + public static List generateArchiveTableBootPreSplitHuman(Configuration conf) { final ArchiveRowKeyFactory rowKeyFactory = new ArchiveRowKeyFactory(conf); int nSplits = conf.getInt(ARCHIVE_TABLE_PRESPLIT_SIZE.key(), ARCHIVE_TABLE_PRESPLIT_SIZE.defaultValue()); @@ -225,15 +236,18 @@ private static List generateArchiveTableBootPreSplitHuman(ArchiveRowKeyF final List preSplits = new ArrayList<>(nSplits * expectedNumBatches); for (int batch = 0; batch <= expectedNumBatches; batch++) { - int finalBatch = batch; - preSplits.addAll(generateBootPreSplitsHuman(nSplits, (chr, position) -> { - long slice = rowKeyFactory.getSliceId(position); - return Bytes.toBytes(rowKeyFactory.generateBlockIdFromSliceAndBatch(finalBatch, chr, slice)); - })); + preSplits.addAll(generateBatchSplitsHuman(rowKeyFactory, nSplits, batch)); } return preSplits; } + public static List generateBatchSplitsHuman(ArchiveRowKeyFactory rowKeyFactory, int nSplits, int batch) { + return generateBootPreSplitsHuman(nSplits, (chr, position) -> { + long slice = rowKeyFactory.getSliceId(position); + return Bytes.toBytes(rowKeyFactory.generateBlockIdFromSliceAndBatch(batch, chr, slice)); + }); + } + public VariantFileMetadata getFileMetadata() { return meta.get(); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/VariantHbaseTransformTask.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/VariantHbaseTransformTask.java index c4ba0284c4c..819da400abf 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/VariantHbaseTransformTask.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/VariantHbaseTransformTask.java @@ -19,9 +19,9 @@ import org.apache.commons.lang3.StringUtils; import org.opencb.biodata.models.variant.Variant; import org.opencb.biodata.models.variant.protobuf.VcfSliceProtos.VcfSlice; -import org.opencb.biodata.tools.variant.converters.proto.VariantToProtoVcfRecord; import org.opencb.biodata.tools.variant.converters.proto.VariantToVcfSliceConverter; import org.opencb.commons.run.ParallelTaskRunner; +import org.opencb.opencga.storage.hadoop.variant.transform.VariantSliceReader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -175,26 +175,7 @@ private void addVariant(String blockKey, Variant var) { } private long[] getCoveredSlicePositions(Variant var) { - return getCoveredSlicePositions(var.getStart(), var.getEnd(), getHelper().getChunkSize()); - } - - public static long[] getCoveredSlicePositions(long start, long end, int chunkSize) { - long startChunk = VariantToProtoVcfRecord.getSlicePosition((int) start, chunkSize); - long endChunk = VariantToProtoVcfRecord.getSlicePosition((int) end, chunkSize); - if (endChunk == startChunk) { - return new long[]{startChunk}; - } else if (endChunk < startChunk) { - // This may happen in insertions starting in a chunk - long aux = endChunk; - endChunk = startChunk; - startChunk = aux; - } - int len = (int) ((endChunk - startChunk) / chunkSize) + 1; - long[] ret = new long[len]; - for (int i = 0; i < len; ++i) { - ret[i] = startChunk + (((long) i) * chunkSize); - } - return ret; + return VariantSliceReader.getCoveredSlicePositions(var.getStart(), var.getEnd(), getHelper().getChunkSize()); } @Override diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/executors/MRExecutor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/executors/MRExecutor.java index ef8f97152ad..a8d9b03745c 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/executors/MRExecutor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/executors/MRExecutor.java @@ -16,18 +16,25 @@ package org.opencb.opencga.storage.hadoop.variant.executors; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.StopWatch; import org.apache.hadoop.util.Tool; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.core.common.GitRepositoryState; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; +import org.opencb.opencga.storage.hadoop.utils.AbstractHBaseDriver; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Properties; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import static org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageOptions.*; @@ -38,9 +45,34 @@ */ public abstract class MRExecutor { + public static final String HADOOP_LIB_VERSION_PROPERTIES = "org/opencb/opencga/storage/hadoop/lib/version.properties"; private ObjectMap options; private List env; private static Logger logger = LoggerFactory.getLogger(MRExecutor.class); + private static final Pattern STDOUT_KEY_VALUE_PATTERN = Pattern.compile("^[a-zA-Z0-9_]+=[^=]+$"); + private static final Pattern EXCEPTION = Pattern.compile("^Exception in thread \"main\" ([^:]+: .+)$"); + + public static class Result { + private final int exitValue; + private final ObjectMap result; + + public Result(int exitValue, ObjectMap result) { + this.exitValue = exitValue; + this.result = result; + } + + public int getExitValue() { + return exitValue; + } + + public ObjectMap getResult() { + return result; + } + + public String getErrorMessage() { + return result.getString(AbstractHBaseDriver.ERROR_MESSAGE); + } + } public MRExecutor init(ObjectMap options) { this.options = options; @@ -51,7 +83,16 @@ public MRExecutor init(ObjectMap options) { public static String getJarWithDependencies(ObjectMap options) throws StorageEngineException { String jar = options.getString(MR_JAR_WITH_DEPENDENCIES.key(), null); if (jar == null) { - jar = "opencga-storage-hadoop-core-" + GitRepositoryState.getInstance().getBuildVersion() + "-jar-with-dependencies.jar"; + Properties properties = new Properties(); + try { + properties.load(MRExecutor.class.getClassLoader() + .getResourceAsStream(HADOOP_LIB_VERSION_PROPERTIES)); + } catch (IOException e) { + throw new StorageEngineException("Error reading classpath file \"" + HADOOP_LIB_VERSION_PROPERTIES + "\" for building the " + + "\"" + MR_JAR_WITH_DEPENDENCIES + "\"" + " parameter", e); + } + jar = "opencga-storage-hadoop-lib-" + properties.getProperty("opencga-hadoop-shaded.id") + "-" + + GitRepositoryState.getInstance().getBuildVersion() + "-jar-with-dependencies.jar"; // throw new StorageEngineException("Missing option " + MR_JAR_WITH_DEPENDENCIES); } if (!Paths.get(jar).isAbsolute()) { @@ -64,24 +105,32 @@ protected static String getOpencgaHome() { return System.getProperty("app.home", ""); } - public void run(Class execClass, String[] args, String taskDescription) + public ObjectMap run(Class execClass, String[] args, String taskDescription) throws StorageEngineException { StopWatch stopWatch = StopWatch.createStarted(); logger.info("------------------------------------------------------"); logger.info(taskDescription); logger.info("------------------------------------------------------"); - int exitValue = run(execClass, args); + Result result = run(execClass, args); + int exitValue = result.getExitValue(); logger.info("------------------------------------------------------"); logger.info("Exit value: {}", exitValue); logger.info("Total time: {}", TimeUtils.durationToString(stopWatch)); if (exitValue != 0) { - throw new StorageEngineException("Error executing MapReduce for : \"" + taskDescription + "\""); + String message = "Error executing MapReduce for : \"" + taskDescription + "\""; + if (StringUtils.isNotEmpty(result.getErrorMessage())) { + message += " : " + result.getErrorMessage(); + } else { + message += " : Unidentified error executing MapReduce job. Check logs for more information."; + } + throw new StorageEngineException(message); } + return result.getResult(); } - protected int run(Class execClass, String[] args) throws StorageEngineException { + protected Result run(Class execClass, String[] args) throws StorageEngineException { String hadoopRoute = options.getString(MR_HADOOP_BIN.key(), MR_HADOOP_BIN.defaultValue()); String jar = getJarWithDependencies(options); String executable = hadoopRoute + " jar " + jar + ' ' + execClass.getName(); @@ -100,7 +149,7 @@ protected int run(Class execClass, String[] args) throws Sto } } - public abstract int run(String executable, String[] args) throws StorageEngineException; + public abstract Result run(String executable, String[] args) throws StorageEngineException; protected ObjectMap getOptions() { return options; @@ -117,4 +166,28 @@ protected static void redactSecureString(String[] args, String key) { } } + protected static ObjectMap readResult(String output) { + ObjectMap result = new ObjectMap(); + List exceptions = new ArrayList<>(); + for (String line : output.split(System.lineSeparator())) { + if (STDOUT_KEY_VALUE_PATTERN.matcher(line).find()) { + String[] split = line.split("="); + if (split.length == 2) { + Object old = result.put(split[0], split[1]); + if (old != null) { + result.put(split[0], old + "," + split[1]); + } + } + } else if (EXCEPTION.matcher(line).find()) { + Matcher matcher = EXCEPTION.matcher(line); + if (matcher.find()) { + exceptions.add(matcher.group(1)); + } + } + } + if (!exceptions.isEmpty() && !result.containsKey(AbstractHBaseDriver.ERROR_MESSAGE)) { + result.put(AbstractHBaseDriver.ERROR_MESSAGE, String.join(", ", exceptions)); + } + return result; + } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/executors/SshMRExecutor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/executors/SshMRExecutor.java index ecb5887a4e9..b205511f830 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/executors/SshMRExecutor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/executors/SshMRExecutor.java @@ -3,17 +3,22 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.util.RunJar; import org.apache.tools.ant.types.Commandline; +import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.exec.Command; import org.opencb.opencga.core.common.UriUtils; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; +import org.opencb.opencga.storage.hadoop.utils.AbstractHBaseDriver; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.ByteArrayOutputStream; +import java.nio.charset.Charset; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; import static org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageOptions.*; @@ -29,23 +34,93 @@ public class SshMRExecutor extends MRExecutor { private static final String HADOOP_SSH_KEY_ENV = "HADOOP_SSH_KEY"; // env-var expected by "sshpass -e" private static final String SSHPASS_ENV = "SSHPASS"; + public static final String PID = "PID"; private static Logger logger = LoggerFactory.getLogger(SshMRExecutor.class); @Override - public int run(String executable, String[] args) throws StorageEngineException { + public SshMRExecutor init(ObjectMap options) { + super.init(options); + return this; + } + + @Override + public Result run(String executable, String[] args) throws StorageEngineException { String commandLine = buildCommand(executable, args); List env = buildEnv(); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + Thread hook = new Thread(() -> { + logger.info("Shutdown hook. Killing MR jobs"); + logger.info("Read output key-value:"); + ObjectMap result = readResult(new String(outputStream.toByteArray(), Charset.defaultCharset())); + for (Map.Entry entry : result.entrySet()) { + logger.info(" - " + entry.getKey() + ": " + entry.getValue()); + } + String remotePid = result.getString(PID); + logger.info("Remote PID: " + remotePid); + List mrJobs = result.getAsStringList(AbstractHBaseDriver.MR_APPLICATION_ID); + logger.info("MR jobs to kill: " + mrJobs); + for (String mrJob : mrJobs) { + logger.info("Killing MR job " + mrJob); + String commandLineKill = buildCommand("yarn", "application", "-kill", mrJob); + Command command = new Command(commandLineKill, env); + command.run(); + int exitValue = command.getExitValue(); + if (exitValue != 0) { + logger.error("Error killing MR job " + mrJob); + } else { + logger.info("MR job " + mrJob + " killed!"); + } + } + if (remotePid != null) { + int remoteProcessGraceKillPeriod = getOptions() + .getInt(MR_EXECUTOR_SSH_HADOOP_TERMINATION_GRACE_PERIOD_SECONDS.key(), + MR_EXECUTOR_SSH_HADOOP_TERMINATION_GRACE_PERIOD_SECONDS.defaultValue()); + logger.info("Wait up to " + remoteProcessGraceKillPeriod + "s for the remote process to finish"); + String commandLineWaitPid = buildCommand("bash", "-c", "" + + "pid=" + remotePid + "; " + + "i=0; " + + "while [ $(( i++ )) -lt " + remoteProcessGraceKillPeriod + " ] && ps -p $pid > /dev/null; do sleep 1; done; " + + "if ps -p $pid > /dev/null; then " + + " echo \"Kill remote ssh process $pid\"; " + + " ps -p $pid; " + + " kill -15 $pid; " + + "else " + + " echo \"Process $pid finished\"; " + + " fi"); + Command command = new Command(commandLineWaitPid, env); + command.run(); + if (command.getExitValue() != 0) { + logger.error("Error waiting for remote process to finish"); + } else { + logger.info("Remote process finished!"); + } + } + }); + Runtime.getRuntime().addShutdownHook(hook); Command command = new Command(commandLine, env); + command.setErrorOutputStream(outputStream); command.run(); int exitValue = command.getExitValue(); - + Runtime.getRuntime().removeShutdownHook(hook); + ObjectMap result = readResult(new String(outputStream.toByteArray(), Charset.defaultCharset())); if (exitValue == 0) { copyOutputFiles(args, env); } - return exitValue; + return new Result(exitValue, result); } + /** + * Copy output files from remote server to local filesystem. + *

    + * This method will look for the "output" argument in the args array. + * The value of the argument is expected to be a path. + * + * @param args Arguments passed to the executable + * @param env Environment variables + * @return Path to the output file + * @throws StorageEngineException if there is an issue copying the files + */ private Path copyOutputFiles(String[] args, List env) throws StorageEngineException { List argsList = Arrays.asList(args); int outputIdx = argsList.indexOf("output"); @@ -76,6 +151,7 @@ private Path copyOutputFiles(String[] args, List env) throws StorageEngi return Paths.get(targetOutput); } } + // Nothing to copy return null; } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/executors/SystemMRExecutor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/executors/SystemMRExecutor.java index bb226be8d44..b46be099230 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/executors/SystemMRExecutor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/executors/SystemMRExecutor.java @@ -17,8 +17,12 @@ package org.opencb.opencga.storage.hadoop.variant.executors; import org.apache.tools.ant.types.Commandline; +import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.exec.Command; +import java.io.ByteArrayOutputStream; +import java.nio.charset.Charset; + import static org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageOptions.MR_EXECUTOR_SSH_PASSWORD; /** @@ -29,14 +33,13 @@ public class SystemMRExecutor extends MRExecutor { @Override - public int run(String executable, String[] args) { - return run(buildCommandLine(executable, args)); - } - - public int run(String commandLine) { - Command command = new Command(commandLine, getEnv()); + public Result run(String executable, String[] args) { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + Command command = new Command(buildCommandLine(executable, args), getEnv()); + command.setErrorOutputStream(outputStream); command.run(); - return command.getExitValue(); + ObjectMap result = readResult(new String(outputStream.toByteArray(), Charset.defaultCharset())); + return new Result(command.getExitValue(), result); } private String buildCommandLine(String executable, String[] args) { diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsFromArchiveMapper.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsFromArchiveMapper.java index 36d408f6fef..9b8df0c579d 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsFromArchiveMapper.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsFromArchiveMapper.java @@ -6,7 +6,7 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.mapreduce.Job; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; -import org.opencb.opencga.storage.hadoop.variant.archive.ArchiveDriver; +import org.opencb.opencga.storage.hadoop.variant.archive.ArchiveTableHelper; import org.opencb.opencga.storage.hadoop.variant.mr.AbstractArchiveTableMapper; import org.opencb.opencga.storage.hadoop.variant.mr.VariantsTableMapReduceHelper; @@ -63,7 +63,7 @@ protected void setup(Context context) throws IOException, InterruptedException { long timestamp = getMrHelper().getTimestamp(); if (isFillGaps(context.getConfiguration())) { Collection samples = getSamples(context.getConfiguration()); - String archiveTableName = context.getConfiguration().get(ArchiveDriver.CONFIG_ARCHIVE_TABLE_NAME); + String archiveTableName = context.getConfiguration().get(ArchiveTableHelper.CONFIG_ARCHIVE_TABLE_NAME); String gapsGenotype = context.getConfiguration().get( FILL_GAPS_GAP_GENOTYPE.key(), FILL_GAPS_GAP_GENOTYPE.defaultValue()); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsMapper.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsMapper.java index 06139af23a5..cacd50da458 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsMapper.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsMapper.java @@ -7,7 +7,7 @@ import org.opencb.biodata.models.variant.Variant; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.hadoop.utils.HBaseManager; -import org.opencb.opencga.storage.hadoop.variant.archive.ArchiveDriver; +import org.opencb.opencga.storage.hadoop.variant.archive.ArchiveTableHelper; import org.opencb.opencga.storage.hadoop.variant.mr.VariantMapper; import java.io.IOException; @@ -28,7 +28,7 @@ protected void setup(Context context) throws IOException, InterruptedException { Configuration configuration = context.getConfiguration(); HBaseManager hBaseManager = new HBaseManager(configuration); - String archiveTableName = context.getConfiguration().get(ArchiveDriver.CONFIG_ARCHIVE_TABLE_NAME); + String archiveTableName = context.getConfiguration().get(ArchiveTableHelper.CONFIG_ARCHIVE_TABLE_NAME); Collection samples = FillGapsFromArchiveMapper.getSamples(configuration); fillGapsTask = new FillGapsFromVariantTask(hBaseManager, archiveTableName, diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexVariantAggregationExecutor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexVariantAggregationExecutor.java index 1bf855183b7..973a922780b 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexVariantAggregationExecutor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexVariantAggregationExecutor.java @@ -124,9 +124,8 @@ protected VariantQueryResult aggregation(Query query, QueryOptions o // Loop long numMatches = 0; - int count = 0; while (sampleVariantIndexEntryIterator.hasNext()) { - count++; + numMatches++; SampleVariantIndexEntry entry = sampleVariantIndexEntryIterator.next(); for (int i = 0; i < accumulators.size(); i++) { FacetFieldAccumulator accumulator = accumulators.get(i); @@ -134,7 +133,6 @@ protected VariantQueryResult aggregation(Query query, QueryOptions o accumulator.accumulate(field, entry); } } - numMatches += count; // Tear down and clean up results. for (int i = 0; i < accumulators.size(); i++) { diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/family/MendelianErrorSampleIndexEntryIterator.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/family/MendelianErrorSampleIndexEntryIterator.java index f089c82f20e..72d869b7ef1 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/family/MendelianErrorSampleIndexEntryIterator.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/family/MendelianErrorSampleIndexEntryIterator.java @@ -122,6 +122,12 @@ public Variant next() { return variant; } + @Override + public Variant nextVariant() { + fetchNextIfNeeded(); + return next; + } + @Override public SampleVariantIndexEntry nextSampleVariantIndexEntry() { AnnotationIndexEntry annotationIndexEntry = nextAnnotationIndexEntry(); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/query/SampleAnnotationIndexQuery.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/query/SampleAnnotationIndexQuery.java index b18ff0453c2..0398bf72ecb 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/query/SampleAnnotationIndexQuery.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/query/SampleAnnotationIndexQuery.java @@ -8,6 +8,7 @@ public class SampleAnnotationIndexQuery { private final byte[] annotationIndexMask; // byte[] = {mask , index} + private final Boolean intergenic; private final IndexFieldFilter consequenceTypeFilter; private final IndexFieldFilter biotypeFilter; private final IndexFieldFilter transcriptFlagFilter; @@ -17,6 +18,7 @@ public class SampleAnnotationIndexQuery { public SampleAnnotationIndexQuery(SampleIndexSchema schema) { this.annotationIndexMask = new byte[]{0, 0}; + this.intergenic = null; this.consequenceTypeFilter = schema.getCtIndex().getField().noOpFilter(); this.biotypeFilter = schema.getBiotypeIndex().getField().noOpFilter(); this.transcriptFlagFilter = schema.getTranscriptFlagIndexSchema().getField().noOpFilter(); @@ -25,12 +27,16 @@ public SampleAnnotationIndexQuery(SampleIndexSchema schema) { this.populationFrequencyFilter = schema.getPopFreqIndex().noOpFilter(); } - public SampleAnnotationIndexQuery(byte[] annotationIndexMask, IndexFieldFilter consequenceTypeFilter, IndexFieldFilter biotypeFilter, + public SampleAnnotationIndexQuery(byte[] annotationIndexMask, + Boolean intergenic, + IndexFieldFilter consequenceTypeFilter, + IndexFieldFilter biotypeFilter, IndexFieldFilter transcriptFlagFilter, CombinationTripleIndexSchema.Filter ctBtTfFilter, IndexFilter clinicalFilter, IndexFilter populationFrequencyFilter) { this.annotationIndexMask = annotationIndexMask; + this.intergenic = intergenic; this.consequenceTypeFilter = consequenceTypeFilter; this.biotypeFilter = biotypeFilter; this.transcriptFlagFilter = transcriptFlagFilter; @@ -47,6 +53,10 @@ public byte getAnnotationIndex() { return annotationIndexMask[1]; } + public Boolean getIntergenic() { + return intergenic; + } + public IndexFieldFilter getConsequenceTypeFilter() { return consequenceTypeFilter; } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/AbstractSampleIndexEntryFilter.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/AbstractSampleIndexEntryFilter.java index 1a0fd0935d9..a6d0eb974bd 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/AbstractSampleIndexEntryFilter.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/AbstractSampleIndexEntryFilter.java @@ -413,6 +413,7 @@ private boolean filterClinicalFields(AnnotationIndexEntry annotationIndexEntry) private boolean filterBtCtTfFields(AnnotationIndexEntry annotationIndexEntry) { if (annotationIndexEntry == null || !annotationIndexEntry.hasSummaryIndex()) { + // Missing annotation. Unable to filter return true; } if (annotationIndexEntry.isIntergenic()) { diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexBuilder.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexBuilder.java index 8b18a5e9a9f..8047fb49cf1 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexBuilder.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexBuilder.java @@ -85,6 +85,7 @@ public void buildSampleIndex(List samples, ObjectMap options, boolean ov } sampleIndexDBAdaptor.createTableIfNeeded(studyId, schema.getVersion(), options); + sampleIndexDBAdaptor.expandTableIfNeeded(studyId, schema.getVersion(), sampleIds, options); if (finalSamplesList.size() < 20) { logger.info("Run sample index build on samples " + finalSamplesList); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexDBAdaptor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexDBAdaptor.java index c7abaedcde9..0d785856776 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexDBAdaptor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexDBAdaptor.java @@ -2,10 +2,7 @@ import com.google.common.collect.Iterators; import org.apache.commons.collections4.CollectionUtils; -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.util.Bytes; import org.opencb.biodata.models.core.Region; @@ -634,6 +631,7 @@ private Scan parse(SingleSampleIndexQuery query, LocusQuery locusQuery, boolean logger.info("---------"); logger.info("Sample = \"" + query.getSample() + "\" , schema version = " + query.getSchema().getVersion()); + logger.info("Table = " + getSampleIndexTableName(query)); printScan(scan); printQuery(locusQuery); printQuery(query); @@ -795,8 +793,11 @@ public boolean createTableIfNeeded(int studyId, int version, ObjectMap options) int preSplitSize = options.getInt( HadoopVariantStorageOptions.SAMPLE_INDEX_TABLE_PRESPLIT_SIZE.key(), HadoopVariantStorageOptions.SAMPLE_INDEX_TABLE_PRESPLIT_SIZE.defaultValue()); + int sampleIndexExtraSplits = options.getInt( + HadoopVariantStorageOptions.SAMPLE_INDEX_TABLE_PRESPLIT_EXTRA_SPLITS.key(), + HadoopVariantStorageOptions.SAMPLE_INDEX_TABLE_PRESPLIT_EXTRA_SPLITS.defaultValue()); - int splits = samples / preSplitSize; + int splits = (samples / preSplitSize) + sampleIndexExtraSplits; ArrayList preSplits = new ArrayList<>(splits); for (int i = 0; i < splits; i++) { preSplits.add(SampleIndexSchema.toRowKey(i * preSplitSize)); @@ -814,6 +815,32 @@ public boolean createTableIfNeeded(int studyId, int version, ObjectMap options) } } + public void expandTableIfNeeded(int studyId, int version, List sampleIds, ObjectMap options) { + String sampleIndexTable = getSampleIndexTableName(studyId, version); + int preSplitSize = options.getInt( + HadoopVariantStorageOptions.SAMPLE_INDEX_TABLE_PRESPLIT_SIZE.key(), + HadoopVariantStorageOptions.SAMPLE_INDEX_TABLE_PRESPLIT_SIZE.defaultValue()); + int sampleIndexExtraBatches = options.getInt( + HadoopVariantStorageOptions.SAMPLE_INDEX_TABLE_PRESPLIT_EXTRA_SPLITS.key(), + HadoopVariantStorageOptions.SAMPLE_INDEX_TABLE_PRESPLIT_EXTRA_SPLITS.defaultValue()); + Set batches = new HashSet<>(); + for (Integer sampleId : sampleIds) { + batches.add((sampleId) / preSplitSize); + } + + try { + int newRegions = hBaseManager.expandTableIfNeeded(sampleIndexTable, batches, + batch -> Collections.singletonList(SampleIndexSchema.toRowKey(batch * preSplitSize)), + sampleIndexExtraBatches, batch -> SampleIndexSchema.toRowKey(batch * preSplitSize)); + if (newRegions != 0) { + // Log number of new regions + logger.info("Sample index table '" + sampleIndexTable + "' expanded with " + newRegions + " new regions"); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + public void updateSampleIndexSchemaStatus(int studyId, int version) throws StorageEngineException { StudyMetadata studyMetadata = metadataManager.getStudyMetadata(studyId); if (studyMetadata.getSampleIndexConfiguration(version).getStatus() diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexDBLoader.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexDBLoader.java index f2700831e94..23c94cf0c28 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexDBLoader.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexDBLoader.java @@ -188,6 +188,7 @@ public String toString() { public boolean open() { super.open(); dbAdaptor.createTableIfNeeded(studyId, schema.getVersion(), options); + dbAdaptor.expandTableIfNeeded(studyId, schema.getVersion(), sampleIds, options); return true; } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexEntryIterator.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexEntryIterator.java index da9c4a3a517..db96af41abd 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexEntryIterator.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexEntryIterator.java @@ -31,6 +31,12 @@ public interface SampleIndexEntryIterator extends Iterator { */ Variant next(); + /** + * Get next variant without moving the cursor. + * @return next variant + */ + Variant nextVariant(); + default SampleVariantIndexEntry nextSampleVariantIndexEntry() { AnnotationIndexEntry annotationIndexEntry = nextAnnotationIndexEntry(); if (annotationIndexEntry != null) { diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParser.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParser.java index 803fa75fea5..d4605b918f4 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParser.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParser.java @@ -1217,7 +1217,6 @@ protected SampleAnnotationIndexQuery parseAnnotationIndexQuery(SampleIndexSchema IndexFilter clinicalFilter = schema.getClinicalIndexSchema().noOpFilter(); final Boolean intergenic = isIntergenicQuery(query); - // BiotypeConsquenceTypeFlagCombination combination = BiotypeConsquenceTypeFlagCombination // .fromQuery(query, Arrays.asList(schema.getTranscriptFlagIndexSchema().getField().getConfiguration().getValues())); BiotypeConsquenceTypeFlagCombination combination = BiotypeConsquenceTypeFlagCombination.fromQuery(query, null); @@ -1285,8 +1284,9 @@ protected SampleAnnotationIndexQuery parseAnnotationIndexQuery(SampleIndexSchema // - The query has the combination CT+TF boolean useCtIndexFilter = intergenic == Boolean.FALSE && (!ctFilterCoveredBySummary - || (!ctBtCombinationCoveredBySummary && combination.isBiotype()) - || combination.isFlag()); + || (!ctBtCombinationCoveredBySummary && combination.isBiotype()) + || combination.isFlag()); + if (useCtIndexFilter) { ctCovered = completeIndex; consequenceTypeFilter = schema.getCtIndex().getField().buildFilter(new OpValue<>("=", soNames)); @@ -1526,7 +1526,7 @@ protected SampleAnnotationIndexQuery parseAnnotationIndexQuery(SampleIndexSchema } } - return new SampleAnnotationIndexQuery(new byte[]{annotationIndexMask, annotationIndex}, + return new SampleAnnotationIndexQuery(new byte[]{annotationIndexMask, annotationIndex}, intergenic, consequenceTypeFilter, biotypeFilter, tfFilter, ctBtTfFilter, clinicalFilter, populationFrequencyFilter); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexVariantBiConverter.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexVariantBiConverter.java index 387a10f60ec..6edc5c9abf2 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexVariantBiConverter.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexVariantBiConverter.java @@ -488,6 +488,11 @@ public AnnotationIndexEntry nextAnnotationIndexEntry() { public Variant next() { throw new NoSuchElementException("Empty iterator"); } + + @Override + public Variant nextVariant() { + throw new NoSuchElementException("Empty iterator"); + } } private static final class CountSampleIndexGtEntryIterator extends SampleIndexGtEntryIterator { @@ -525,6 +530,11 @@ public Variant next() { return DUMMY_VARIANT; } + @Override + public Variant nextVariant() { + return DUMMY_VARIANT; + } + @Override public int getApproxSize() { return count; @@ -575,13 +585,19 @@ public boolean hasNext() { public Variant next() { nextAnnotationIndexEntry(); // ensure read annotation increaseCounters(); + Variant variant = nextVariant(); + movePointer(); + return variant; + } + + @Override + public Variant nextVariant() { Variant variant; if (encodedRefAlt) { variant = toVariantEncodedAlleles(chromosome, batchStart, bytes, currentOffset); } else { variant = toVariant(chromosome, batchStart, bytes, currentOffset, referenceLength, alternateLength); } - movePointer(); return variant; } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/mr/CustomPhoenixInputFormat.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/mr/CustomPhoenixInputFormat.java index a496e84c560..5b280facb0e 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/mr/CustomPhoenixInputFormat.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/mr/CustomPhoenixInputFormat.java @@ -20,11 +20,10 @@ import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.util.PhoenixRuntime; +import org.opencb.opencga.storage.hadoop.HBaseCompat; import java.io.Closeable; import java.io.IOException; -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; import java.sql.Connection; import java.sql.Statement; import java.util.List; @@ -48,28 +47,8 @@ public RecordReader createRecordReader(InputSplit split, TaskAt final QueryPlan queryPlan = getQueryPlan(context, configuration); @SuppressWarnings("unchecked") final Class inputClass = (Class) PhoenixConfigurationUtil.getInputClass(configuration); - PhoenixRecordReader phoenixRecordReader; - try { - // hdp2.6 - Constructor constructor = PhoenixRecordReader.class - .getConstructor(Class.class, Configuration.class, QueryPlan.class, MapReduceParallelScanGrouper.class); - constructor.setAccessible(true); - phoenixRecordReader = constructor.newInstance(inputClass, configuration, queryPlan, MapReduceParallelScanGrouper.getInstance()); - } catch (InstantiationException | InvocationTargetException | IllegalAccessException e) { - throw new IOException(e); - } catch (NoSuchMethodException ignore) { - // Search other constructor - try { - // emg5.31 - Constructor constructor = PhoenixRecordReader.class - .getConstructor(Class.class, Configuration.class, QueryPlan.class); - constructor.setAccessible(true); - phoenixRecordReader = constructor.newInstance(inputClass, configuration, queryPlan); - } catch (InstantiationException | InvocationTargetException | IllegalAccessException | NoSuchMethodException e) { - throw new IOException(e); - } - } - + PhoenixRecordReader phoenixRecordReader = HBaseCompat.getInstance() + .getPhoenixCompat().newPhoenixRecordReader(inputClass, configuration, queryPlan); return new CloseValueRecordReader<>(phoenixRecordReader); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/mr/VariantTableHelper.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/mr/VariantTableHelper.java index a1b12db30fa..3daa349abb1 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/mr/VariantTableHelper.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/mr/VariantTableHelper.java @@ -30,7 +30,7 @@ import org.opencb.opencga.storage.hadoop.variant.GenomeHelper; import org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageOptions; import org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.VariantPhoenixKeyFactory; -import org.opencb.opencga.storage.hadoop.variant.archive.ArchiveDriver; +import org.opencb.opencga.storage.hadoop.variant.archive.ArchiveTableHelper; import org.opencb.opencga.storage.hadoop.variant.utils.HBaseVariantTableNameGenerator; import java.io.IOException; @@ -122,7 +122,7 @@ public static String getVariantsTable(Configuration conf) { } public static void setArchiveTable(Configuration conf, String archiveTable) { - conf.set(ArchiveDriver.CONFIG_ARCHIVE_TABLE_NAME, archiveTable); + conf.set(ArchiveTableHelper.CONFIG_ARCHIVE_TABLE_NAME, archiveTable); } public HBaseVariantTableNameGenerator getHBaseVariantTableNameGenerator() { diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/search/HadoopVariantSearchDataWriter.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/search/HadoopVariantSearchDataWriter.java index 17c61739496..39c63923c02 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/search/HadoopVariantSearchDataWriter.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/search/HadoopVariantSearchDataWriter.java @@ -68,7 +68,7 @@ protected void add(List batch) throws Exception { return PhoenixHelper.toBytes(studyIds, PIntegerArray.INSTANCE); }); - byte[] row = VariantPhoenixKeyFactory.generateVariantRowKey(new Variant(document.getFieldValue("id").toString())); + byte[] row = VariantPhoenixKeyFactory.generateVariantRowKey(new Variant(document.getFieldValue("attr_id").toString())); variantRows.add(row); mutations.add(new Put(row) .addColumn(family, VariantPhoenixSchema.VariantColumn.INDEX_STUDIES.bytes(), bytes)); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/transform/VariantSliceReader.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/transform/VariantSliceReader.java index 4f12d631930..36b6d400a89 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/transform/VariantSliceReader.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/transform/VariantSliceReader.java @@ -19,9 +19,9 @@ import org.apache.commons.lang3.tuple.ImmutablePair; import org.opencb.biodata.models.variant.StudyEntry; import org.opencb.biodata.models.variant.Variant; +import org.opencb.biodata.tools.variant.converters.proto.VariantToProtoVcfRecord; import org.opencb.commons.ProgressLogger; import org.opencb.commons.io.DataReader; -import org.opencb.opencga.storage.hadoop.variant.archive.VariantHbaseTransformTask; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -210,7 +210,27 @@ private void addVariant(Variant variant, String chromosome, long slicePos) { private long[] getCoveredSlicePositions(Variant var) { - return VariantHbaseTransformTask.getCoveredSlicePositions(var.getStart(), var.getEnd(), chunkSize); + return getCoveredSlicePositions(var.getStart(), var.getEnd(), chunkSize); } + public static long[] getCoveredSlicePositions(long start, long end, int chunkSize) { + long startChunk = VariantToProtoVcfRecord.getSlicePosition((int) start, chunkSize); + long endChunk = VariantToProtoVcfRecord.getSlicePosition((int) end, chunkSize); + if (endChunk == startChunk) { + return new long[]{startChunk}; + } else if (endChunk < startChunk) { + // This may happen in insertions starting in a chunk + long aux = endChunk; + endChunk = startChunk; + startChunk = aux; + } + int len = (int) ((endChunk - startChunk) / chunkSize) + 1; + long[] ret = new long[len]; + for (int i = 0; i < len; ++i) { + ret[i] = startChunk + (((long) i) * chunkSize); + } + return ret; + } + + } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/utils/PersistentResultScannerTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/utils/PersistentResultScannerTest.java index cc21d397061..a7cd3fea113 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/utils/PersistentResultScannerTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/utils/PersistentResultScannerTest.java @@ -63,7 +63,8 @@ public void test() throws Exception { int i = 0; for (Iterator iterator = scanner.iterator(); iterator.hasNext(); ) { Result result = iterator.next(); - System.out.println(Bytes.toString(result.getRow())); + assertNotNull(result); + assertNotNull(result.getRow()); i++; if (i == 50 || i == 99) { int scannersCount = persistentScanner.getScannersCount(); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/AutoScaleHBaseTableTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/AutoScaleHBaseTableTest.java new file mode 100644 index 00000000000..ae3ddabd3fb --- /dev/null +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/AutoScaleHBaseTableTest.java @@ -0,0 +1,119 @@ +package org.opencb.opencga.storage.hadoop.variant; + +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.ExternalResource; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.opencga.core.testclassification.duration.LongTests; +import org.opencb.opencga.storage.core.variant.VariantStorageBaseTest; +import org.opencb.opencga.storage.core.variant.VariantStorageOptions; +import org.opencb.opencga.storage.hadoop.HBaseCompat; +import org.opencb.opencga.storage.hadoop.utils.HBaseManager; +import org.opencb.opencga.storage.hadoop.variant.adaptors.VariantHadoopDBAdaptor; +import org.opencb.opencga.storage.hadoop.variant.index.sample.SampleIndexDBAdaptor; + +import static org.junit.Assert.assertEquals; + +@Category(LongTests.class) +public class AutoScaleHBaseTableTest extends VariantStorageBaseTest implements HadoopVariantStorageTest { + + @ClassRule + public static ExternalResource externalResource = new HadoopExternalResource(); + + private VariantHadoopDBAdaptor dbAdaptor; + private SampleIndexDBAdaptor sampleIndexDBAdaptor; + private HadoopVariantStorageEngine engine; + + @Before + public void before() throws Exception { + clearDB(DB_NAME); + engine = getVariantStorageEngine(); + dbAdaptor = engine.getDBAdaptor(); + sampleIndexDBAdaptor = engine.getSampleIndexDBAdaptor(); + + } + + @Test + public void testAutoScaleTables() throws Exception { + + int archiveSplitsPerBatch = 10; + int samplesPerSplit = 2; + int extraSplits = 3; + + ObjectMap params = new ObjectMap() + .append(VariantStorageOptions.STUDY.key(), STUDY_NAME) + .append(VariantStorageOptions.ANNOTATE.key(), false) + .append(VariantStorageOptions.STATS_CALCULATE.key(), false) + .append(HadoopVariantStorageOptions.SAMPLE_INDEX_TABLE_PRESPLIT_SIZE.key(), samplesPerSplit) + .append(HadoopVariantStorageOptions.SAMPLE_INDEX_TABLE_PRESPLIT_EXTRA_SPLITS.key(), extraSplits) + .append(HadoopVariantStorageOptions.ARCHIVE_TABLE_PRESPLIT_SIZE.key(), archiveSplitsPerBatch) + .append(HadoopVariantStorageOptions.ARCHIVE_TABLE_PRESPLIT_EXTRA_SPLITS.key(), extraSplits) + .append(HadoopVariantStorageOptions.ARCHIVE_FILE_BATCH_SIZE.key(), 2) + .append(HadoopVariantStorageOptions.EXPECTED_SAMPLES_NUMBER.key(), 1) + .append(HadoopVariantStorageOptions.EXPECTED_FILES_NUMBER.key(), 1); + + + // -- Batch 1 + int batches = 1; + runETL(engine, getPlatinumFile(1), outputUri, params); + // Each batch starts with one extra split, expect the first batch. So, -1 + // Then, a fixed number of extra splits + checkArchiveTableSplits(((archiveSplitsPerBatch + 1) * batches) - 1 + extraSplits); + checkSampleIndexTableSplits(batches + extraSplits); + + // -- Batch 2 + // First batch has 1 fewer elements than the rest of the batches. + batches = 2; + runETL(engine, getPlatinumFile(2), outputUri, params); + checkArchiveTableSplits(((archiveSplitsPerBatch + 1) * batches) - 1 + extraSplits); + checkSampleIndexTableSplits(batches + extraSplits); + + runETL(engine, getPlatinumFile(3), outputUri, params); + checkArchiveTableSplits(((archiveSplitsPerBatch + 1) * batches) - 1 + extraSplits); + checkSampleIndexTableSplits(batches + extraSplits); + + // -- Batch 3 + batches = 3; + runETL(engine, getPlatinumFile(4), outputUri, params); + checkArchiveTableSplits(((archiveSplitsPerBatch + 1) * batches) - 1 + extraSplits); + checkSampleIndexTableSplits(batches + extraSplits); + + runETL(engine, getPlatinumFile(5), outputUri, params); + checkArchiveTableSplits(((archiveSplitsPerBatch + 1) * batches) - 1 + extraSplits); + checkSampleIndexTableSplits(batches + extraSplits); + + // -- Batch 4 + batches = 4; + runETL(engine, getPlatinumFile(6), outputUri, params); + checkArchiveTableSplits(((archiveSplitsPerBatch + 1) * batches) - 1 + extraSplits); + checkSampleIndexTableSplits(batches + extraSplits); + +// VariantHbaseTestUtils.printVariants(dbAdaptor, newOutputUri()); + } + + private void checkArchiveTableSplits(int expectedSplits) throws Exception { + int studyId = engine.getMetadataManager().getStudyId(STUDY_NAME); + + String archiveTableName = engine.getArchiveTableName(studyId); + HBaseManager hBaseManager = dbAdaptor.getHBaseManager(); + int archiveNumRegions = hBaseManager.act(archiveTableName, + (table, admin) -> HBaseCompat.getInstance().getTableStartKeys(admin, table).length); + // numRegions == numSplits + 1 + assertEquals(archiveTableName, expectedSplits + 1, archiveNumRegions); + } + + private void checkSampleIndexTableSplits(int expectedSplits) throws Exception { + int studyId = engine.getMetadataManager().getStudyId(STUDY_NAME); + + String sampleIndexTableName = sampleIndexDBAdaptor.getSampleIndexTableName(studyId, 1); + HBaseManager hBaseManager = dbAdaptor.getHBaseManager(); + int sampleIndexNumRegions = hBaseManager.act(sampleIndexTableName, + (table, admin) -> HBaseCompat.getInstance().getTableStartKeys(admin, table).length); + // numRegions == numSplits + 1 + assertEquals(sampleIndexTableName, expectedSplits + 1, sampleIndexNumRegions); + } + + +} diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineBNDTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineBNDTest.java index aea720d356a..b613df935ba 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineBNDTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineBNDTest.java @@ -19,7 +19,7 @@ public class HadoopVariantStorageEngineBNDTest extends VariantStorageEngineBNDTe @Override protected void loadFiles() throws Exception { super.loadFiles(); - VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri()); + VariantHbaseTestUtils.printVariants(((HadoopVariantStorageEngine) variantStorageEngine).getDBAdaptor(), newOutputUri()); } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSVTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSVTest.java index fbcbd773477..26bcb49bb0f 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSVTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSVTest.java @@ -1,8 +1,6 @@ package org.opencb.opencga.storage.hadoop.variant; -import org.junit.Assert; -import org.junit.ClassRule; -import org.junit.Test; +import org.junit.*; import org.junit.experimental.categories.Category; import org.opencb.biodata.models.variant.StudyEntry; import org.opencb.biodata.models.variant.Variant; @@ -12,13 +10,15 @@ import org.opencb.opencga.storage.core.variant.adaptors.GenotypeClass; import org.opencb.opencga.storage.core.variant.adaptors.VariantQuery; import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; +import org.opencb.opencga.storage.core.variant.solr.VariantSolrExternalResource; +import org.opencb.opencga.storage.hadoop.HBaseCompat; import org.opencb.opencga.storage.hadoop.variant.adaptors.VariantHadoopDBAdaptor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.*; -import static org.junit.Assert.assertEquals; +import static org.junit.Assert.*; import static org.opencb.opencga.core.api.ParamConstants.OVERWRITE; /** @@ -33,9 +33,46 @@ public class HadoopVariantStorageEngineSVTest extends VariantStorageEngineSVTest public static HadoopExternalResource externalResource = new HadoopExternalResource(); private Logger logger = LoggerFactory.getLogger(getClass()); + public static VariantSolrExternalResource solr = new VariantSolrExternalResource(); + + @BeforeClass + public static void beforeClass() throws Exception { + if (HBaseCompat.getInstance().isSolrTestingAvailable()) { + solr.before(); + solr.configure(externalResource.getVariantStorageEngine()); + System.out.println("Start embedded solr"); + } else { + System.out.println("Skip embedded solr tests"); + } + } + + @AfterClass + public static void afterClass() throws Exception { + if (HBaseCompat.getInstance().isSolrTestingAvailable()) { + solr.after(); + } + } + + @Override + public void before() throws Exception { + super.before(); + if (HBaseCompat.getInstance().isSolrTestingAvailable()) { + solr.configure(variantStorageEngine); + } + } + @Override protected void loadFiles() throws Exception { + if (HBaseCompat.getInstance().isSolrTestingAvailable()) { + solr.configure(variantStorageEngine); + } super.loadFiles(); + if (HBaseCompat.getInstance().isSolrTestingAvailable()) { + variantStorageEngine.secondaryIndex(); + assertTrue(variantStorageEngine.secondaryAnnotationIndexActiveAndAlive()); + } else { + assertFalse(variantStorageEngine.secondaryAnnotationIndexActiveAndAlive()); + } VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri(getTestName().getMethodName())); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSplitDataTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSplitDataTest.java index 1401a7b5ad9..6d111878d57 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSplitDataTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSplitDataTest.java @@ -501,9 +501,10 @@ public void testDuplicatedVariantsAccepted(String... files) throws Exception { for (Boolean nativeQuery : Arrays.asList(true, false)) { String name = fileMetadata.getName(); Query query = new Query(VariantQueryParam.FILE.key(), name); + QueryOptions options = new QueryOptions(NATIVE, nativeQuery); System.out.println("-----------------------"); - System.out.println("FILE-QUERY = " + query.toJson()); - for (Variant variant : variantStorageEngine.get(query, new QueryOptions(NATIVE, nativeQuery)).getResults()) { + System.out.println("FILE-QUERY = " + query.toJson() + " " + options.toJson()); + for (Variant variant : variantStorageEngine.get(query, new QueryOptions(options)).getResults()) { StudyEntry studyEntry = variant.getStudies().get(0); assertEquals(0, studyEntry.getIssues().size()); assertEquals(name, studyEntry.getFiles().get(0).getFileId()); @@ -511,8 +512,8 @@ public void testDuplicatedVariantsAccepted(String... files) throws Exception { for (Integer sample : fileMetadata.getSamples()) { String sampleName = metadataManager.getSampleName(studyId, sample); query = new Query(VariantQueryParam.FILE.key(), name).append(VariantQueryParam.SAMPLE.key(), sampleName); - System.out.println("SAMPLE-QUERY = " + query.toJson()); - for (Variant variant : variantStorageEngine.get(query, new QueryOptions(NATIVE, nativeQuery)).getResults()) { + System.out.println("SAMPLE-QUERY = " + query.toJson() + " " + options.toJson()); + for (Variant variant : variantStorageEngine.get(query, new QueryOptions(options)).getResults()) { StudyEntry studyEntry = variant.getStudies().get(0); assertEquals(0, studyEntry.getIssues().size()); assertEquals(name, studyEntry.getFiles().get(0).getFileId()); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageSearchIntersectTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageSearchIntersectTest.java index ae92a1ae712..e8987078bc8 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageSearchIntersectTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageSearchIntersectTest.java @@ -33,7 +33,10 @@ @Category(LongTests.class) public class HadoopVariantStorageSearchIntersectTest extends VariantStorageSearchIntersectTest implements HadoopVariantStorageTest { - @ClassRule + @ClassRule(order = 1) + public static HadoopSolrSupport solrSupport = new HadoopSolrSupport(); + + @ClassRule(order = 20) public static ExternalResource externalResource = new HadoopExternalResource(); @Override diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageTest.java index c0a1e53d277..cf4d7984ada 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageTest.java @@ -42,6 +42,9 @@ import org.apache.hadoop.hbase.regionserver.wal.FSHLog; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSTableDescriptors; +import org.apache.hadoop.hbase.util.VersionInfo; +import org.apache.hadoop.hbase.wal.FSHLogProvider; +import org.apache.hadoop.hbase.wal.WALFactory; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.datanode.DataNode; @@ -69,6 +72,7 @@ import org.apache.zookeeper.server.PrepRequestProcessor; import org.apache.zookeeper.server.ZooKeeperServer; import org.junit.Assert; +import org.junit.Assume; import org.junit.rules.ExternalResource; import org.opencb.biodata.models.variant.VariantFileMetadata; import org.opencb.biodata.models.variant.avro.VariantType; @@ -82,12 +86,13 @@ import org.opencb.opencga.storage.core.variant.VariantStorageEngine; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; import org.opencb.opencga.storage.core.variant.VariantStorageTest; +import org.opencb.opencga.storage.hadoop.HBaseCompat; import org.opencb.opencga.storage.hadoop.utils.HBaseManager; -import org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.PhoenixHelper; -import org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.VariantPhoenixSchema; +import org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.VariantPhoenixSchemaManager; import org.opencb.opencga.storage.hadoop.variant.executors.MRExecutor; import org.opencb.opencga.storage.hadoop.variant.index.IndexUtils; import org.opencb.opencga.storage.hadoop.variant.index.sample.SampleIndexSchema; +import org.opencb.opencga.storage.hadoop.variant.utils.HBaseVariantTableNameGenerator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -112,6 +117,18 @@ public interface HadoopVariantStorageTest /*extends VariantStorageManagerTestUti // Set managers = new ConcurrentHashSet<>(); AtomicReference manager = new AtomicReference<>(); + class HadoopSolrSupport extends ExternalResource { + @Override + protected void before() throws Throwable { + super.before(); + Assume.assumeTrue(isSolrTestingAvailable()); + } + + public static boolean isSolrTestingAvailable() { + return HBaseCompat.getInstance().isSolrTestingAvailable(); + } + } + class HadoopExternalResource extends ExternalResource implements HadoopVariantStorageTest { Logger logger = LoggerFactory.getLogger(this.getClass()); @@ -215,8 +232,8 @@ public void before() throws Exception { Configurator.setLevel(MapTask.class.getName(), Level.WARN); Configurator.setLevel(TableInputFormatBase.class.getName(), Level.WARN); - utility.set(new HBaseTestingUtility()); - Configuration conf = utility.get().getConfiguration(); + HBaseTestingUtility testingUtility = new HBaseTestingUtility(); + Configuration conf = testingUtility.getConfiguration(); HadoopVariantStorageTest.configuration.set(conf); @@ -250,8 +267,18 @@ public void before() throws Exception { // Do not put up web UI conf.setInt("hbase.regionserver.info.port", -1); conf.setInt("hbase.master.info.port", -1); + + if (VersionInfo.getVersion().startsWith("2.4") && org.apache.hadoop.util.VersionInfo.getVersion().startsWith("3.3")) { + // Disable async wal provider, not supported in HBase 2.4 and HDFS 3.3 + System.out.println("Disabling async wal provider"); + conf.set(WALFactory.WAL_PROVIDER, FSHLogProvider.class.getName()); + conf.set(WALFactory.META_WAL_PROVIDER, FSHLogProvider.class.getName()); +// conf.setBoolean(WALFactory.WAL_ENABLED, false); + } + //org.apache.commons.configuration2.Configuration - utility.get().startMiniCluster(1); + testingUtility.startMiniCluster(1); + utility.set(testingUtility); // MiniMRCluster miniMRCluster = utility.startMiniMapReduceCluster(); // MiniMRClientCluster miniMRClientCluster = MiniMRClientClusterFactory.create(HadoopVariantStorageManagerTestUtils.class, 1, configuration); @@ -466,10 +493,9 @@ default void clearDB(String tableName) throws Exception { default void deleteTable(String tableName) throws Exception { LoggerFactory.getLogger(HadoopVariantStorageTest.class).info("Drop table " + tableName); - PhoenixHelper phoenixHelper = new PhoenixHelper(configuration.get()); - try (java.sql.Connection con = phoenixHelper.openJdbcConnection()) { - if (phoenixHelper.tableExists(con, tableName)) { - phoenixHelper.dropTable(con, tableName, VariantPhoenixSchema.DEFAULT_TABLE_TYPE, true, true); + if (HBaseVariantTableNameGenerator.isValidVariantsTable(tableName)) { + try (HBaseManager hbaseManager = new HBaseManager(configuration.get(), utility.get().getConnection())) { + VariantPhoenixSchemaManager.dropView(hbaseManager, tableName, true); } } utility.get().deleteTableIfAny(TableName.valueOf(tableName)); @@ -509,7 +535,7 @@ public static void setStaticConfiguration(Configuration staticConfiguration) { @Override - public int run(Class clazz, String[] args) throws StorageEngineException { + public Result run(Class clazz, String[] args) throws StorageEngineException { try { // Copy configuration Configuration conf = new Configuration(false); @@ -522,14 +548,14 @@ public int run(Class clazz, String[] args) throws StorageEng if (((Number) o).intValue() != 0) { throw new StorageEngineException("Error executing MapReduce. Exit code: " + o); } - return ((Number) o).intValue(); + return new Result(((Number) o).intValue(), new ObjectMap()); } catch (Exception e) { throw new StorageEngineException("Error executing MapReduce.", e); } } @Override - public int run(String executable, String[] args) { + public Result run(String executable, String[] args) { try { // Copy configuration Configuration conf = new Configuration(false); @@ -544,7 +570,7 @@ public int run(String executable, String[] args) { if (((Number) o).intValue() != 0) { throw new RuntimeException("Exit code = " + o); } - return ((Number) o).intValue(); + return new Result(((Number) o).intValue(), new ObjectMap()); } catch (Exception e) { // e.printStackTrace(); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HadoopVariantDBAdaptorTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HadoopVariantDBAdaptorTest.java index ca0d268d4f5..8cead56116a 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HadoopVariantDBAdaptorTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HadoopVariantDBAdaptorTest.java @@ -119,13 +119,14 @@ public void before() throws Exception { e.printStackTrace(); } } + variantStorageEngine.getOptions().append(VariantStorageOptions.ASSEMBLY.key(), "GRCH38"); cellBaseUtils = variantStorageEngine.getCellBaseUtils(); - expectedConnections = GlobalClientMetrics.GLOBAL_OPEN_PHOENIX_CONNECTIONS.getMetric().getTotalSum(); + expectedConnections = GlobalClientMetrics.GLOBAL_OPEN_PHOENIX_CONNECTIONS.getMetric().getValue(); } @After public void after() throws IOException { - assertEquals(expectedConnections, GlobalClientMetrics.GLOBAL_OPEN_PHOENIX_CONNECTIONS.getMetric().getTotalSum()); + assertEquals(expectedConnections, GlobalClientMetrics.GLOBAL_OPEN_PHOENIX_CONNECTIONS.getMetric().getValue()); super.after(); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixKeyFactoryTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixKeyFactoryTest.java index 80692ba9a48..5f514f7483c 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixKeyFactoryTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixKeyFactoryTest.java @@ -18,6 +18,7 @@ import org.opencb.biodata.models.variant.avro.StructuralVariation; import org.opencb.opencga.core.testclassification.duration.ShortTests; import org.opencb.opencga.storage.hadoop.variant.GenomeHelper; +import org.opencb.opencga.storage.hadoop.HBaseCompat; import java.sql.SQLException; import java.util.*; @@ -110,7 +111,7 @@ public byte[] generateVariantRowKeyPhoenix(Variant variant) { VariantPhoenixSchema.VariantColumn.ALTERNATE )); - PTableImpl table; + PTable table; try { List columns = new ArrayList<>(); for (PhoenixHelper.Column column : VariantPhoenixSchema.PRIMARY_KEY) { @@ -121,8 +122,7 @@ public byte[] generateVariantRowKeyPhoenix(Variant variant) { .setNullable(nullableColumn.contains(column)) .setSortOrder(SortOrder.ASC.getSystemValue()).build())); } - - table = PTableImpl.makePTable(new PTableImpl(), columns); + table = HBaseCompat.getInstance().getPhoenixCompat().makePTable(columns); } catch (SQLException e) { throw new RuntimeException(e); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveDeleteHBaseColumnTaskTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveDeleteHBaseColumnTaskTest.java new file mode 100644 index 00000000000..5513bcdcb7a --- /dev/null +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveDeleteHBaseColumnTaskTest.java @@ -0,0 +1,59 @@ +package org.opencb.opencga.storage.hadoop.variant.archive; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.junit.Assert; +import org.junit.Test; +import org.opencb.commons.datastore.core.ObjectMap; + +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.opencb.opencga.storage.hadoop.variant.archive.ArchiveDeleteHBaseColumnTask.*; + +public class ArchiveDeleteHBaseColumnTaskTest { + + private ArchiveDeleteHBaseColumnTask task; + + @Test + public void testConfigureTask() { + ObjectMap options = new ObjectMap(); + task = new ArchiveDeleteHBaseColumnTask(); + configureTask(options, Arrays.asList(1, 2, 3)); + assertEquals(Arrays.asList(0), options.getAsIntegerList(FILE_BATCHES_WITH_FILES_TO_DELETE_FROM_ARCHIVE_INDEX)); + List> regionsToDelete = task.getRegionsToDelete(toConf(options)); + assertEquals(1, regionsToDelete.size()); + + assertArrayEquals(Bytes.toBytes("00000_"), regionsToDelete.get(0).getFirst()); + assertArrayEquals(Bytes.toBytes("00001_"), regionsToDelete.get(0).getSecond()); + } + + @Test + public void testConfigureTaskMultiRegions() { + ObjectMap options = new ObjectMap(); + task = new ArchiveDeleteHBaseColumnTask(); + configureTask(options, Arrays.asList(5300, 6053, 9032)); + assertEquals(Arrays.asList(5, 6, 9), options.getAsIntegerList(FILE_BATCHES_WITH_FILES_TO_DELETE_FROM_ARCHIVE_INDEX)); + List> regionsToDelete = task.getRegionsToDelete(toConf(options)); + assertEquals(3, regionsToDelete.size()); + + assertArrayEquals(Bytes.toBytes("00005_"), regionsToDelete.get(0).getFirst()); + assertArrayEquals(Bytes.toBytes("00006_"), regionsToDelete.get(0).getSecond()); + assertArrayEquals(Bytes.toBytes("00006_"), regionsToDelete.get(1).getFirst()); + assertArrayEquals(Bytes.toBytes("00007_"), regionsToDelete.get(1).getSecond()); + assertArrayEquals(Bytes.toBytes("00009_"), regionsToDelete.get(2).getFirst()); + assertArrayEquals(Bytes.toBytes("00010_"), regionsToDelete.get(2).getSecond()); + } + + private static Configuration toConf(ObjectMap options) { + Configuration conf = new Configuration(); + for (String key : options.keySet()) { + conf.set(key, options.getString(key)); + } + return conf; + } + +} \ No newline at end of file diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTaskTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTaskTest.java index 3d9db73719e..63f579cb99e 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTaskTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTaskTest.java @@ -13,6 +13,7 @@ import org.opencb.biodata.models.variant.metadata.VariantFileHeaderComplexLine; import org.opencb.biodata.models.variant.protobuf.VcfSliceProtos; import org.opencb.biodata.tools.variant.converters.proto.VariantToVcfSliceConverter; +import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.core.testclassification.duration.ShortTests; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; @@ -44,6 +45,7 @@ public class FillGapsTaskTest { public void setUp() throws Exception { DummyVariantStorageMetadataDBAdaptorFactory.clear(); metadataManager = new VariantStorageMetadataManager(new DummyVariantStorageMetadataDBAdaptorFactory()); + metadataManager.getAndUpdateProjectMetadata(new ObjectMap()); studyMetadata = metadataManager.createStudy("S"); metadataManager.updateStudyMetadata("S", sm -> { sm.getAttributes().put(VariantStorageOptions.EXTRA_FORMAT_FIELDS.key(), "DP"); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexTest.java index f67d5734d7c..a6aeba91145 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexTest.java @@ -63,13 +63,13 @@ public class FamilyIndexTest extends VariantStorageBaseTest implements HadoopVar @Before public void before() throws Exception { + HadoopVariantStorageEngine variantStorageEngine = getVariantStorageEngine(); + variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL); + variantStorageEngine.getConfiguration().getCellbase().setVersion("v5.2"); + variantStorageEngine.getConfiguration().getCellbase().setDataRelease("3"); + variantStorageEngine.getOptions().put(VariantStorageOptions.ASSEMBLY.key(), "grch38"); + variantStorageEngine.reloadCellbaseConfiguration(); if (!loaded) { - HadoopVariantStorageEngine variantStorageEngine = getVariantStorageEngine(); - variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL); - variantStorageEngine.getConfiguration().getCellbase().setVersion("v5.2"); - variantStorageEngine.getConfiguration().getCellbase().setDataRelease("3"); - variantStorageEngine.getOptions().put(VariantStorageOptions.ASSEMBLY.key(), "grch38"); - variantStorageEngine.reloadCellbaseConfiguration(); URI outputUri = newOutputUri(); ObjectMap params = new ObjectMap(VariantStorageOptions.ANNOTATE.key(), false) @@ -91,7 +91,7 @@ public void before() throws Exception { variantStorageEngine.annotate(outputUri, new ObjectMap()); - VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri(getTestName().getMethodName())); + VariantHbaseTestUtils.printVariants(variantStorageEngine.getDBAdaptor(), newOutputUri(getTestName().getMethodName())); mendelianErrorVariants = new HashSet<>(); deNovoVariants = new HashSet<>(); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexDBAdaptorTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexDBAdaptorTest.java index a46a3a5c99b..dc87e7d6d11 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexDBAdaptorTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexDBAdaptorTest.java @@ -15,6 +15,7 @@ import org.opencb.opencga.storage.hadoop.variant.index.query.LocusQuery; import org.opencb.opencga.storage.hadoop.variant.index.query.SampleAnnotationIndexQuery; import org.opencb.opencga.storage.hadoop.variant.index.query.SampleIndexQuery; +import org.opencb.opencga.storage.hadoop.variant.utils.HBaseVariantTableNameGenerator; import java.util.Arrays; import java.util.Collections; @@ -46,7 +47,9 @@ public void testSampleIdFF() throws Exception { Collections.singletonMap(sampleName, Collections.singletonList("0/1")), Collections.emptySet(), null, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), new SampleAnnotationIndexQuery(SampleIndexSchema.defaultSampleIndexSchema()), Collections.emptySet(), null, false, VariantQueryUtils.QueryOperation.AND, null); - new SampleIndexDBAdaptor(new HBaseManager(new Configuration()), null, metadataManager).parse(query.forSample(sampleName), null); + new SampleIndexDBAdaptor(new HBaseManager(new Configuration()), + new HBaseVariantTableNameGenerator("default", "my_dbname"), metadataManager) + .parse(query.forSample(sampleName), null); } @Test diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexEntryFilterTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexEntryFilterTest.java index 76cc13feeb9..a631c07e099 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexEntryFilterTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexEntryFilterTest.java @@ -259,6 +259,7 @@ private SingleSampleIndexQuery getSingleSampleIndexQuery(Query query) { private SingleSampleIndexQuery getSingleSampleIndexQuery(VariantQueryUtils.QueryOperation op, RangeIndexFieldFilter... frequencyQuery) { SampleAnnotationIndexQuery annotationIndexQuery = new SampleAnnotationIndexQuery( new byte[2], + null, schema.getCtIndex().getField().noOpFilter(), schema.getBiotypeIndex().getField().noOpFilter(), schema.getTranscriptFlagIndexSchema().getField().noOpFilter(), diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParserTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParserTest.java index 212b0521b26..e9c46d5214f 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParserTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParserTest.java @@ -1387,13 +1387,13 @@ public void parseIntergenicTest() { checkIntergenic(null, new Query(ANNOT_CONSEQUENCE_TYPE.key(), VariantAnnotationConstants.REGULATORY_REGION_VARIANT)); checkIntergenic(false, new Query(ANNOT_CONSEQUENCE_TYPE.key(), VariantAnnotationConstants.REGULATORY_REGION_VARIANT) .append(ANNOT_BIOTYPE.key(), "protein_coding")); - // Nonsense combination checkIntergenic(false, new Query(ANNOT_CONSEQUENCE_TYPE.key(), "intergenic_variant").append(ANNOT_BIOTYPE.key(), "protein_coding")); } private void checkIntergenic(Boolean intergenic, Query query) { SampleAnnotationIndexQuery annotationIndexQuery = parseAnnotationIndexQuery(query); + assertEquals(intergenic, annotationIndexQuery.getIntergenic()); if (intergenic == null) { assertEquals(EMPTY_MASK, INTERGENIC_MASK & annotationIndexQuery.getAnnotationIndexMask()); } else { diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexTest.java index 144083e5bd0..98062c27b8f 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexTest.java @@ -942,7 +942,6 @@ private void testAggregationCorrectness(Query baseQuery, String ct) throws Excep .append(SAMPLE.key(), "NA12877") .append(ANNOT_CONSEQUENCE_TYPE.key(), ct); assertTrue(executor.canUseThisExecutor(query, new QueryOptions(QueryOptions.FACET, "consequenceType"))); - AtomicInteger count = new AtomicInteger(0); sampleIndexDBAdaptor.iterator(new Query(query), new QueryOptions()).forEachRemaining(v -> count.incrementAndGet()); FacetField facet = executor.aggregation(new Query(query), new QueryOptions(QueryOptions.FACET, "consequenceType")).first(); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/io/HadoopVariantExporterTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/io/HadoopVariantExporterTest.java index ddb7e993549..f7f89519477 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/io/HadoopVariantExporterTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/io/HadoopVariantExporterTest.java @@ -2,10 +2,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; +import org.junit.*; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -21,6 +18,7 @@ import org.opencb.opencga.storage.core.variant.io.VariantExporter; import org.opencb.opencga.storage.core.variant.io.VariantWriterFactory; import org.opencb.opencga.storage.core.variant.solr.VariantSolrExternalResource; +import org.opencb.opencga.storage.hadoop.HBaseCompat; import org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageEngine; import org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageTest; import org.opencb.opencga.storage.hadoop.variant.VariantHbaseTestUtils; @@ -55,7 +53,6 @@ public HadoopVariantExporterTest(String name, Boolean exportToLocal) { @ClassRule public static HadoopExternalResource externalResource = new HadoopExternalResource(); - @ClassRule public static VariantSolrExternalResource solr = new VariantSolrExternalResource(); private static HadoopVariantStorageEngine variantStorageEngine; @@ -65,7 +62,12 @@ public HadoopVariantExporterTest(String name, Boolean exportToLocal) { @BeforeClass public static void beforeClass() throws Exception { variantStorageEngine = externalResource.getVariantStorageEngine(); - solr.configure(variantStorageEngine); + if (HBaseCompat.getInstance().isSolrTestingAvailable()) { + solr.before(); + solr.configure(variantStorageEngine); + } else { + System.out.println("Skip embedded solr tests"); + } // URI inputUri = VariantStorageBaseTest.getResourceUri("sample1.genome.vcf"); URI inputUri = VariantStorageBaseTest.getResourceUri("platinum/1K.end.platinum-genomes-vcf-NA12877_S1.genome.vcf.gz"); @@ -87,12 +89,21 @@ public static void beforeClass() throws Exception { .append(VariantStorageOptions.STATS_CALCULATE.key(), false) ); - variantStorageEngine.secondaryIndex(); + if (HBaseCompat.getInstance().isSolrTestingAvailable()) { + variantStorageEngine.secondaryIndex(); + } VariantHbaseTestUtils.printVariants(variantStorageEngine.getDBAdaptor(), newOutputUri()); } + @AfterClass + public static void afterClass() { + if (HBaseCompat.getInstance().isSolrTestingAvailable()) { + solr.after(); + } + } + @Before public void before() throws Exception { // Do not clean database! diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/load/VariantHadoopDBWriterTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/load/VariantHadoopDBWriterTest.java index 607069f93c9..9bfcaedee46 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/load/VariantHadoopDBWriterTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/load/VariantHadoopDBWriterTest.java @@ -4,11 +4,11 @@ import org.junit.experimental.categories.Category; import org.opencb.biodata.models.variant.Variant; import org.opencb.opencga.core.testclassification.duration.ShortTests; -import org.opencb.opencga.storage.hadoop.variant.archive.VariantHbaseTransformTask; +import org.opencb.opencga.storage.hadoop.variant.transform.VariantSliceReader; import java.util.*; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; @Category(ShortTests.class) public class VariantHadoopDBWriterTest { @@ -24,7 +24,7 @@ public void testFilterVariantsNotFromThisSlice() { Map> slices = new TreeMap<>(); for (Variant variant : variants) { - for (long l : VariantHbaseTransformTask.getCoveredSlicePositions(variant.getStart(), variant.getEnd(), chunkSize)) { + for (long l : VariantSliceReader.getCoveredSlicePositions(variant.getStart(), variant.getEnd(), chunkSize)) { slices.computeIfAbsent(l, k -> new LinkedList<>()).add(variant); } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/query/executors/HadoopVariantQueryExecutorTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/query/executors/HadoopVariantQueryExecutorTest.java index 7cc495bbe63..aa62af0944f 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/query/executors/HadoopVariantQueryExecutorTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/query/executors/HadoopVariantQueryExecutorTest.java @@ -9,8 +9,6 @@ import org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageTest; import org.opencb.opencga.storage.hadoop.variant.VariantHbaseTestUtils; -import static org.opencb.opencga.storage.hadoop.variant.VariantHbaseTestUtils.printVariants; - @Category(LongTests.class) public class HadoopVariantQueryExecutorTest extends VariantQueryExecutorTest implements HadoopVariantStorageTest { @@ -18,6 +16,15 @@ public class HadoopVariantQueryExecutorTest extends VariantQueryExecutorTest imp @ClassRule public static ExternalResource externalResource = new HadoopExternalResource(); + @Override + public void initSolr() throws Exception { + if (HadoopVariantStorageTest.HadoopSolrSupport.isSolrTestingAvailable()) { + super.initSolr(); + } else { + System.out.println("Solr testing not available"); + } + } + @Override @Before public void setUp() throws Exception { diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/search/HadoopSolrTestingSupportTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/search/HadoopSolrTestingSupportTest.java new file mode 100644 index 00000000000..feabf69984b --- /dev/null +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/search/HadoopSolrTestingSupportTest.java @@ -0,0 +1,33 @@ +package org.opencb.opencga.storage.hadoop.variant.search; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.opencb.opencga.core.testclassification.duration.ShortTests; +import org.opencb.opencga.storage.core.variant.solr.VariantSolrExternalResource; +import org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageTest; + +@Category(ShortTests.class) +public class HadoopSolrTestingSupportTest { + + @Test + public void testSupported() { + VariantSolrExternalResource externalResource = new VariantSolrExternalResource(); + try { + externalResource.before(); + externalResource.after(); + if (!HadoopVariantStorageTest.HadoopSolrSupport.isSolrTestingAvailable()) { + Assert.fail("Solr testing should not be available"); + } else { + System.out.println("As expected :: Solr testing available"); + } + } catch (Throwable throwable) { + if (HadoopVariantStorageTest.HadoopSolrSupport.isSolrTestingAvailable()) { + Assert.fail("Solr testing should be available"); + } else { + System.out.println("As expected :: Solr testing not available: " + throwable.getMessage()); + } + } + } + +} diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/search/HadoopVariantSearchIndexTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/search/HadoopVariantSearchIndexTest.java index 7f6869a59c0..240ece95e01 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/search/HadoopVariantSearchIndexTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/search/HadoopVariantSearchIndexTest.java @@ -1,7 +1,6 @@ package org.opencb.opencga.storage.hadoop.variant.search; import org.junit.ClassRule; -import org.junit.Rule; import org.junit.experimental.categories.Category; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; @@ -20,6 +19,9 @@ @Category(LongTests.class) public class HadoopVariantSearchIndexTest extends VariantSearchIndexTest implements HadoopVariantStorageTest { + @ClassRule + public static HadoopSolrSupport solrSupport = new HadoopSolrSupport(); + @ClassRule public static HadoopExternalResource externalResource = new HadoopExternalResource(); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-cdh5.13/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-cdh5.13/pom.xml deleted file mode 100644 index 15a012e5b00..00000000000 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-cdh5.13/pom.xml +++ /dev/null @@ -1,331 +0,0 @@ - - - - 4.0.0 - - - org.opencb.opencga - opencga-storage-hadoop-deps - 2.6.0-SNAPSHOT - ../pom.xml - - - - opencga-storage-hadoop-deps-cdh5.13 - jar - - - ${artifactId} - cdh5.13.0 - - 2.6.0-${chd.dependencies.version} - 1.2.0-${chd.dependencies.version} - 4.7.0-clabs-phoenix1.3.0 - 0.7.0 - - - - - cloudera - https://repository.cloudera.com/artifactory/cloudera-repos/ - - - - - - - com.google.protobuf - protobuf-java - ${protobuf2.version} - true - - - com.google.guava - guava - ${guava.version} - true - - - - - - - - - - org.apache.hadoop - hadoop-common - ${hadoop.version} - true - - - org.apache.hadoop - hadoop-client - ${hadoop.version} - true - - - - - org.apache.hbase - hbase-server - ${hbase.version} - true - - - - - org.apache.phoenix - phoenix-core - ${phoenix.version} - true - - - - - - - - org.apache.hadoop - hadoop-minicluster - ${hadoop.version} - true - - - org.apache.hbase - hbase-testing-util - ${hbase.version} - test-jar - true - - - org.apache.hbase - hbase-it - ${hbase.version} - test-jar - true - - - - - - - - - org.apache.htrace - htrace-core - 3.1.0-incubating - - - commons-cli - commons-cli - 1.2 - - - - - org.apache.avro - avro - - - org.apache.avro - avro-ipc - - - org.apache.avro - avro-mapred - hadoop2 - - - - - commons-configuration - commons-configuration - - - commons-lang - commons-lang - - - commons-logging - commons-logging - - - commons-io - commons-io - ${commons-io.version} - - - commons-net - commons-net - ${commons-net.version} - - - commons-httpclient - commons-httpclient - ${commons-httpclient.version} - - - org.apache.curator - curator-recipes - ${curator.version} - - - org.apache.curator - curator-client - ${curator.version} - - - org.apache.curator - curator-framework - ${curator.version} - - - org.apache.zookeeper - zookeeper - ${zookeeper.version} - - - - - org.fusesource.leveldbjni - leveldbjni-all - ${leveldbjni-all.version} - - - - - org.apache.directory.server - apacheds-kerberos-codec - compile - ${apacheds-kerberos-codec.version} - - - - - com.lmax - disruptor - ${disruptor.version} - - - com.yammer.metrics - metrics-core - ${metrics-core.version} - - - org.apache.commons - commons-math - ${commons-math.version} - - - org.apache.commons - commons-compress - ${commons-compress.version} - - - io.netty - netty-all - ${netty.version} - - - org.jamon - jamon-runtime - ${jamon-runtime.version} - - - org.codehaus.jackson - jackson-core-asl - ${codehaus.jackson.version} - - - org.codehaus.jackson - jackson-mapper-asl - ${codehaus.jackson.version} - - - org.codehaus.jackson - jackson-jaxrs - ${codehaus.jackson.version} - - - org.codehaus.jackson - jackson-xc - ${codehaus.jackson.version} - - - - - - org.antlr - antlr-runtime - ${antlr.version} - - - jline - jline - ${jline.version} - - - sqlline - sqlline - ${sqlline.version} - - - co.cask.tephra - tephra-api - ${tephra.version} - - - co.cask.tephra - tephra-core - ${tephra.version} - - - co.cask.tephra - tephra-hbase-compat-1.1 - ${tephra.version} - - - org.apache.httpcomponents - httpclient - 4.0.1 - - - org.iq80.snappy - snappy - ${snappy.version} - - - commons-codec - commons-codec - ${commons-codec.version} - - - commons-collections - commons-collections - ${collections.version} - - - org.apache.commons - commons-csv - ${commons-csv.version} - - - - \ No newline at end of file diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr5.31/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr5.31/pom.xml deleted file mode 100644 index df161e6f122..00000000000 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr5.31/pom.xml +++ /dev/null @@ -1,387 +0,0 @@ - - - - 4.0.0 - - - org.opencb.opencga - opencga-storage-hadoop-deps - 2.6.0-SNAPSHOT - ../pom.xml - - - - opencga-storage-hadoop-deps-emr5.31 - jar - - - ${artifactId} - - 2.10.0-amzn-0 - 1.4.13 - 4.14.3-HBase-1.4 - 3.4.14 - 3.0.0 - - - 0.12.0-incubating - 0.7.0 - - - - - - - emr-5.31.0-artifacts - EMR 5.31.0 Releases Repository - - true - - - false - - https://s3.eu-west-2.amazonaws.com/eu-west-2-emr-artifacts/emr-5.31.0/repos/maven/ - - - - - - - com.google.protobuf - protobuf-java - ${protobuf2.version} - true - - - com.google.guava - guava - ${guava.version} - true - - - - - - - - - - org.apache.hadoop - hadoop-common - ${hadoop.version} - true - - - org.apache.hadoop - hadoop-client - ${hadoop.version} - true - - - org.apache.hadoop - hadoop-hdfs - ${hadoop.version} - true - - - - - org.apache.hbase - hbase-server - ${hbase.version} - true - - - org.apache.hbase.thirdparty - hbase-shaded-gson - ${hbase.shaded.gson.version} - - true - - - - - org.apache.phoenix - phoenix-core - ${phoenix.version} - true - - - - - - - - org.apache.hadoop - hadoop-minicluster - ${hadoop.version} - true - - - org.apache.hbase - hbase-testing-util - ${hbase.version} - test-jar - true - - - org.apache.hbase - hbase-it - ${hbase.version} - test-jar - true - - - - - - - - - org.apache.htrace - htrace-core - 3.1.0-incubating - - - commons-cli - commons-cli - 1.2 - - - - - org.apache.avro - avro - - - org.apache.avro - avro-ipc - - - org.apache.avro - avro-mapred - hadoop2 - - - - - commons-configuration - commons-configuration - - - commons-lang - commons-lang - - - commons-logging - commons-logging - - - commons-io - commons-io - ${commons-io.version} - - - commons-net - commons-net - ${commons-net.version} - - - commons-httpclient - commons-httpclient - ${commons-httpclient.version} - - - org.apache.curator - curator-recipes - ${curator.version} - - - org.apache.curator - curator-client - ${curator.version} - - - org.apache.curator - curator-framework - ${curator.version} - - - org.apache.zookeeper - zookeeper - ${zookeeper.version} - - - - - org.fusesource.leveldbjni - leveldbjni-all - ${leveldbjni-all.version} - - - - - org.apache.directory.server - apacheds-kerberos-codec - compile - ${apacheds-kerberos-codec.version} - - - - - com.lmax - disruptor - ${disruptor.version} - - - com.yammer.metrics - metrics-core - ${metrics-core.version} - - - org.apache.commons - commons-math - ${commons-math.version} - - - org.apache.commons - commons-compress - ${commons-compress.version} - - - io.netty - netty-all - ${netty.version} - - - org.jamon - jamon-runtime - ${jamon-runtime.version} - - - org.codehaus.jackson - jackson-core-asl - ${codehaus.jackson.version} - - - org.codehaus.jackson - jackson-mapper-asl - ${codehaus.jackson.version} - - - org.codehaus.jackson - jackson-jaxrs - ${codehaus.jackson.version} - - - org.codehaus.jackson - jackson-xc - ${codehaus.jackson.version} - - - - - - org.antlr - antlr-runtime - ${antlr.version} - - - jline - jline - ${jline.version} - - - sqlline - sqlline - ${sqlline.version} - - - - - - - - - - - - - - - - - - org.apache.tephra - tephra-api - ${apache.tephra.version} - - - org.apache.tephra - tephra-core - ${apache.tephra.version} - - - ch.qos.logback - logback-core - - - ch.qos.logback - logback-classic - - - - - org.apache.tephra - tephra-hbase-compat-1.1 - ${apache.tephra.version} - - - org.apache.httpcomponents - httpclient - 4.0.1 - - - org.iq80.snappy - snappy - ${snappy.version} - - - commons-codec - commons-codec - ${commons-codec.version} - - - commons-collections - commons-collections - ${collections.version} - - - org.apache.commons - commons-csv - ${commons-csv.version} - - - - \ No newline at end of file diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr5.8/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr5.8/pom.xml deleted file mode 100644 index 1cae0132bf2..00000000000 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr5.8/pom.xml +++ /dev/null @@ -1,353 +0,0 @@ - - - - 4.0.0 - - - org.opencb.opencga - opencga-storage-hadoop-deps - 2.6.0-SNAPSHOT - ../pom.xml - - - - opencga-storage-hadoop-deps-emr5.8 - jar - - - ${artifactId} - - 2.7.3 - 1.3.1 - 4.11.0-HBase-1.3 - - - 0.12.0-incubating - 0.7.0 - - - - - - com.google.protobuf - protobuf-java - ${protobuf2.version} - true - - - com.google.guava - guava - ${guava.version} - true - - - - - - - - - - org.apache.hadoop - hadoop-common - ${hadoop.version} - true - - - org.apache.hadoop - hadoop-client - ${hadoop.version} - true - - - - - org.apache.hbase - hbase-server - ${hbase.version} - true - - - - - org.apache.phoenix - phoenix-core - ${phoenix.version} - true - - - - - - - - org.apache.hadoop - hadoop-minicluster - ${hadoop.version} - true - - - org.apache.hbase - hbase-testing-util - ${hbase.version} - test-jar - true - - - org.apache.hbase - hbase-it - ${hbase.version} - test-jar - true - - - - - - - - - org.apache.htrace - htrace-core - 3.1.0-incubating - - - commons-cli - commons-cli - 1.2 - - - - - org.apache.avro - avro - - - org.apache.avro - avro-ipc - - - org.apache.avro - avro-mapred - hadoop2 - - - - - commons-configuration - commons-configuration - - - commons-lang - commons-lang - - - commons-logging - commons-logging - - - commons-io - commons-io - ${commons-io.version} - - - commons-net - commons-net - ${commons-net.version} - - - commons-httpclient - commons-httpclient - ${commons-httpclient.version} - - - org.apache.curator - curator-recipes - ${curator.version} - - - org.apache.curator - curator-client - ${curator.version} - - - org.apache.curator - curator-framework - ${curator.version} - - - org.apache.zookeeper - zookeeper - ${zookeeper.version} - - - - - org.fusesource.leveldbjni - leveldbjni-all - ${leveldbjni-all.version} - - - - - org.apache.directory.server - apacheds-kerberos-codec - compile - ${apacheds-kerberos-codec.version} - - - - - com.lmax - disruptor - ${disruptor.version} - - - com.yammer.metrics - metrics-core - ${metrics-core.version} - - - org.apache.commons - commons-math - ${commons-math.version} - - - org.apache.commons - commons-compress - ${commons-compress.version} - - - io.netty - netty-all - ${netty.version} - - - org.jamon - jamon-runtime - ${jamon-runtime.version} - - - org.codehaus.jackson - jackson-core-asl - ${codehaus.jackson.version} - - - org.codehaus.jackson - jackson-mapper-asl - ${codehaus.jackson.version} - - - org.codehaus.jackson - jackson-jaxrs - ${codehaus.jackson.version} - - - org.codehaus.jackson - jackson-xc - ${codehaus.jackson.version} - - - - - - org.antlr - antlr-runtime - ${antlr.version} - - - jline - jline - ${jline.version} - - - sqlline - sqlline - ${sqlline.version} - - - - - - - - - - - - - - - - - - org.apache.tephra - tephra-api - ${apache.tephra.version} - - - org.apache.tephra - tephra-core - ${apache.tephra.version} - - - ch.qos.logback - logback-core - - - ch.qos.logback - logback-classic - - - - - org.apache.tephra - tephra-hbase-compat-1.1 - ${apache.tephra.version} - - - org.apache.httpcomponents - httpclient - 4.0.1 - - - org.iq80.snappy - snappy - ${snappy.version} - - - commons-codec - commons-codec - ${commons-codec.version} - - - commons-collections - commons-collections - ${collections.version} - - - org.apache.commons - commons-csv - ${commons-csv.version} - - - - \ No newline at end of file diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml deleted file mode 100644 index 84c787463ec..00000000000 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml +++ /dev/null @@ -1,519 +0,0 @@ - - - - 4.0.0 - - - org.opencb.opencga - opencga-storage-hadoop-deps - 3.2.1-SNAPSHOT - ../pom.xml - - - - opencga-storage-hadoop-deps-emr6.1 - jar - - - ${artifactId} - - 3.2.1-amzn-1 - 2.2.5 - - - 2.2.1 - - 3.4.14 - 5.0.0-HBase-2.0 - 0.14.0-incubating - 0.7.0 - - 3.3.6 - 3.2.1 - 3.6.1 - - 1.18 - 2.5 - 3.6 - 2.12.0 - - 28.0-jre - - - - - - emr-6.1.0-artifacts - EMR 6.1.0 Releases Repository - - true - - - false - - https://s3.eu-west-2.amazonaws.com/eu-west-2-emr-artifacts/emr-6.1.0/repos/maven/ - - - - - - - - - com.google.protobuf - protobuf-java - ${protobuf2.version} - true - - - com.google.guava - guava - ${guava.version} - true - - - - - - - - - - org.apache.hadoop - hadoop-common - ${hadoop.version} - true - - - org.apache.hadoop - hadoop-client - ${hadoop.version} - true - - - org.apache.hadoop - hadoop-hdfs - ${hadoop.version} - true - - - org.apache.hadoop - hadoop-aws - ${hadoop.version} - true - - - - - org.apache.hbase - hbase-server - ${hbase.version} - true - - - org.apache.hbase - hbase-mapreduce - ${hbase.version} - true - - - - - org.apache.phoenix - phoenix-core - ${phoenix.version} - true - - - - - - - - org.apache.hadoop - hadoop-minicluster - ${hadoop.version} - true - - - org.apache.hbase - hbase-testing-util - ${hbase.version} - test-jar - true - - - org.apache.hbase - hbase-it - ${hbase.version} - test-jar - true - - - - - - - - - - org.apache.htrace - htrace-core - 3.1.0-incubating - - - commons-cli - commons-cli - 1.2 - - - - - org.apache.avro - avro - - - org.apache.avro - avro-ipc - - - org.apache.avro - avro-mapred - hadoop2 - - - - - - - - - commons-lang - commons-lang - - - commons-logging - commons-logging - - - commons-io - commons-io - ${commons-io.version} - - - commons-net - commons-net - ${commons-net.version} - - - commons-httpclient - commons-httpclient - ${commons-httpclient.version} - - - org.apache.curator - curator-recipes - ${curator.version} - - - org.apache.curator - curator-client - ${curator.version} - - - org.apache.curator - curator-framework - ${curator.version} - - - org.apache.zookeeper - zookeeper - ${zookeeper.version} - - - - - org.fusesource.leveldbjni - leveldbjni-all - ${leveldbjni-all.version} - - - - - org.apache.directory.server - apacheds-kerberos-codec - compile - ${apacheds-kerberos-codec.version} - - - - - com.microsoft.azure - azure-storage - 2.0.0 - - - - org.mortbay.jetty - jetty-util - ${mortbay.jetty.version} - - - - - com.lmax - disruptor - ${disruptor.version} - - - - - - - - org.apache.commons - commons-math - ${commons-math.version} - - - org.apache.commons - commons-compress - ${commons-compress.version} - - - io.netty - netty-all - ${netty.version} - - - org.jamon - jamon-runtime - ${jamon-runtime.version} - - - org.codehaus.jackson - jackson-core-asl - ${codehaus.jackson.version} - - - org.codehaus.jackson - jackson-mapper-asl - ${codehaus.jackson.version} - - - org.codehaus.jackson - jackson-jaxrs - ${codehaus.jackson.version} - - - org.codehaus.jackson - jackson-xc - ${codehaus.jackson.version} - - - - - - org.antlr - antlr-runtime - ${antlr.version} - - - jline - jline - ${jline.version} - - - sqlline - sqlline - ${sqlline.version} - - - - - - - - - - - - - - - - - - org.apache.httpcomponents - httpclient - 4.0.1 - - - org.iq80.snappy - snappy - ${snappy.version} - - - commons-codec - commons-codec - ${commons-codec.version} - - - commons-collections - commons-collections - ${collections.version} - - - org.apache.commons - commons-csv - ${commons-csv.version} - - - joda-time - joda-time - - - - - org.apache.hbase.thirdparty - hbase-shaded-netty - ${hbase-thirdparty.version} - - true - - - org.apache.tephra - tephra-api - ${apache.tephra.version} - - - org.apache.tephra - tephra-core - ${apache.tephra.version} - - - ch.qos.logback - logback-core - - - ch.qos.logback - logback-classic - - - - - org.apache.tephra - tephra-hbase-compat-1.1 - ${apache.tephra.version} - - - - - - org.apache.commons - commons-configuration2 - 2.1.1 - - - org.apache.commons - commons-lang3 - - - - - - - org.apache.commons - commons-math3 - ${commons-math3.version} - - - - - io.dropwizard.metrics - metrics-core - ${io.dropwizard.metrics-core.version} - - - - - org.apache.htrace - htrace-core4 - 4.1.0-incubating - - - - com.google.re2j - re2j - 1.1 - - - - org.apache.hbase.thirdparty - hbase-shaded-protobuf - ${hbase-thirdparty.version} - - - - org.apache.hbase.thirdparty - hbase-shaded-miscellaneous - ${hbase-thirdparty.version} - - - org.apache.yetus - audience-annotations - 0.5.0 - - - - org.apache.hadoop - hadoop-azure-datalake - ${hadoop.version} - true - - - - org.wildfly.openssl - wildfly-openssl - 1.0.7.Final - - - - - \ No newline at end of file diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.5/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.5/pom.xml deleted file mode 100644 index 2237a95640c..00000000000 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.5/pom.xml +++ /dev/null @@ -1,337 +0,0 @@ - - - - 4.0.0 - - - org.opencb.opencga - opencga-storage-hadoop-deps - 2.6.0-SNAPSHOT - ../pom.xml - - - - opencga-storage-hadoop-deps-hdp2.5 - ${parent.version} - jar - - - ${artifactId} - 2.5.6.13-3 - - 2.7.3.${hdp.dependencies.version} - 1.1.2.${hdp.dependencies.version} - 4.7.0.${hdp.dependencies.version} - 0.7.0 - - - - - hortonworks-releases - https://repo.hortonworks.com/content/repositories/releases/ - - - hortonworks-public - https://repo.hortonworks.com/content/groups/public - - - - - - - - com.google.protobuf - protobuf-java - ${protobuf2.version} - true - - - com.google.guava - guava - ${guava.version} - true - - - - - - - - - - org.apache.hadoop - hadoop-common - ${hadoop.version} - true - - - org.apache.hadoop - hadoop-client - ${hadoop.version} - true - - - - - org.apache.hbase - hbase-server - ${hbase.version} - true - - - - - org.apache.phoenix - phoenix-core - ${phoenix.version} - true - - - - - - - - org.apache.hadoop - hadoop-minicluster - ${hadoop.version} - true - - - org.apache.hbase - hbase-testing-util - ${hbase.version} - test-jar - true - - - org.apache.hbase - hbase-it - ${hbase.version} - test-jar - true - - - - - - - - - org.apache.htrace - htrace-core - 3.1.0-incubating - - - commons-cli - commons-cli - 1.2 - - - - - org.apache.avro - avro - - - org.apache.avro - avro-ipc - - - org.apache.avro - avro-mapred - hadoop2 - - - - - commons-configuration - commons-configuration - - - commons-lang - commons-lang - - - commons-logging - commons-logging - - - commons-io - commons-io - ${commons-io.version} - - - commons-net - commons-net - ${commons-net.version} - - - commons-httpclient - commons-httpclient - ${commons-httpclient.version} - - - org.apache.curator - curator-recipes - ${curator.version} - - - org.apache.curator - curator-client - ${curator.version} - - - org.apache.curator - curator-framework - ${curator.version} - - - org.apache.zookeeper - zookeeper - ${zookeeper.version} - - - - - org.fusesource.leveldbjni - leveldbjni-all - ${leveldbjni-all.version} - - - - - org.apache.directory.server - apacheds-kerberos-codec - compile - ${apacheds-kerberos-codec.version} - - - - - com.lmax - disruptor - ${disruptor.version} - - - com.yammer.metrics - metrics-core - ${metrics-core.version} - - - org.apache.commons - commons-math - ${commons-math.version} - - - org.apache.commons - commons-compress - ${commons-compress.version} - - - io.netty - netty-all - ${netty.version} - - - org.jamon - jamon-runtime - ${jamon-runtime.version} - - - org.codehaus.jackson - jackson-core-asl - ${codehaus.jackson.version} - - - org.codehaus.jackson - jackson-mapper-asl - ${codehaus.jackson.version} - - - org.codehaus.jackson - jackson-jaxrs - ${codehaus.jackson.version} - - - org.codehaus.jackson - jackson-xc - ${codehaus.jackson.version} - - - - - - org.antlr - antlr-runtime - ${antlr.version} - - - jline - jline - ${jline.version} - - - sqlline - sqlline - ${sqlline.version} - - - co.cask.tephra - tephra-api - ${tephra.version} - - - co.cask.tephra - tephra-core - ${tephra.version} - - - co.cask.tephra - tephra-hbase-compat-1.1 - ${tephra.version} - - - org.apache.httpcomponents - httpclient - 4.0.1 - - - org.iq80.snappy - snappy - ${snappy.version} - - - commons-codec - commons-codec - ${commons-codec.version} - - - commons-collections - commons-collections - ${collections.version} - - - org.apache.commons - commons-csv - ${commons-csv.version} - - - - \ No newline at end of file diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml deleted file mode 100644 index f9c584917b1..00000000000 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml +++ /dev/null @@ -1,376 +0,0 @@ - - - - 4.0.0 - - - org.opencb.opencga - opencga-storage-hadoop-deps - 3.2.1-SNAPSHOT - ../pom.xml - - - - opencga-storage-hadoop-deps-hdp2.6 - jar - - - ${artifactId} - - 2.6.5.298-4 - - - 2.7.3.${hdp.dependencies.version} - 1.1.2.${hdp.dependencies.version} - 4.7.0.${hdp.dependencies.version} - 0.7.0 - - - - - hortonworks-releases - https://repo.hortonworks.com/content/repositories/releases/ - - - hortonworks-public - https://repo.hortonworks.com/content/groups/public - - - - - - - com.google.protobuf - protobuf-java - ${protobuf2.version} - true - - - com.google.guava - guava - ${guava.version} - true - - - - - - - - - - org.apache.hadoop - hadoop-common - ${hadoop.version} - true - - - org.apache.hadoop - hadoop-client - ${hadoop.version} - true - - - org.apache.hadoop - hadoop-azure - ${hadoop.version} - - - - org.apache.hadoop - hadoop-azure-datalake - ${hadoop.version} - - - org.apache.hadoop - hadoop-common - - - org.apache.commons - commons-lang3 - - - - - - - - org.apache.hbase - hbase-server - ${hbase.version} - true - - - - - org.apache.phoenix - phoenix-core - ${phoenix.version} - true - - - - - - - - org.apache.hadoop - hadoop-minicluster - ${hadoop.version} - true - - - org.apache.hbase - hbase-testing-util - ${hbase.version} - test-jar - true - - - org.apache.hbase - hbase-it - ${hbase.version} - test-jar - true - - - - - - - - - org.apache.htrace - htrace-core - 3.1.0-incubating - - - commons-cli - commons-cli - 1.2 - - - - - org.apache.avro - avro - - - org.apache.avro - avro-ipc - - - org.apache.avro - avro-mapred - hadoop2 - - - - - commons-configuration - commons-configuration - - - commons-lang - commons-lang - - - commons-logging - commons-logging - - - commons-io - commons-io - ${commons-io.version} - - - commons-net - commons-net - ${commons-net.version} - - - commons-httpclient - commons-httpclient - ${commons-httpclient.version} - - - org.apache.curator - curator-recipes - ${curator.version} - - - org.apache.curator - curator-client - ${curator.version} - - - org.apache.curator - curator-framework - ${curator.version} - - - org.apache.zookeeper - zookeeper - ${zookeeper.version} - - - - - org.fusesource.leveldbjni - leveldbjni-all - ${leveldbjni-all.version} - - - - - org.apache.directory.server - apacheds-kerberos-codec - compile - ${apacheds-kerberos-codec.version} - - - - - com.microsoft.azure - azure-storage - 2.0.0 - - - - org.mortbay.jetty - jetty-util - ${mortbay.jetty.version} - - - - - com.lmax - disruptor - ${disruptor.version} - - - com.yammer.metrics - metrics-core - ${metrics-core.version} - - - org.apache.commons - commons-math - ${commons-math.version} - - - org.apache.commons - commons-compress - ${commons-compress.version} - - - io.netty - netty-all - ${netty.version} - - - org.jamon - jamon-runtime - ${jamon-runtime.version} - - - org.codehaus.jackson - jackson-core-asl - ${codehaus.jackson.version} - - - org.codehaus.jackson - jackson-mapper-asl - ${codehaus.jackson.version} - - - org.codehaus.jackson - jackson-jaxrs - ${codehaus.jackson.version} - - - org.codehaus.jackson - jackson-xc - ${codehaus.jackson.version} - - - - - - org.antlr - antlr-runtime - ${antlr.version} - - - jline - jline - ${jline.version} - - - sqlline - sqlline - ${sqlline.version} - - - co.cask.tephra - tephra-api - ${tephra.version} - - - co.cask.tephra - tephra-core - ${tephra.version} - - - co.cask.tephra - tephra-hbase-compat-1.1 - ${tephra.version} - - - org.apache.httpcomponents - httpclient - 4.0.1 - - - org.iq80.snappy - snappy - ${snappy.version} - - - commons-codec - commons-codec - ${commons-codec.version} - - - commons-collections - commons-collections - ${collections.version} - - - org.apache.commons - commons-csv - ${commons-csv.version} - - - joda-time - joda-time - - - - \ No newline at end of file diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml deleted file mode 100644 index bc999c1656f..00000000000 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml +++ /dev/null @@ -1,542 +0,0 @@ - - - - 4.0.0 - - - org.opencb.opencga - opencga-storage-hadoop-deps - 3.2.1-SNAPSHOT - ../pom.xml - - - - opencga-storage-hadoop-deps-hdp3.1 - jar - - - ${artifactId} - 3.1.4.41-5 - - 3.1.1.${hdp.dependencies.version} - 2.0.2.${hdp.dependencies.version} - - 2.1.0 - 3.4.6 - 5.0.0.${hdp.dependencies.version} - 0.14.0-incubating - 0.7.0 - - 3.3.6 - 3.2.1 - 3.6.1 - - 1.18 - 2.5 - 3.6 - 2.12.0 - - 9.3.24.v20180605 - - 28.0-jre - - - - hortonworks-releases - https://repo.hortonworks.com/content/repositories/releases/ - - - hortonworks-public - https://repo.hortonworks.com/content/groups/public - - - - - - - - - com.google.protobuf - protobuf-java - ${protobuf2.version} - true - - - com.google.guava - guava - ${guava.version} - true - - - - - - - - - - org.apache.hadoop - hadoop-common - ${hadoop.version} - true - - - org.apache.hadoop - hadoop-client - ${hadoop.version} - true - - - org.apache.hadoop - hadoop-azure - ${hadoop.version} - true - - - - - org.apache.hbase - hbase-server - ${hbase.version} - true - - - - - org.apache.phoenix - phoenix-core - ${phoenix.version} - true - - - - - - - - org.apache.hadoop - hadoop-minicluster - ${hadoop.version} - true - - - org.apache.hbase - hbase-testing-util - ${hbase.version} - test-jar - true - - - org.apache.hbase - hbase-it - ${hbase.version} - test-jar - true - - - - - - - - - - org.apache.htrace - htrace-core - 3.1.0-incubating - - - commons-cli - commons-cli - 1.2 - - - - - org.apache.avro - avro - - - org.apache.avro - avro-ipc - - - org.apache.avro - avro-mapred - hadoop2 - - - - - - - - - commons-lang - commons-lang - - - commons-logging - commons-logging - - - commons-io - commons-io - ${commons-io.version} - - - commons-net - commons-net - ${commons-net.version} - - - commons-httpclient - commons-httpclient - ${commons-httpclient.version} - - - org.apache.curator - curator-recipes - ${curator.version} - - - org.apache.curator - curator-client - ${curator.version} - - - org.apache.curator - curator-framework - ${curator.version} - - - org.apache.zookeeper - zookeeper - ${zookeeper.version} - - - org.eclipse.jetty - jetty-server - ${jetty.version} - true - - - org.eclipse.jetty - jetty-util - ${jetty.version} - true - - - org.eclipse.jetty - jetty-servlet - ${jetty.version} - true - - - org.eclipse.jetty - jetty-webapp - ${jetty.version} - true - - - org.eclipse.jetty - jetty-security - ${jetty.version} - true - - - org.eclipse.jetty - jetty-http - ${jetty.version} - true - - - org.eclipse.jetty - jetty-io - ${jetty.version} - true - - - - - org.fusesource.leveldbjni - leveldbjni-all - ${leveldbjni-all.version} - - - - - org.apache.directory.server - apacheds-kerberos-codec - compile - ${apacheds-kerberos-codec.version} - - - - - com.microsoft.azure - azure-storage - 2.0.0 - - - - org.mortbay.jetty - jetty-util - ${mortbay.jetty.version} - - - - - com.lmax - disruptor - ${disruptor.version} - - - - - - - - org.apache.commons - commons-math - ${commons-math.version} - - - org.apache.commons - commons-compress - ${commons-compress.version} - - - io.netty - netty-all - ${netty.version} - - - org.jamon - jamon-runtime - ${jamon-runtime.version} - - - org.codehaus.jackson - jackson-core-asl - ${codehaus.jackson.version} - - - org.codehaus.jackson - jackson-mapper-asl - ${codehaus.jackson.version} - - - org.codehaus.jackson - jackson-jaxrs - ${codehaus.jackson.version} - - - org.codehaus.jackson - jackson-xc - ${codehaus.jackson.version} - - - - - - org.antlr - antlr-runtime - ${antlr.version} - - - jline - jline - ${jline.version} - - - sqlline - sqlline - ${sqlline.version} - - - - - - - - - - - - - - - - - - org.apache.httpcomponents - httpclient - 4.0.1 - - - org.iq80.snappy - snappy - ${snappy.version} - - - commons-codec - commons-codec - ${commons-codec.version} - - - commons-collections - commons-collections - ${collections.version} - - - org.apache.commons - commons-csv - ${commons-csv.version} - - - joda-time - joda-time - - - - - org.apache.hbase.thirdparty - hbase-shaded-netty - ${hbase-thirdparty.version} - - true - - - org.apache.tephra - tephra-api - ${apache.tephra.version} - - - org.apache.tephra - tephra-core - ${apache.tephra.version} - - - ch.qos.logback - logback-core - - - ch.qos.logback - logback-classic - - - - - org.apache.tephra - tephra-hbase-compat-1.1 - ${apache.tephra.version} - - - - - - org.apache.commons - commons-configuration2 - 2.1.1 - - - org.apache.commons - commons-lang3 - - - - - - - org.apache.commons - commons-math3 - ${commons-math3.version} - - - - - io.dropwizard.metrics - metrics-core - ${io.dropwizard.metrics-core.version} - - - - - org.apache.htrace - htrace-core4 - 4.1.0-incubating - - - - com.google.re2j - re2j - 1.1 - - - - org.apache.hbase.thirdparty - hbase-shaded-protobuf - ${hbase-thirdparty.version} - - - - org.apache.hbase.thirdparty - hbase-shaded-miscellaneous - ${hbase-thirdparty.version} - - - org.apache.yetus - audience-annotations - 0.5.0 - - - - org.apache.hadoop - hadoop-azure-datalake - ${hadoop.version} - true - - - - org.wildfly.openssl - wildfly-openssl - 1.0.7.Final - - - - - \ No newline at end of file diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml deleted file mode 100644 index b35c5350d64..00000000000 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml +++ /dev/null @@ -1,1042 +0,0 @@ - - - - - - 4.0.0 - - - org.opencb.opencga - opencga-storage-hadoop - 3.2.1-SNAPSHOT - ../pom.xml - - - - opencga-storage-hadoop-deps-hdp3.1 - opencga-storage-hadoop-deps-emr6.1 - opencga-storage-hadoop-deps-hdp2.6 - - - - - - - opencga-storage-hadoop-deps - pom - - - true - - - 2.7.3 - 1.1.2 - 4.7.0-HBase-1.1 - - 2.5.0 - 15.0 - - - 3.1 - 1.4.1 - 2.5 - 3.1 - 2.7.1 - 2.0.0-M15 - 1.8 - 3.4.6 - - - 2.2.0 - 3.3.0 - 2.2 - 4.0.23.Final - 2.4.1 - 1.9.13 - 6.1.26 - - - 1.6 - - 2.5 - 1.2 - 1.0 - 1.7 - 3.2.1 - 1.1.9 - 2.11 - 3.5.2 - 0.7.0 - 0.3 - - 1.6 - - - - - - - - - - com.google.protobuf - protobuf-java - ${protobuf2.version} - true - - - com.google.guava - guava - ${guava.version} - true - - - - - - - - - - org.apache.hadoop - hadoop-common - ${hadoop.version} - true - - - org.apache.hadoop - hadoop-client - ${hadoop.version} - true - - - org.apache.hadoop - hadoop-azure - ${hadoop.version} - true - - - org.apache.hadoop - hadoop-common - - - org.apache.commons - commons-lang3 - - - - - - - org.apache.hbase - hbase-server - ${hbase.version} - true - - - org.mortbay.jetty - jetty - - - org.mortbay.jetty - jetty-util - - - org.mortbay.jetty - jsp-2.1 - - - org.mortbay.jetty - jsp-api-2.1 - - - org.mortbay.jetty - servlet-api-2.5 - - - org.mortbay.jetty - jetty-sslengine - - - com.sun.jersey - jersey-core - - - com.sun.jersey - jersey-json - - - com.sun.jersey - jersey-server - - - org.slf4j - slf4j-api - - - org.slf4j - slf4j-log4j12 - - - com.lmax - disruptor - - - - - - - org.apache.phoenix - phoenix-core - ${phoenix.version} - true - - - org.apache.hadoop - hadoop-common - - - org.apache.hadoop - hadoop-client - - - org.apache.hadoop - hadoop-mapreduce-client-core - - - org.apache.hadoop - hadoop-annotations - - - org.apache.hadoop - hadoop-azure - - - org.apache.hbase - hbase-server - - - org.apache.hbase - hbase-client - - - org.apache.hbase - hbase-compact - - - org.apache.hbase - hbase-hadoop-compat - - - org.apache.hbase - hbase-common - - - org.apache.hbase - hbase-protocol - - - org.apache.hbase - hbase-annotations - - - - - - - - - - - org.apache.htrace - htrace-core - 3.1.0-incubating - - - commons-cli - commons-cli - 1.2 - - - - - org.apache.avro - avro - ${avro.version} - - - org.apache.avro - avro-ipc - ${avro.version} - - - org.mortbay.jetty - servlet-api - - - jetty - org.mortbay.jetty - - - jetty-util - org.mortbay.jetty - - - - - org.apache.avro - avro-mapred - hadoop2 - ${avro.version} - - - org.mortbay.jetty - servlet-api - - - jetty - org.mortbay.jetty - - - jetty-util - org.mortbay.jetty - - - - - - - commons-configuration - commons-configuration - 1.6 - - - commons-lang - commons-lang - 2.6 - - - commons-logging - commons-logging - 1.1.3 - - - commons-io - commons-io - ${commons-io.version} - - - commons-net - commons-net - ${commons-net.version} - - - commons-httpclient - commons-httpclient - ${commons-httpclient.version} - - - org.apache.curator - curator-recipes - ${curator.version} - - - org.apache.curator - curator-client - ${curator.version} - - - org.apache.curator - curator-framework - ${curator.version} - - - org.apache.zookeeper - zookeeper - ${zookeeper.version} - - - jline - jline - - - org.jboss.netty - netty - - - - junit - junit - - - com.sun.jdmk - jmxtools - - - com.sun.jmx - jmxri - - - org.slf4j - slf4j-log4j12 - - - log4j - log4j - - - - - - - org.fusesource.leveldbjni - leveldbjni-all - ${leveldbjni-all.version} - - - - - org.apache.directory.server - apacheds-kerberos-codec - compile - ${apacheds-kerberos-codec.version} - - - org.apache.directory.api - api-asn1-ber - - - org.apache.directory.api - api-i18n - - - org.apache.directory.api - api-ldap-model - - - net.sf.ehcache - ehcache-core - - - - - - - com.microsoft.azure - azure-storage - 2.0.0 - - - org.apache.hadoop - hadoop-common - - - org.apache.commons - commons-lang3 - - - - - - org.mortbay.jetty - jetty-util - ${mortbay.jetty.version} - - - org.mortbay.jetty - servlet-api - - - - - - - com.lmax - disruptor - ${disruptor.version} - - - com.yammer.metrics - metrics-core - ${metrics-core.version} - - - org.apache.commons - commons-math - ${commons-math.version} - - - org.apache.commons - commons-compress - ${commons-compress.version} - - - io.netty - netty-all - ${netty.version} - - - org.jamon - jamon-runtime - ${jamon-runtime.version} - - - org.codehaus.jackson - jackson-core-asl - ${codehaus.jackson.version} - - - org.codehaus.jackson - jackson-mapper-asl - ${codehaus.jackson.version} - - - org.codehaus.jackson - jackson-jaxrs - ${codehaus.jackson.version} - - - org.codehaus.jackson - jackson-xc - ${codehaus.jackson.version} - - - - - - org.antlr - antlr-runtime - ${antlr.version} - - - jline - jline - ${jline.version} - - - sqlline - sqlline - ${sqlline.version} - - - co.cask.tephra - tephra-api - ${tephra.version} - - - co.cask.tephra - tephra-core - ${tephra.version} - - - ch.qos.logback - logback-core - - - ch.qos.logback - logback-classic - - - - - co.cask.tephra - tephra-hbase-compat-1.1 - ${tephra.version} - - - org.apache.httpcomponents - httpclient - 4.0.1 - - - org.iq80.snappy - snappy - ${snappy.version} - - - commons-codec - commons-codec - ${commons-codec.version} - - - commons-collections - commons-collections - ${collections.version} - - - org.apache.commons - commons-csv - ${commons-csv.version} - - - joda-time - joda-time - ${joda.version} - - - - - - - - org.apache.hadoop - hadoop-minicluster - ${hadoop.version} - true - - - org.apache.hbase - hbase-testing-util - ${hbase.version} - test-jar - true - - - org.apache.hbase - hbase-it - ${hbase.version} - test-jar - true - - - hbase-server - org.apache.hbase - - - - - - log4j - log4j - 1.2.17 - true - - - - - - - - org.apache.maven.plugins - maven-install-plugin - 2.5.2 - - - my_install - - install - - - - - - - - - - org.apache.maven.plugins - maven-jar-plugin - - - jar - - jar - - - true - - - - test-jar - - test-jar - - - tests - true - - - - - - org.apache.maven.plugins - maven-shade-plugin - - - shade-main-jar - package - - shade - - - shaded - true - - true - true - true - - - - com.sun.jersey:jersey-server - - com/sun/jersey/server/impl/provider/RuntimeDelegateImpl* - - - - com.sun.jersey:jersey-client - - com/sun/ws/rs/ext/RuntimeDelegateImpl* - - - - - com.google.guava:guava - - com/google/common/io/Closeables* - com/google/common/base/Stopwatch* - com/google/common/base/Preconditions* - - - - org.apache.phoenix:phoenix-core - - com/codahale/metrics/* - - - - - - - - - - - - javax.ws.rs - shaded.javax.ws.rs - - - com.google.protobuf - shaded.com.google.protobuf - - - - com.google.common - shaded.com.google.common - - com.google.common.io.Closeables - com.google.common.base.Stopwatch - com.google.common.base.Preconditions - - - - - - - org.eclipse.jetty - shaded.org.eclipse.jetty - - org.eclipse.jetty.server.** - org.eclipse.jetty.webapp.** - org.eclipse.jetty.servlet.** - org.eclipse.jetty.security.** - org.eclipse.jetty.http.** - org.eclipse.jetty.io.** - org.eclipse.jetty.util.** - - - - - - - - - javax.ws.rs - - - com.sun.jersey - - - org.apache.hadoop - - org.apache.hbase - - - - - - - - - - org.apache.phoenix - - - - - com.google.guava - com.google.protobuf - - - org.apache.hbase.thirdparty:hbase-shaded-netty - - org.eclipse.jetty:jetty-server - org.eclipse.jetty:jetty-webapp - org.eclipse.jetty:jetty-servlet - org.eclipse.jetty:jetty-security - org.eclipse.jetty:jetty-http - org.eclipse.jetty:jetty-io - org.eclipse.jetty:jetty-util - - - - org.apache.hbase:**:tests - org.apache.hadoop:hadoop-minicluster - org.apache.hadoop:**:tests - org.mortbay.jetty:** - log4j:** - - - - - - - - - - - - - shade-test-jar - package - - shade - - - true - tests - true - - - - - - - - javax.ws.rs - shaded.javax.ws.rs - - - com.google.protobuf - shaded.com.google.protobuf - - - - com.google.common - shaded.com.google.common - - com.google.common.io.Closeables - com.google.common.base.Stopwatch - - - - - org.eclipse.jetty - shaded.org.eclipse.jetty - - org.eclipse.jetty.server.** - org.eclipse.jetty.webapp.** - org.eclipse.jetty.servlet.** - org.eclipse.jetty.security.** - org.eclipse.jetty.http.** - org.eclipse.jetty.io.** - org.eclipse.jetty.util.** - - - - - - - - - - - - org.apache.hbase:**:tests - org.apache.hadoop:hadoop-minicluster - org.apache.hadoop:**:tests - org.mortbay.jetty:servlet-api - org.mortbay.jetty:jetty - org.mortbay.jetty:jetty-util - org.mortbay.jetty:jetty-sslengine - log4j:** - - - - - - - - - - - - - - org.apache.maven.plugins - maven-dependency-plugin - 3.6.0 - - - analyze - - analyze-only - - - false - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.1/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.1/pom.xml new file mode 100644 index 00000000000..44bbbe4acbf --- /dev/null +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.1/pom.xml @@ -0,0 +1,30 @@ + + + 4.0.0 + + + org.opencb.opencga + opencga-storage-hadoop-lib + 3.2.1-SNAPSHOT + ../pom.xml + + + opencga-storage-hadoop-lib-emr6.1 + + + emr6.1 + hbase2.2 + + 9.4.20.v20190813 + + + + + + maven-assembly-plugin + + + + \ No newline at end of file diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.13/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.13/pom.xml new file mode 100644 index 00000000000..674db7f319c --- /dev/null +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.13/pom.xml @@ -0,0 +1,28 @@ + + + 4.0.0 + + + org.opencb.opencga + opencga-storage-hadoop-lib + 3.2.1-SNAPSHOT + ../pom.xml + + + opencga-storage-hadoop-lib-emr6.13 + + + emr6.13 + hbase2.4 + + + + + + maven-assembly-plugin + + + + \ No newline at end of file diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdi5.1/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdi5.1/pom.xml new file mode 100644 index 00000000000..312d6ce845d --- /dev/null +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdi5.1/pom.xml @@ -0,0 +1,28 @@ + + + 4.0.0 + + + org.opencb.opencga + opencga-storage-hadoop-lib + 3.2.1-SNAPSHOT + ../pom.xml + + + opencga-storage-hadoop-lib-hdi5.1 + + + hdi5.1 + hbase2.4 + + + + + + maven-assembly-plugin + + + + \ No newline at end of file diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdp3.1/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdp3.1/pom.xml new file mode 100644 index 00000000000..3529773711c --- /dev/null +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdp3.1/pom.xml @@ -0,0 +1,29 @@ + + + 4.0.0 + + + org.opencb.opencga + opencga-storage-hadoop-lib + 3.2.1-SNAPSHOT + ../pom.xml + + + opencga-storage-hadoop-lib-hdp3.1 + + + hdp3.1 + hbase2.0 + 9.3.25.v20180904 + + + + + + maven-assembly-plugin + + + + \ No newline at end of file diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/pom.xml new file mode 100644 index 00000000000..9f791862b43 --- /dev/null +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/pom.xml @@ -0,0 +1,148 @@ + + + + + 4.0.0 + + + org.opencb.opencga + opencga-storage-hadoop + 3.2.1-SNAPSHOT + ../pom.xml + + + opencga-storage-hadoop-lib + pom + + + + org.opencb.opencga + opencga-storage-hadoop-core + + + org.opencb.opencga.hadoop.thirdparty + ${opencga-hadoop-shaded.artifactId} + ${opencga.hadoop.thirdparty.version} + + + org.opencb.opencga + opencga-storage-hadoop-compat-${opencga-storage-hadoop-compat.id} + ${project.parent.version} + + + + org.opencb.opencga + opencga-storage-hadoop-core + test-jar + test + + + org.opencb.opencga.hadoop.thirdparty + ${opencga-hadoop-shaded.artifactId} + ${opencga.hadoop.thirdparty.version} + test-jar + test + + + + + + + ../src/main/resources + true + + + + + + + thisHadoopLib + + + !allHadoopLibs + + + + ${opencga-hadoop-lib.artifactId} + + + + allHadoopLibs + + + allHadoopLibs + + + + opencga-storage-hadoop-lib-hdp3.1 + opencga-storage-hadoop-lib-hdi5.1 + opencga-storage-hadoop-lib-emr6.1 + opencga-storage-hadoop-lib-emr6.13 + + + + storage-hadoop + + + !skipStorageHadoop + + + + + + + + maven-assembly-plugin + 3.2.0 + + ${project.parent.basedir} + + + + make-assembly + package + + single + + + + src/main/assembly/src.xml + + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + test-jar + + test-jar + + + + + + + + + + \ No newline at end of file diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/src/main/assembly/src.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/src/main/assembly/src.xml new file mode 100644 index 00000000000..497d6819cf0 --- /dev/null +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/src/main/assembly/src.xml @@ -0,0 +1,39 @@ + + jar-with-dependencies + + jar + + false + + + / + true + + true + runtime + + com.sun.jersey:* + org.eclipse.jetty:* + org.eclipse.jetty.http2:* + org.eclipse.jetty.websocket:* + org.apache.logging.log4j:log4j-slf4j-impl + org.apache.commons:commons-lang3 + org.apache.curator:* + org.apache.tephra:* + org.apache.zookeeper:* + commons-io:commons-io + com.microsoft.azure:* + com.google.inject.extensions:* + com.google.inject:* + com.lmax:disruptor + com.google.errorprone:* + com.google.code.findbugs:* + com.github.stephenc.findbugs:* + io.netty:* + + + + + diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/src/main/resources/org/opencb/opencga/storage/hadoop/lib/version.properties b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/src/main/resources/org/opencb/opencga/storage/hadoop/lib/version.properties new file mode 100644 index 00000000000..1504ca8565f --- /dev/null +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/src/main/resources/org/opencb/opencga/storage/hadoop/lib/version.properties @@ -0,0 +1,2 @@ +opencga-hadoop-shaded.id=${opencga-hadoop-shaded.id} +opencga-hadoop-shaded.artifactId=${opencga-hadoop-shaded.artifactId} diff --git a/opencga-storage/opencga-storage-hadoop/pom.xml b/opencga-storage/opencga-storage-hadoop/pom.xml index 52de24ee0b9..e72f24d30d9 100644 --- a/opencga-storage/opencga-storage-hadoop/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/pom.xml @@ -20,11 +20,6 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - - ${project.parent.version} - false - - org.opencb.opencga opencga-storage @@ -37,94 +32,8 @@ opencga-storage-hadoop-core + opencga-storage-hadoop-compat + opencga-storage-hadoop-lib - - - - opencga-storage-hadoop-deps - - opencga-storage-hadoop-deps - - - - - - - - maven-antrun-plugin - 1.8 - - - compile-hadoop-deps - generate-resources - - - - - - - - - - - - - - - - - - - ${settings.localRepository}/org/opencb/opencga/${opencga-storage-hadoop-deps.artifactId}/${opencga-storage-hadoop-deps.version}/${opencga-storage-hadoop-deps.artifactId}-${opencga-storage-hadoop-deps.version}-shaded.jar - Compile ${opencga-storage-hadoop-deps.artifactId}-${opencga-storage-hadoop-deps.version} - - - - - - - - - - - - - - - - - - - - - - ------------------------------------------------------------------------ - Skip compile opencga-storage-hadoop-deps. If required, compile with -Popencga-storage-hadoop-deps,${opencga-storage-hadoop-deps.id} - Shaded jar file already exists: ${settings.localRepository}/org/opencb/opencga/${opencga-storage-hadoop-deps.artifactId}/${opencga-storage-hadoop-deps.version}/${opencga-storage-hadoop-deps.artifactId}-${opencga-storage-hadoop-deps.version}-shaded.jar - ------------------------------------------------------------------------ - - - - - - - - run - - - - - - - - ant-contrib - ant-contrib - 20020829 - - - - - - \ No newline at end of file diff --git a/opencga-storage/pom.xml b/opencga-storage/pom.xml index 4a47e44df35..83f9de6ce82 100644 --- a/opencga-storage/pom.xml +++ b/opencga-storage/pom.xml @@ -101,7 +101,7 @@ analyze-only - true + false diff --git a/opencga-test/pom.xml b/opencga-test/pom.xml index 2e7253d9889..88dbdeb7783 100644 --- a/opencga-test/pom.xml +++ b/opencga-test/pom.xml @@ -96,38 +96,6 @@ - - org.codehaus.mojo - appassembler-maven-plugin - 1.10 - - - package - - assemble - - - - - libs - flat - false - - conf - - false - ../conf/${opencga.env.file} - - - opencga-test.sh - org.opencb.opencga.test.manager.TestGeneratorManager - - unix - - - - - diff --git a/pom.xml b/pom.xml index 884bb5c1d8a..95f54ac4a10 100644 --- a/pom.xml +++ b/pom.xml @@ -43,35 +43,37 @@ - 2.12.7 - 2.12.7 - 5.8.4 - 2.12.3 - 4.12.0 - 2.12.7 + 3.2.1_dev + 3.2.1_dev + 6.2.1-SNAPSHOT + 3.2.1-SNAPSHOT + 5.2.1-SNAPSHOT + 3.2.1-SNAPSHOT + 0.2.0 - 2.11.4 + 2.14.3 2.30.1 4.4 1.7.7 3.11.4 1.28.1 - 1.7.32 + 1.7.36 2.17.2 - 8.8.2 + 8.11.3 0.11.2 - 9.4.17.v20190418 - 19.0 + + 9.4.53.v20231009 + ${jetty.version} + 28.0-jre 2.23.0 4.3.1 4.8.2 - 3.12.0 - 2.5 + 2.8.0 + 3.14.0 1.7 0.9.10 2.2.27 - 2.6 2.0.1 4.13.2 3.12.1 @@ -87,15 +89,16 @@ 2.3 2.8.0 9.31 - 1.5.4 + 1.7.14 4.0.1 + 1.13.0 true 11.0.0 4.5.6 5.64.4 1.6.1 - 4.3.0 + 6.13.1 3.1.0 1.10.12 4.4.13 @@ -110,24 +113,23 @@ 2.2.0 2.1.0 1.0.0 - 1.1.7.6 + 1.1.8.2 ${parquet-common.version} + 5.0 opencga opencga-env.sh - - hdp3.1 - - - opencga-storage-hadoop-deps-${opencga-storage-hadoop-deps.id-default} - - - ${opencga-storage-hadoop-deps.id-default} - - - opencga-storage-hadoop-deps-${opencga-storage-hadoop-deps.id} - - 5.0 + + 1.0.0-SNAPSHOT + + hdp3.1 + ${opencga-hadoop-shaded.id.default} + opencga-storage-hadoop-lib-${opencga-hadoop-shaded.id} + opencga-hadoop-shaded-${opencga-hadoop-shaded.id} + + hbase2.0 + ${opencga-storage-hadoop-compat.id-default} + opencb https://sonarcloud.io @@ -141,10 +143,16 @@ ${parquet-common.version} 1.7.0 1.1.3 - 3.4.6 1.7.24 2.5-20081211 + + + true 2.23.0 1.18.30 3.4.0 @@ -249,41 +257,34 @@ org.opencb.opencga opencga-storage-app ${project.version} + + + org.opencb.opencga + opencga-storage-hadoop-${opencga-hadoop-shaded.id.default} + + org.opencb.opencga - opencga-storage-core + opencga-test ${project.version} org.opencb.opencga - opencga-storage-hadoop-core + opencga-storage-core ${project.version} - - - org.opencb.opencga - ${opencga-storage-hadoop-deps.artifactId-default} - - org.opencb.opencga opencga-storage-hadoop-core ${project.version} - test-jar - test - - - org.opencb.opencga - ${opencga-storage-hadoop-deps.artifactId-default} - - org.opencb.opencga - opencga-storage-hadoop-deps-hdp3.1 + opencga-storage-hadoop-core ${project.version} - shaded + test-jar + test org.opencb.opencga @@ -307,6 +308,12 @@ org.opencb.opencga opencga-server ${project.version} + + + org.opencb.opencga + opencga-storage-hadoop-${opencga-hadoop-shaded.id.default} + + @@ -405,11 +412,6 @@ commons-lang3 ${commons-lang3.version} - - commons-lang - commons-lang - ${commons-lang26.version} - commons-io commons-io @@ -465,6 +467,11 @@ jackson-dataformat-yaml ${jackson.version} + + com.fasterxml.jackson.dataformat + jackson-dataformat-xml + ${jackson.version} + com.fasterxml.jackson.dataformat jackson-dataformat-cbor @@ -632,6 +639,26 @@ jetty-util ${jetty.version} + + org.eclipse.jetty + jetty-util-ajax + ${jetty.version} + + + org.eclipse.jetty + jetty-xml + ${jetty.version} + + + org.eclipse.jetty + jetty-rewrite + ${jetty.version} + + + org.eclipse.jetty + jetty-security + ${jetty.version} + org.eclipse.jetty jetty-client @@ -652,6 +679,26 @@ jetty-alpn-java-client ${jetty.version} + + org.glassfish.jersey.core + jersey-client + ${jersey.version} + + + org.glassfish.jersey.core + jersey-common + ${jersey.version} + + + org.glassfish.jersey.core + jersey-server + ${jersey.version} + + + org.glassfish.jersey.containers + jersey-container-servlet-core + ${jersey.version} + org.glassfish.jersey.containers jersey-container-servlet @@ -663,8 +710,8 @@ ${jersey.version} - org.glassfish.jersey.core - jersey-client + org.glassfish.jersey.media + jersey-media-json-jackson ${jersey.version} @@ -673,13 +720,8 @@ ${jersey.version} - org.glassfish.jersey.core - jersey-server - ${jersey.version} - - - org.glassfish.jersey.containers - jersey-container-servlet-core + org.glassfish.jersey.media + jersey-media-jaxb ${jersey.version} @@ -779,17 +821,6 @@ bson ${mongodb-driver.version} - - org.apache.zookeeper - zookeeper - ${zookeeper.version} - - - org.slf4j - slf4j-log4j12 - - - commons-logging commons-logging @@ -829,9 +860,6 @@ com.google.guava guava - - - ${guava.version} @@ -948,6 +976,21 @@ kubernetes-model ${kubernetes.version} + + io.fabric8 + kubernetes-model-core + ${kubernetes.version} + + + io.fabric8 + kubernetes-client-api + ${kubernetes.version} + + + io.fabric8 + kubernetes-model-batch + ${kubernetes.version} + javax.servlet javax.servlet-api @@ -968,11 +1011,6 @@ log4j ${log4j.version} - - org.codehaus.jackson - jackson-mapper-asl - ${jackson-codehaus.version} - org.codehaus.jackson jackson-core-asl @@ -1097,11 +1135,6 @@ - - org.glassfish.jersey.media - jersey-media-json-jackson - 2.30.1 - com.esotericsoftware.kryo kryo @@ -1125,7 +1158,7 @@ org.apache.maven.plugins maven-shade-plugin - 2.4.3 + 3.5.1 org.apache.maven.plugins @@ -1335,6 +1368,7 @@ opencga LOCAL + 5 https://ws.opencb.org/opencga-prod @@ -1350,12 +1384,6 @@ 20 - http://localhost:8983/solr/ - 30000 - 2000 - - - false hadoop @@ -1497,29 +1525,57 @@ hdp3.1 + + + hadoop + hdp3.1 + + + + hdp3.1 + hbase2.0 + 9.3.25.v20180904 + + + + hdi5.1 + + + hadoop + hdi5.1 + + - hdp3.1 - - opencga-storage-hadoop-deps-${opencga-storage-hadoop-deps.id} - + hdi5.1 + hbase2.4 emr6.1 + + + hadoop + emr6.1 + + - emr6.1 - - opencga-storage-hadoop-deps-${opencga-storage-hadoop-deps.id} - + emr6.1 + hbase2.2 + + 9.4.20.v20190813 - hdp2.6 + emr6.13 + + + hadoop + emr6.13 + + - hdp2.6 - - opencga-storage-hadoop-deps-${opencga-storage-hadoop-deps.id} - + emr6.13 + hbase2.4